From c583513db1070b2751e4416e162e17b11b857eca Mon Sep 17 00:00:00 2001 From: Dongyang Geng Date: Tue, 12 May 2026 10:54:39 +0800 Subject: [PATCH 01/17] chore: ignore AI tool artifacts --- .gitignore | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.gitignore b/.gitignore index db6c0826ca..c060b2d8b2 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,14 @@ resources/*.xml *.o .vscode cpp/pixels-retina/third_party/ + +# AI tools +.codex +.claude/ +.cursor/ +.continue/ +.aider* +.ai/ +.notes/ +CLAUDE.local.md +AGENTS.md.local From db0554eb4d645bda2a2cb6347c95ee5c4ad9f6be Mon Sep 17 00:00:00 2001 From: Dongyang Geng Date: Tue, 12 May 2026 12:02:44 +0800 Subject: [PATCH 02/17] fix: metadata mutation return values and barrier checks --- .../pixels/cli/executor/CompactExecutor.java | 6 +++++- .../pixels/cli/executor/ImportExecutor.java | 6 +++++- .../pixels/cli/executor/LoadExecutor.java | 5 ++++- .../pixels/cli/load/AbstractPixelsConsumer.java | 10 ++++++++-- .../pixels/common/metadata/MetadataService.java | 6 +++--- .../pixelsdb/pixels/retina/FileWriterManager.java | 10 ++++++++-- .../pixels/retina/StorageGarbageCollector.java | 15 ++++++++++++--- 7 files changed, 45 insertions(+), 13 deletions(-) diff --git a/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/executor/CompactExecutor.java b/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/executor/CompactExecutor.java index ec8c0501c0..b2a6d20281 100644 --- a/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/executor/CompactExecutor.java +++ b/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/executor/CompactExecutor.java @@ -21,6 +21,7 @@ import com.google.common.base.Joiner; import io.pixelsdb.pixels.cli.Main; +import io.pixelsdb.pixels.common.exception.MetadataException; import io.pixelsdb.pixels.common.exception.RetinaException; import io.pixelsdb.pixels.common.metadata.MetadataService; import io.pixelsdb.pixels.common.metadata.domain.Compact; @@ -261,7 +262,10 @@ public void execute(Namespace ns, String command) throws Exception // Issue #192: wait for the compaction to complete. compactExecutor.shutdown(); while (!compactExecutor.awaitTermination(100, TimeUnit.SECONDS)); - metadataService.addFiles(compactFiles); + if (!metadataService.addFiles(compactFiles)) + { + throw new MetadataException("failed to add compact files to metadata"); + } if (retinaService.isEnabled()) { diff --git a/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/executor/ImportExecutor.java b/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/executor/ImportExecutor.java index 140ded28c6..c2c7b8c3b7 100644 --- a/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/executor/ImportExecutor.java +++ b/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/executor/ImportExecutor.java @@ -20,6 +20,7 @@ package io.pixelsdb.pixels.cli.executor; import com.google.common.collect.ImmutableList; +import io.pixelsdb.pixels.common.exception.MetadataException; import io.pixelsdb.pixels.common.metadata.MetadataService; import io.pixelsdb.pixels.common.metadata.domain.File; import io.pixelsdb.pixels.common.metadata.domain.Layout; @@ -67,7 +68,10 @@ public void execute(Namespace ns, String command) throws Exception try { List importFiles = getImportFiles(ordered, writableLayout); - metadataService.addFiles(importFiles); + if (!metadataService.addFiles(importFiles)) + { + throw new MetadataException("failed to import pixels files into metadata"); + } System.out.println(command + " is successful"); } catch (Exception e) diff --git a/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/executor/LoadExecutor.java b/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/executor/LoadExecutor.java index 765f031a39..fde71d3da1 100644 --- a/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/executor/LoadExecutor.java +++ b/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/executor/LoadExecutor.java @@ -93,7 +93,10 @@ public void execute(Namespace ns, String command) throws Exception { File file = loadedInfo.loadedFile; Path path = loadedInfo.loadedPath; - metadataService.updateFile(file); + if (!metadataService.updateFile(file)) + { + throw new MetadataException("failed to publish loaded file " + file.getName()); + } try { diff --git a/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/load/AbstractPixelsConsumer.java b/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/load/AbstractPixelsConsumer.java index cb1d3c32f5..f80459c25b 100644 --- a/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/load/AbstractPixelsConsumer.java +++ b/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/load/AbstractPixelsConsumer.java @@ -167,7 +167,10 @@ private void cleanupTemporaryFiles() { try { - metadataService.deleteFiles(Collections.singletonList((tmpFile.getId()))); + if (!metadataService.deleteFiles(Collections.singletonList((tmpFile.getId())))) + { + throw new MetadataException("failed to delete temporary load file " + tmpFile.getId()); + } } catch (MetadataException e) { e.printStackTrace(); @@ -211,7 +214,10 @@ protected File openTmpFile(String fileName, Path filePath) throws MetadataExcept file.setNumRowGroup(1); file.setPathId(filePath.getId()); String tmpFilePath = filePath.getUri() + "/" + fileName; - this.metadataService.addFiles(Collections.singletonList(file)); + if (!this.metadataService.addFiles(Collections.singletonList(file))) + { + throw new MetadataException("failed to add temporary load file " + tmpFilePath); + } file.setId(metadataService.getFileId(tmpFilePath)); return file; } diff --git a/pixels-common/src/main/java/io/pixelsdb/pixels/common/metadata/MetadataService.java b/pixels-common/src/main/java/io/pixelsdb/pixels/common/metadata/MetadataService.java index 8835f63ac7..3b4b2d6479 100644 --- a/pixels-common/src/main/java/io/pixelsdb/pixels/common/metadata/MetadataService.java +++ b/pixels-common/src/main/java/io/pixelsdb/pixels/common/metadata/MetadataService.java @@ -1361,7 +1361,7 @@ public boolean addFiles(Collection files) throws MetadataException { throw new MetadataException("failed to add file", e); } - return false; + return true; } /** @@ -1476,7 +1476,7 @@ public boolean updateFile(File file) throws MetadataException { throw new MetadataException("failed to update file", e); } - return false; + return true; } public boolean deleteFiles(List fileIds) throws MetadataException @@ -1502,7 +1502,7 @@ public boolean deleteFiles(List fileIds) throws MetadataException { throw new MetadataException("failed to delete files", e); } - return false; + return true; } /** diff --git a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/FileWriterManager.java b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/FileWriterManager.java index f470cb728e..0e6c5cec1f 100644 --- a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/FileWriterManager.java +++ b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/FileWriterManager.java @@ -87,7 +87,10 @@ public FileWriterManager(long tableId, TypeDescription schema, this.file.setType(File.Type.TEMPORARY); this.file.setNumRowGroup(1); this.file.setPathId(targetOrderedDirPath.getId()); - metadataService.addFiles(Collections.singletonList(file)); + if (!metadataService.addFiles(Collections.singletonList(file))) + { + throw new MetadataException("failed to add metadata for ingest file " + targetFilePath); + } this.file.setId(metadataService.getFileId(targetFilePath)); } catch (MetadataException e) { @@ -177,7 +180,10 @@ public CompletableFuture finish() // Update the file's type. this.file.setType(File.Type.REGULAR); MetadataService metadataService = MetadataService.Instance(); - metadataService.updateFile(this.file); + if (!metadataService.updateFile(this.file)) + { + throw new MetadataException("failed to publish ingest file " + this.file.getId() + " as REGULAR"); + } future.complete(null); } catch (Exception e) diff --git a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/StorageGarbageCollector.java b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/StorageGarbageCollector.java index fbc6da0e22..c0b21ec7d8 100644 --- a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/StorageGarbageCollector.java +++ b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/StorageGarbageCollector.java @@ -896,7 +896,10 @@ RewriteResult rewriteFileGroup(FileGroup group, long safeGcTs, newFile.setMinRowId(minRowId); newFile.setMaxRowId(maxRowId); newFile.setPathId(group.files.get(0).file.getPathId()); - metadataService.addFiles(Collections.singletonList(newFile)); + if (!metadataService.addFiles(Collections.singletonList(newFile))) + { + throw new MetadataException("failed to add metadata for GC rewrite file " + newFilePath); + } newFileId = metadataService.getFileId(newFilePath); for (int rgId = 0; rgId < newFileRgCount; rgId++) @@ -939,7 +942,10 @@ private void cleanupTemporaryFile(Storage storage, String newFilePath, } try { - metadataService.deleteFiles(Collections.singletonList(newFileId)); + if (!metadataService.deleteFiles(Collections.singletonList(newFileId))) + { + throw new MetadataException("failed to delete temporary GC catalog entry for fileId=" + newFileId); + } } catch (Exception ex) { @@ -1263,7 +1269,10 @@ void rollback(RewriteResult result) try { - metadataService.deleteFiles(Collections.singletonList(result.newFileId)); + if (!metadataService.deleteFiles(Collections.singletonList(result.newFileId))) + { + throw new MetadataException("failed to rollback GC catalog entry for fileId=" + result.newFileId); + } } catch (Exception ex) { From 206c6fc4547193161688a2e7777b04b5334eb679 Mon Sep 17 00:00:00 2001 From: Dongyang Geng Date: Wed, 13 May 2026 00:05:09 +0800 Subject: [PATCH 03/17] fix: make sqlite main index flushes retryable --- .../pixels/common/index/MainIndexBuffer.java | 71 +- .../pixels-index-main-sqlite/README.md | 156 ++++ .../index/main/sqlite/SqliteMainIndex.java | 308 ++++++- .../main/sqlite/TestSqliteMainIndex.java | 831 +++++++++++++++++- .../sqlite/TestSqliteMainIndexBenchmark.java | 462 ++++++++++ .../main/sqlite/TestSqliteMainIndexQuery.java | 185 +++- 6 files changed, 1909 insertions(+), 104 deletions(-) create mode 100644 pixels-index/pixels-index-main-sqlite/README.md create mode 100644 pixels-index/pixels-index-main-sqlite/src/test/java/io/pixelsdb/pixels/index/main/sqlite/TestSqliteMainIndexBenchmark.java diff --git a/pixels-common/src/main/java/io/pixelsdb/pixels/common/index/MainIndexBuffer.java b/pixels-common/src/main/java/io/pixelsdb/pixels/common/index/MainIndexBuffer.java index 5ee71ba582..e8efb46fc5 100644 --- a/pixels-common/src/main/java/io/pixelsdb/pixels/common/index/MainIndexBuffer.java +++ b/pixels-common/src/main/java/io/pixelsdb/pixels/common/index/MainIndexBuffer.java @@ -53,6 +53,40 @@ public class MainIndexBuffer implements Closeable private final MainIndexCache indexCache; private boolean populateCache = false; + public static final class FlushSnapshot + { + private final long fileId; + private final int entryCount; + private final List rowIdRanges; + + private FlushSnapshot(long fileId, int entryCount, List rowIdRanges) + { + this.fileId = fileId; + this.entryCount = entryCount; + this.rowIdRanges = Collections.unmodifiableList(new ArrayList<>(rowIdRanges)); + } + + public long getFileId() + { + return fileId; + } + + public int getEntryCount() + { + return entryCount; + } + + public List getRowIdRanges() + { + return rowIdRanges; + } + + public boolean isEmpty() + { + return entryCount == 0; + } + } + /** * Create a main index buffer and bind the main index cache to it. * Entries put into this buffer will also be put into the cache. @@ -143,20 +177,19 @@ public IndexProto.RowLocation lookup(long rowId) throws MainIndexException } /** - * Flush the (row id -> row location) mappings of the given file id into ranges and remove them from the buffer. - * This method does not evict the main index cache bind to this buffer as the cached entries are not out of date. - * However, this method may disable synchronous cache population and clear the cache if remaining file ids in the - * buffer is below or equals to the {@link #CACHE_POP_ENABLE_THRESHOLD}. + * Build a stable snapshot of the (row id -> row location) mappings of the given file id. + * This method must not mutate the buffer or cache; callers should only discard the buffered + * entries after the snapshot has been durably committed. * @param fileId the given file id to flush - * @return the flushed row id ranges to be persisited into the storage + * @return the row id range snapshot to be persisted into the storage * @throws MainIndexException */ - public List flush(long fileId) throws MainIndexException + public FlushSnapshot snapshotForFlush(long fileId) throws MainIndexException { Map fileBuffer = this.indexBuffer.get(fileId); if (fileBuffer == null) { - return null; + return new FlushSnapshot(fileId, 0, Collections.emptyList()); } ImmutableList.Builder ranges = ImmutableList.builder(); RowIdRange.Builder currRangeBuilder = new RowIdRange.Builder(); @@ -210,16 +243,34 @@ public List flush(long fileId) throws MainIndexException // release the flushed file index buffer if(fileBuffer.size() != rowIds.length) { - throw new MainIndexException("FileBuffer Changed while flush"); + throw new MainIndexException("FileBuffer changed while building flush snapshot"); + } + return new FlushSnapshot(fileId, rowIds.length, ranges.build()); + } + + /** + * Discard a flush snapshot after the backing store has durably committed it. + * @param snapshot the committed snapshot + * @throws MainIndexException if the buffer no longer matches the committed snapshot + */ + public void discardFlushed(FlushSnapshot snapshot) throws MainIndexException + { + if (snapshot.isEmpty()) + { + return; + } + Map fileBuffer = this.indexBuffer.get(snapshot.getFileId()); + if (fileBuffer == null || fileBuffer.size() != snapshot.getEntryCount()) + { + throw new MainIndexException("FileBuffer changed before committed flush discard"); } fileBuffer.clear(); - this.indexBuffer.remove(fileId); + this.indexBuffer.remove(snapshot.getFileId()); if (this.indexBuffer.size() <= CACHE_POP_ENABLE_THRESHOLD) { this.populateCache = false; this.indexCache.evictAllEntries(); } - return ranges.build(); } public List cachedFileIds() diff --git a/pixels-index/pixels-index-main-sqlite/README.md b/pixels-index/pixels-index-main-sqlite/README.md new file mode 100644 index 0000000000..74d53c74aa --- /dev/null +++ b/pixels-index/pixels-index-main-sqlite/README.md @@ -0,0 +1,156 @@ +# SQLite MainIndex + +This module implements the SQLite-backed `MainIndex`. It stores +`rowId -> RowLocation` mappings as row-id ranges in SQLite and uses a per-file +durable marker to make file-scoped persistence retryable. + +The primary table is `row_id_ranges`. A file-scoped persistence operation writes +the ranges for one file and one row in `row_id_range_flush_markers` in the same +SQLite transaction. The marker records the `file_id`, entry count, range count, +and a deterministic SHA-256 hash of the persisted ranges. + +If a later retry sees a matching marker, the file's ranges are already durable. +If it sees conflicting marker metadata, or ranges without a matching marker, the +backend fails closed instead of silently accepting ambiguous index state. + +## Test Setup + +Commands below assume they are run from the repository root: + +```bash +cd /path/to/pixels +``` + +If you are currently in this module directory, run: + +```bash +cd ../.. +``` + +The root `pom.xml` configures Surefire with `skipTests=true`, so +`mvn test -Dtest=...` still reports `Tests are skipped` for this module. To run +only a few SQLite tests without changing the POM, compile the module first and +then invoke Maven Failsafe directly. Failsafe is not bound by the inherited +Surefire `skipTests=true` setting. + +## Compile The Module + +```bash +mvn -pl pixels-index/pixels-index-main-sqlite -am \ + test-compile +``` + +This compiles the module and its reactor dependencies, including test classes, +but does not execute the JUnit tests. + +## Correctness Tests + +Run the main correctness suite: + +```bash +mvn -pl pixels-index/pixels-index-main-sqlite -am \ + test-compile \ + org.apache.maven.plugins:maven-failsafe-plugin:2.22.2:integration-test \ + org.apache.maven.plugins:maven-failsafe-plugin:2.22.2:verify \ + -Dit.test=TestSqliteMainIndex \ + -DfailIfNoTests=false +``` + +This covers normal put/get/delete behavior and the durable flush marker cases: + +- missing `fileId` flush is a no-op success; +- normal put -> flush -> lookup/delete; +- matching durable marker is accepted as an idempotent retry; +- marker metadata/hash conflicts fail closed and leave buffer retryable; +- dirty ranges without marker fail closed and leave buffer retryable; +- marker insert failure rolls back the range inserts; +- close/reopen flushes cached ranges and keeps rows readable. + +Run the JDBC range query correctness test: + +```bash +mvn -pl pixels-index/pixels-index-main-sqlite -am \ + test-compile \ + org.apache.maven.plugins:maven-failsafe-plugin:2.22.2:integration-test \ + org.apache.maven.plugins:maven-failsafe-plugin:2.22.2:verify \ + -Dit.test=TestSqliteMainIndexQuery \ + -DfailIfNoTests=false +``` + +This test writes a small file-scoped set of entries, flushes it, queries +`row_id_ranges` through JDBC, and asserts the persisted ranges are correct. + +## Performance Benchmark + +The benchmark is not a correctness gate. It is disabled by default and only runs +when explicitly enabled: + +```bash +mvn -pl pixels-index/pixels-index-main-sqlite -am \ + test-compile \ + org.apache.maven.plugins:maven-failsafe-plugin:2.22.2:integration-test \ + org.apache.maven.plugins:maven-failsafe-plugin:2.22.2:verify \ + -Dit.test=TestSqliteMainIndexBenchmark \ + -DfailIfNoTests=false \ + -Dpixels.sqlite.main.index.benchmark=true \ + -Dpixels.sqlite.main.index.benchmark.contiguousRows=1000000 \ + -Dpixels.sqlite.main.index.benchmark.fragmentedRows=10000 +``` + +Parameters: + +- `pixels.sqlite.main.index.benchmark`: must be `true` to run the benchmark. +- `pixels.sqlite.main.index.benchmark.contiguousRows`: row count for contiguous + rowId workloads. Default: `1000000`. +- `pixels.sqlite.main.index.benchmark.fragmentedRows`: row count for fragmented + rowId workloads. Default: `100000`. + +The benchmark prints a parameter block first, for example: + +```text +SQLite MainIndex benchmark parameters + -Dpixels.sqlite.main.index.benchmark=true + -Dpixels.sqlite.main.index.benchmark.contiguousRows=1000000 + -Dpixels.sqlite.main.index.benchmark.fragmentedRows=10000 + index.sqlite.path=/tmp/sqlite + java.version=23.0.2 + os.name=Linux + os.arch=amd64 +``` + +Then it prints a summary table: + +```text +SQLite MainIndex benchmark summary +rows = logical MainIndex entries; ranges = persisted row_id_ranges. +markerRetry = retry when a matching per-file durable marker already exists. +emptyRetry = immediate second flush after marker retry discarded the buffer. +workload shape rows ranges markers put(ms) put rows/s flush(ms) flush ranges/s markerRetry(ms) emptyRetry(ms) get(ms) get rows/s +hot put/get path contiguous, pre-flush get 1,000,000 1 1 ... +contiguous first flush contiguous rows -> 1 range 1,000,000 1 1 ... +fragmented first flush 1-row gaps -> many ranges 10,000 10,000 1 ... +marker-hit retry flush matching marker already durable 10,000 10,000 1 ... +``` + +How to read the table: + +- `rows`: logical entries inserted into `MainIndex`. +- `ranges`: persisted `row_id_ranges` count after flush. +- `markers`: persisted `row_id_range_flush_markers` count. +- `put(ms)` / `put rows/s`: in-memory `putEntry` hot path. +- `flush(ms)` / `flush ranges/s`: first durable flush path. +- `markerRetry(ms)`: retry path when SQLite already has a matching durable marker. +- `emptyRetry(ms)`: immediate second flush after marker retry discarded the buffer. +- `get(ms)` / `get rows/s`: lookup cost after the workload setup. + +For durable flush marker overhead, focus on: + +- `contiguous first flush` `flush(ms)`: best-case file flush, many rows become one + range plus one marker. +- `fragmented first flush` `flush(ms)`: many persisted ranges plus one marker. +- `marker-hit retry flush` `markerRetry(ms)`: crash/retry path after the previous + transaction committed but the in-memory buffer was not discarded. + +Large fragmented workloads can take much longer than contiguous workloads. That +is expected because `N` fragmented rows produce `N` SQLite ranges, while +contiguous rows often collapse into a single range. diff --git a/pixels-index/pixels-index-main-sqlite/src/main/java/io/pixelsdb/pixels/index/main/sqlite/SqliteMainIndex.java b/pixels-index/pixels-index-main-sqlite/src/main/java/io/pixelsdb/pixels/index/main/sqlite/SqliteMainIndex.java index be60cbf016..6958692a18 100644 --- a/pixels-index/pixels-index-main-sqlite/src/main/java/io/pixelsdb/pixels/index/main/sqlite/SqliteMainIndex.java +++ b/pixels-index/pixels-index-main-sqlite/src/main/java/io/pixelsdb/pixels/index/main/sqlite/SqliteMainIndex.java @@ -36,7 +36,10 @@ import java.io.File; import java.io.IOException; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.sql.*; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -64,6 +67,13 @@ public class SqliteMainIndex implements MainIndex "(row_id_start BIGINT NOT NULL, row_id_end BIGINT NOT NULL, file_id BIGINT NOT NULL, rg_id INT NOT NULL," + "rg_row_offset_start INT NOT NULL, rg_row_offset_end INT NOT NULL, PRIMARY KEY (row_id_start, row_id_end))"; + /** + * The SQL statement to create the per-file flush marker table. + */ + private static final String createFlushMarkerTableSql = "CREATE TABLE IF NOT EXISTS row_id_range_flush_markers " + + "(file_id BIGINT NOT NULL PRIMARY KEY, entry_count BIGINT NOT NULL, range_count BIGINT NOT NULL, " + + "range_hash BLOB NOT NULL, committed_at_ms BIGINT NOT NULL)"; + /** * The SQL statement to query the row id range that covers the given row id (the two ? are of the same value). */ @@ -85,6 +95,42 @@ public class SqliteMainIndex implements MainIndex */ private static final String insertRangeSql = "INSERT INTO row_id_ranges VALUES(?, ?, ?, ?, ?, ?)"; + /** + * The SQL statement to query a per-file flush marker. + */ + private static final String queryFlushMarkerSql = + "SELECT entry_count, range_count, range_hash FROM row_id_range_flush_markers WHERE file_id = ?"; + + /** + * The SQL statement to insert a per-file flush marker. + */ + private static final String insertFlushMarkerSql = + "INSERT INTO row_id_range_flush_markers VALUES(?, ?, ?, ?, ?)"; + + private static final class FlushMarker + { + private final long fileId; + private final long entryCount; + private final long rangeCount; + private final byte[] rangeHash; + + private FlushMarker(long fileId, long entryCount, long rangeCount, byte[] rangeHash) + { + this.fileId = fileId; + this.entryCount = entryCount; + this.rangeCount = rangeCount; + this.rangeHash = rangeHash; + } + + private boolean matches(MainIndexBuffer.FlushSnapshot snapshot, byte[] snapshotHash) + { + return this.fileId == snapshot.getFileId() + && this.entryCount == snapshot.getEntryCount() + && this.rangeCount == snapshot.getRowIdRanges().size() + && Arrays.equals(this.rangeHash, snapshotHash); + } + } + private final long tableId; private final String sqlitePath; private final MainIndexBuffer indexBuffer; @@ -116,6 +162,7 @@ public SqliteMainIndex(long tableId, String sqlitePath) throws MainIndexExceptio try (Statement statement = connection.createStatement()) { statement.execute(createTableSql); + statement.execute(createFlushMarkerTableSql); } } catch (SQLException e) @@ -312,31 +359,68 @@ public List putEntries(List primaryEntrie @Override public boolean deleteRowIdRange(RowIdRange rowIdRange) throws MainIndexException { + long rowIdStart = rowIdRange.getRowIdStart(); + long rowIdEnd = rowIdRange.getRowIdEnd(); + if (rowIdEnd <= rowIdStart) + { + throw new MainIndexException("Invalid row id range to delete: [" + rowIdStart + ", " + rowIdEnd + ")"); + } + this.dbRwLock.writeLock().lock(); - try (PreparedStatement pst = connection.prepareStatement(deleteRangesSql)) - { - long rowIdStart = rowIdRange.getRowIdStart(); - long rowIdEnd = rowIdRange.getRowIdEnd(); - pst.setLong(1, rowIdStart); - pst.setLong(2, rowIdEnd); - RowIdRange leftBorderRange = getRowIdRangeFromSqlite(rowIdStart); - RowIdRange rightBorderRange = getRowIdRangeFromSqlite(rowIdEnd - 1); - boolean res = true; - if (leftBorderRange != null) + try + { + boolean originalAutoCommit = this.connection.getAutoCommit(); + try { - int width = (int) (rowIdStart - leftBorderRange.getRowIdStart()); - RowIdRange newLeftBorderRange = leftBorderRange.toBuilder() - .setRowIdEnd(rowIdStart).setRgRowOffsetEnd(leftBorderRange.getRgRowOffsetStart() + width).build(); - res &= updateRowIdRangeWidth(leftBorderRange, newLeftBorderRange); + this.connection.setAutoCommit(false); + RowIdRange leftBorderRange = getRowIdRangeFromSqlite(rowIdStart); + RowIdRange rightBorderRange = getRowIdRangeFromSqlite(rowIdEnd - 1); + boolean res = true; + try (PreparedStatement pst = connection.prepareStatement(deleteRangesSql)) + { + pst.setLong(1, rowIdStart); + pst.setLong(2, rowIdEnd); + pst.executeUpdate(); + } + if (leftBorderRange != null && rightBorderRange != null && + leftBorderRange.getRowIdStart() == rightBorderRange.getRowIdStart() && + leftBorderRange.getRowIdEnd() == rightBorderRange.getRowIdEnd()) + { + res &= trimSingleOverlappingRange(leftBorderRange, rowIdStart, rowIdEnd); + } + else + { + if (leftBorderRange != null && leftBorderRange.getRowIdStart() < rowIdStart && + rowIdStart < leftBorderRange.getRowIdEnd()) + { + int width = (int) (rowIdStart - leftBorderRange.getRowIdStart()); + RowIdRange newLeftBorderRange = leftBorderRange.toBuilder() + .setRowIdEnd(rowIdStart) + .setRgRowOffsetEnd(leftBorderRange.getRgRowOffsetStart() + width).build(); + res &= updateRowIdRangeWidth(leftBorderRange, newLeftBorderRange); + } + if (rightBorderRange != null && rightBorderRange.getRowIdStart() < rowIdEnd && + rowIdEnd < rightBorderRange.getRowIdEnd()) + { + int width = (int) (rightBorderRange.getRowIdEnd() - rowIdEnd); + RowIdRange newRightBorderRange = rightBorderRange.toBuilder() + .setRowIdStart(rowIdEnd) + .setRgRowOffsetStart(rightBorderRange.getRgRowOffsetEnd() - width).build(); + res &= updateRowIdRangeWidth(rightBorderRange, newRightBorderRange); + } + } + this.connection.commit(); + return res; } - if (rightBorderRange != null) + catch (SQLException | RowIdException e) { - int width = (int) (rightBorderRange.getRowIdEnd() - rowIdEnd); - RowIdRange newRightBorderRange = rightBorderRange.toBuilder() - .setRowIdStart(rowIdEnd).setRgRowOffsetStart(rightBorderRange.getRgRowOffsetEnd() - width).build(); - res &= updateRowIdRangeWidth(rightBorderRange, newRightBorderRange); + rollbackQuietly(e); + throw e; + } + finally + { + this.connection.setAutoCommit(originalAutoCommit); } - return res; } catch (SQLException | RowIdException e) { @@ -350,6 +434,46 @@ public boolean deleteRowIdRange(RowIdRange rowIdRange) throws MainIndexException } } + private boolean trimSingleOverlappingRange(RowIdRange range, long rowIdStart, long rowIdEnd) + throws RowIdException, SQLException + { + if (range.getRowIdStart() < rowIdStart && rowIdEnd < range.getRowIdEnd()) + { + int leftWidth = (int) (rowIdStart - range.getRowIdStart()); + RowIdRange newLeftRange = range.toBuilder() + .setRowIdEnd(rowIdStart) + .setRgRowOffsetEnd(range.getRgRowOffsetStart() + leftWidth).build(); + int rightWidth = (int) (range.getRowIdEnd() - rowIdEnd); + RowIdRange newRightRange = range.toBuilder() + .setRowIdStart(rowIdEnd) + .setRgRowOffsetStart(range.getRgRowOffsetEnd() - rightWidth).build(); + boolean res = updateRowIdRangeWidth(range, newLeftRange); + try (PreparedStatement pst = this.connection.prepareStatement(insertRangeSql)) + { + bindRangeInsertStatement(pst, newRightRange); + res &= pst.executeUpdate() > 0; + } + return res; + } + if (range.getRowIdStart() < rowIdStart && rowIdStart < range.getRowIdEnd()) + { + int width = (int) (rowIdStart - range.getRowIdStart()); + RowIdRange newLeftRange = range.toBuilder() + .setRowIdEnd(rowIdStart) + .setRgRowOffsetEnd(range.getRgRowOffsetStart() + width).build(); + return updateRowIdRangeWidth(range, newLeftRange); + } + if (range.getRowIdStart() < rowIdEnd && rowIdEnd < range.getRowIdEnd()) + { + int width = (int) (range.getRowIdEnd() - rowIdEnd); + RowIdRange newRightRange = range.toBuilder() + .setRowIdStart(rowIdEnd) + .setRgRowOffsetStart(range.getRgRowOffsetEnd() - width).build(); + return updateRowIdRangeWidth(range, newRightRange); + } + return true; + } + /** * Get the row id range that contains the given row id from sqlite. * @param rowId the given row id @@ -392,6 +516,16 @@ private RowIdRange getRowIdRangeFromSqlite (long rowId) throws RowIdException } } + private static void bindRangeInsertStatement(PreparedStatement pst, RowIdRange range) throws SQLException + { + pst.setLong(1, range.getRowIdStart()); + pst.setLong(2, range.getRowIdEnd()); + pst.setLong(3, range.getFileId()); + pst.setInt(4, range.getRgId()); + pst.setInt(5, range.getRgRowOffsetStart()); + pst.setInt(6, range.getRgRowOffsetEnd()); + } + /** * Update the width of an existing row id range. * @param oldRange the old row id range @@ -424,22 +558,52 @@ public boolean flushCache(long fileId) throws MainIndexException this.dbRwLock.writeLock().lock(); try { - List rowIdRanges = this.indexBuffer.flush(fileId); - try (PreparedStatement pst = this.connection.prepareStatement(insertRangeSql)) + MainIndexBuffer.FlushSnapshot snapshot = this.indexBuffer.snapshotForFlush(fileId); + if (snapshot.isEmpty()) { - for (RowIdRange range : rowIdRanges) + return true; + } + + byte[] snapshotHash = buildRangeHash(snapshot.getRowIdRanges()); + FlushMarker marker = readFlushMarker(snapshot.getFileId()); + if (marker != null) + { + if (!marker.matches(snapshot, snapshotHash)) { - pst.setLong(1, range.getRowIdStart()); - pst.setLong(2, range.getRowIdEnd()); - pst.setLong(3, range.getFileId()); - pst.setInt(4, range.getRgId()); - pst.setInt(5, range.getRgRowOffsetStart()); - pst.setInt(6, range.getRgRowOffsetEnd()); - pst.addBatch(); + throw new MainIndexException("Conflicting flush marker already exists for fileId=" + fileId); } - pst.executeBatch(); + this.indexBuffer.discardFlushed(snapshot); return true; } + + boolean originalAutoCommit = this.connection.getAutoCommit(); + try + { + this.connection.setAutoCommit(false); + try (PreparedStatement pst = this.connection.prepareStatement(insertRangeSql)) + { + for (RowIdRange range : snapshot.getRowIdRanges()) + { + bindRangeInsertStatement(pst, range); + pst.addBatch(); + } + pst.executeBatch(); + } + insertFlushMarker(snapshot, snapshotHash); + this.connection.commit(); + } + catch (SQLException e) + { + rollbackQuietly(e); + throw e; + } + finally + { + this.connection.setAutoCommit(originalAutoCommit); + } + + this.indexBuffer.discardFlushed(snapshot); + return true; } catch (MainIndexException | SQLException e) { @@ -452,6 +616,86 @@ public boolean flushCache(long fileId) throws MainIndexException } } + private FlushMarker readFlushMarker(long fileId) throws SQLException + { + try (PreparedStatement pst = this.connection.prepareStatement(queryFlushMarkerSql)) + { + pst.setLong(1, fileId); + try (ResultSet rs = pst.executeQuery()) + { + if (!rs.next()) + { + return null; + } + return new FlushMarker(fileId, rs.getLong("entry_count"), + rs.getLong("range_count"), rs.getBytes("range_hash")); + } + } + } + + private void insertFlushMarker(MainIndexBuffer.FlushSnapshot snapshot, byte[] rangeHash) throws SQLException + { + try (PreparedStatement pst = this.connection.prepareStatement(insertFlushMarkerSql)) + { + pst.setLong(1, snapshot.getFileId()); + pst.setLong(2, snapshot.getEntryCount()); + pst.setLong(3, snapshot.getRowIdRanges().size()); + pst.setBytes(4, rangeHash); + pst.setLong(5, System.currentTimeMillis()); + pst.executeUpdate(); + } + } + + private byte[] buildRangeHash(List rowIdRanges) throws MainIndexException + { + try + { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + for (RowIdRange range : rowIdRanges) + { + updateLong(digest, range.getRowIdStart()); + updateLong(digest, range.getRowIdEnd()); + updateLong(digest, range.getFileId()); + updateInt(digest, range.getRgId()); + updateInt(digest, range.getRgRowOffsetStart()); + updateInt(digest, range.getRgRowOffsetEnd()); + } + return digest.digest(); + } + catch (NoSuchAlgorithmException e) + { + throw new MainIndexException("Failed to build range hash for main index flush", e); + } + } + + private static void updateLong(MessageDigest digest, long value) + { + for (int shift = 56; shift >= 0; shift -= 8) + { + digest.update((byte) (value >>> shift)); + } + } + + private static void updateInt(MessageDigest digest, int value) + { + for (int shift = 24; shift >= 0; shift -= 8) + { + digest.update((byte) (value >>> shift)); + } + } + + private void rollbackQuietly(Exception failure) + { + try + { + this.connection.rollback(); + } + catch (SQLException rollbackException) + { + failure.addSuppressed(rollbackException); + } + } + @Override public void close() throws IOException { @@ -517,4 +761,4 @@ public boolean closeAndRemove() throws MainIndexException } return true; } -} \ No newline at end of file +} diff --git a/pixels-index/pixels-index-main-sqlite/src/test/java/io/pixelsdb/pixels/index/main/sqlite/TestSqliteMainIndex.java b/pixels-index/pixels-index-main-sqlite/src/test/java/io/pixelsdb/pixels/index/main/sqlite/TestSqliteMainIndex.java index ddf1a0aae3..9313977b12 100644 --- a/pixels-index/pixels-index-main-sqlite/src/test/java/io/pixelsdb/pixels/index/main/sqlite/TestSqliteMainIndex.java +++ b/pixels-index/pixels-index-main-sqlite/src/test/java/io/pixelsdb/pixels/index/main/sqlite/TestSqliteMainIndex.java @@ -29,11 +29,21 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import java.io.File; import java.io.IOException; +import java.security.MessageDigest; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.Statement; +import java.time.Duration; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; @@ -42,16 +52,19 @@ public class TestSqliteMainIndex { - long tableId = 100L; + private static long nextTableId = 100L; + long tableId; + String sqlitePath; MainIndex mainIndex; @BeforeEach public void setUp() throws MainIndexException { + tableId = nextTableId++; // Create SQLite Directory try { - String sqlitePath = ConfigFactory.Instance().getProperty("index.sqlite.path"); + sqlitePath = ConfigFactory.Instance().getProperty("index.sqlite.path"); FileUtils.forceMkdir(new File(sqlitePath)); } catch (IOException e) @@ -65,12 +78,11 @@ public void setUp() throws MainIndexException @AfterEach public void tearDown() throws Exception { - mainIndex.close(); + MainIndexFactory.Instance().closeIndex(tableId, true); // Clear SQLite Directory try { - String sqlitePath = ConfigFactory.Instance().getProperty("index.sqlite.path"); FileUtils.deleteDirectory(new File(sqlitePath)); } catch (IOException e) @@ -79,6 +91,428 @@ public void tearDown() throws Exception } } + @Test + public void testFlushCacheMissingFileIsNoop() throws MainIndexException + { + Assertions.assertTrue(mainIndex.flushCache(987654321L)); + } + + @Test + public void testFlushCacheAcceptsMatchingCommittedMarker() throws Exception + { + long fileId = 42L; + RowIdRange firstRange = new RowIdRange(5000L, 5002L, fileId, 0, 0, 2); + RowIdRange secondRange = new RowIdRange(5010L, 5011L, fileId, 1, 0, 1); + List ranges = new ArrayList<>(); + ranges.add(firstRange); + ranges.add(secondRange); + putMainIndexEntry(5000L, fileId, 0, 0); + putMainIndexEntry(5001L, fileId, 0, 1); + putMainIndexEntry(5010L, fileId, 1, 0); + + insertRange(firstRange); + insertRange(secondRange); + insertFlushMarker(fileId, 3, ranges); + Assertions.assertEquals(2, countRangesForFile(fileId)); + Assertions.assertEquals(1, countFlushMarkersForFile(fileId)); + + Assertions.assertTrue(mainIndex.flushCache(fileId)); + Assertions.assertEquals(2, countRangesForFile(fileId)); + assertLocation(5000L, fileId, 0, 0); + assertLocation(5001L, fileId, 0, 1); + assertLocation(5010L, fileId, 1, 0); + + Assertions.assertTrue(mainIndex.flushCache(fileId)); + Assertions.assertEquals(2, countRangesForFile(fileId)); + } + + @Test + public void testFlushCacheConflictingMarkerKeepsBufferRetryable() throws Exception + { + long fileId = 43L; + putMainIndexEntry(6000L, fileId, 0, 0); + putMainIndexEntry(6001L, fileId, 0, 1); + putMainIndexEntry(6010L, fileId, 1, 0); + + insertFlushMarker(fileId, 3, new ArrayList<>()); + + Assertions.assertThrows(MainIndexException.class, () -> mainIndex.flushCache(fileId)); + Assertions.assertEquals(0, countExactRanges(6010L, 6011L)); + assertLocation(6000L, fileId, 0, 0); + assertLocation(6010L, fileId, 1, 0); + + deleteFlushMarker(fileId); + Assertions.assertTrue(mainIndex.flushCache(fileId)); + Assertions.assertEquals(2, countRangesForFile(fileId)); + Assertions.assertEquals(1, countFlushMarkersForFile(fileId)); + assertLocation(6000L, fileId, 0, 0); + assertLocation(6010L, fileId, 1, 0); + } + + @Test + public void testFlushCacheRangeWithoutMarkerFailsAndKeepsBufferRetryable() throws Exception + { + long fileId = 44L; + putMainIndexEntry(7000L, fileId, 0, 0); + putMainIndexEntry(7001L, fileId, 0, 1); + putMainIndexEntry(7010L, fileId, 1, 0); + + insertRange(new RowIdRange(7000L, 7002L, fileId, 0, 0, 2)); + + Assertions.assertThrows(MainIndexException.class, () -> mainIndex.flushCache(fileId)); + Assertions.assertEquals(0, countExactRanges(7010L, 7011L)); + Assertions.assertEquals(0, countFlushMarkersForFile(fileId)); + assertLocation(7000L, fileId, 0, 0); + assertLocation(7010L, fileId, 1, 0); + + deleteExactRange(7000L, 7002L); + Assertions.assertTrue(mainIndex.flushCache(fileId)); + Assertions.assertEquals(2, countRangesForFile(fileId)); + Assertions.assertEquals(1, countFlushMarkersForFile(fileId)); + } + + @Test + public void testFlushCacheRejectsFlushMarkerMetadataMismatches() throws Exception + { + long fileId = 45L; + putMainIndexEntry(8000L, fileId, 0, 0); + putMainIndexEntry(8001L, fileId, 0, 1); + + List ranges = Arrays.asList(new RowIdRange(8000L, 8002L, fileId, 0, 0, 2)); + byte[] rangeHash = buildRangeHash(ranges); + + insertFlushMarker(fileId, 1, ranges.size(), rangeHash); + assertFlushFailsAndBufferSurvives(fileId, 8000L, 8001L); + + deleteFlushMarker(fileId); + insertFlushMarker(fileId, 2, ranges.size() + 1, rangeHash); + assertFlushFailsAndBufferSurvives(fileId, 8000L, 8001L); + + deleteFlushMarker(fileId); + byte[] badHash = rangeHash.clone(); + badHash[0] = (byte) (badHash[0] ^ 0x7f); + insertFlushMarker(fileId, 2, ranges.size(), badHash); + assertFlushFailsAndBufferSurvives(fileId, 8000L, 8001L); + + deleteFlushMarker(fileId); + Assertions.assertTrue(mainIndex.flushCache(fileId)); + Assertions.assertEquals(1, countRangesForFile(fileId)); + Assertions.assertEquals(1, countFlushMarkersForFile(fileId)); + } + + @Test + public void testFlushCacheRollsBackRangesWhenMarkerInsertFails() throws Exception + { + long fileId = 46L; + putMainIndexEntry(9000L, fileId, 0, 0); + putMainIndexEntry(9001L, fileId, 0, 1); + putMainIndexEntry(9010L, fileId, 1, 0); + + createFailingFlushMarkerTrigger(fileId); + Assertions.assertThrows(MainIndexException.class, () -> mainIndex.flushCache(fileId)); + Assertions.assertEquals(0, countRangesForFile(fileId)); + Assertions.assertEquals(0, countFlushMarkersForFile(fileId)); + assertLocation(9000L, fileId, 0, 0); + assertLocation(9010L, fileId, 1, 0); + + dropFailingFlushMarkerTrigger(); + Assertions.assertTrue(mainIndex.flushCache(fileId)); + Assertions.assertEquals(2, countRangesForFile(fileId)); + Assertions.assertEquals(1, countFlushMarkersForFile(fileId)); + } + + @Test + public void testFlushCacheConvergesAfterUnknownCommittedStateWithOutOfOrderBuffer() throws Exception + { + long fileId = 48L; + List committedRanges = Arrays.asList( + new RowIdRange(11000L, 11003L, fileId, 0, 0, 3), + new RowIdRange(11010L, 11012L, fileId, 1, 7, 9)); + + putMainIndexEntry(11002L, fileId, 0, 2); + putMainIndexEntry(11000L, fileId, 0, 0); + putMainIndexEntry(11010L, fileId, 1, 7); + putMainIndexEntry(11001L, fileId, 0, 1); + putMainIndexEntry(11011L, fileId, 1, 8); + + for (RowIdRange range : committedRanges) + { + insertRange(range); + } + insertFlushMarker(fileId, 5, committedRanges); + + Assertions.assertTrue(mainIndex.flushCache(fileId)); + Assertions.assertEquals(2, countRangesForFile(fileId)); + Assertions.assertEquals(1, countFlushMarkersForFile(fileId)); + assertNoInvalidRanges(fileId); + assertLocation(11000L, fileId, 0, 0); + assertLocation(11002L, fileId, 0, 2); + assertLocation(11011L, fileId, 1, 8); + } + + @Test + public void testFlushCacheFailureForOneFileDoesNotDiscardOtherFileBuffers() throws Exception + { + long failingFileId = 49L; + long healthyFileId = 50L; + putMainIndexEntry(12000L, failingFileId, 0, 0); + putMainIndexEntry(12001L, failingFileId, 0, 1); + putMainIndexEntry(12100L, healthyFileId, 0, 0); + putMainIndexEntry(12101L, healthyFileId, 0, 1); + + createFailingFlushMarkerTrigger(failingFileId); + Assertions.assertThrows(MainIndexException.class, () -> mainIndex.flushCache(failingFileId)); + Assertions.assertEquals(0, countRangesForFile(failingFileId)); + Assertions.assertEquals(0, countFlushMarkersForFile(failingFileId)); + assertLocation(12000L, failingFileId, 0, 0); + + Assertions.assertTrue(mainIndex.flushCache(healthyFileId)); + Assertions.assertEquals(1, countRangesForFile(healthyFileId)); + Assertions.assertEquals(1, countFlushMarkersForFile(healthyFileId)); + assertLocation(12101L, healthyFileId, 0, 1); + + dropFailingFlushMarkerTrigger(); + Assertions.assertTrue(mainIndex.flushCache(failingFileId)); + Assertions.assertEquals(1, countRangesForFile(failingFileId)); + Assertions.assertEquals(1, countFlushMarkersForFile(failingFileId)); + } + + @Test + public void testPutEntriesFlushesDurableRangesAndLocations() throws Exception + { + long fileId = 51L; + List entries = Arrays.asList( + primaryEntry(13002L, fileId, 0, 2), + primaryEntry(13000L, fileId, 0, 0), + primaryEntry(13001L, fileId, 0, 1), + primaryEntry(13020L, fileId, 2, 4), + primaryEntry(13021L, fileId, 2, 5)); + + assertAllTrue(mainIndex.putEntries(entries)); + Assertions.assertTrue(mainIndex.flushCache(fileId)); + + List ranges = listRangesForFile(fileId); + Assertions.assertEquals(2, ranges.size()); + assertRange(ranges.get(0), 13000L, 13003L, fileId, 0, 0, 3); + assertRange(ranges.get(1), 13020L, 13022L, fileId, 2, 4, 6); + assertNoInvalidRanges(fileId); + + List locations = mainIndex.getLocations(Arrays.asList(13000L, 13002L, 13021L)); + Assertions.assertEquals(3, locations.size()); + Assertions.assertEquals(0, locations.get(0).getRgRowOffset()); + Assertions.assertEquals(2, locations.get(1).getRgRowOffset()); + Assertions.assertEquals(5, locations.get(2).getRgRowOffset()); + } + + @Test + public void testCloseConvergesWhenPreviousFlushCommittedButBufferSurvived() throws Exception + { + long fileId = 52L; + RowIdRange committedRange = new RowIdRange(14000L, 14002L, fileId, 0, 0, 2); + putMainIndexEntry(14000L, fileId, 0, 0); + putMainIndexEntry(14001L, fileId, 0, 1); + + insertRange(committedRange); + insertFlushMarker(fileId, 2, Arrays.asList(committedRange)); + + MainIndexFactory.Instance().closeIndex(tableId, false); + mainIndex = MainIndexFactory.Instance().getMainIndex(tableId); + + Assertions.assertEquals(1, countRangesForFile(fileId)); + Assertions.assertEquals(1, countFlushMarkersForFile(fileId)); + assertLocation(14000L, fileId, 0, 0); + assertLocation(14001L, fileId, 0, 1); + } + + @Test + public void testDeleteRowIdRangeRemovesExactRangeWithoutInvalidResidue() throws Exception + { + long fileId = 53L; + putContiguousEntries(fileId, 0, 15000L, 15004L, 0); + Assertions.assertTrue(mainIndex.flushCache(fileId)); + + Assertions.assertTrue(mainIndex.deleteRowIdRange(new RowIdRange(15000L, 15004L, fileId, 0, 0, 4))); + + Assertions.assertEquals(0, countRangesForFile(fileId)); + assertNoInvalidRanges(fileId); + for (long rowId = 15000L; rowId < 15004L; rowId++) + { + assertLocationMissing(rowId); + } + } + + @Test + public void testDeleteRowIdRangeSplitsMiddleRangeForRecoveryCleanup() throws Exception + { + long fileId = 54L; + putContiguousEntries(fileId, 0, 16000L, 16010L, 0); + Assertions.assertTrue(mainIndex.flushCache(fileId)); + + Assertions.assertTrue(mainIndex.deleteRowIdRange(new RowIdRange(16003L, 16007L, fileId, 0, 3, 7))); + + List ranges = listRangesForFile(fileId); + Assertions.assertEquals(2, ranges.size()); + assertRange(ranges.get(0), 16000L, 16003L, fileId, 0, 0, 3); + assertRange(ranges.get(1), 16007L, 16010L, fileId, 0, 7, 10); + assertNoInvalidRanges(fileId); + assertLocation(16002L, fileId, 0, 2); + assertLocationMissing(16003L); + assertLocationMissing(16006L); + assertLocation(16007L, fileId, 0, 7); + } + + @Test + public void testDeleteRowIdRangeTrimsBordersAndDeletesCoveredRanges() throws Exception + { + long fileId = 55L; + putContiguousEntries(fileId, 0, 17000L, 17005L, 0); + putContiguousEntries(fileId, 1, 17010L, 17015L, 0); + putContiguousEntries(fileId, 2, 17020L, 17025L, 0); + Assertions.assertTrue(mainIndex.flushCache(fileId)); + + Assertions.assertTrue(mainIndex.deleteRowIdRange(new RowIdRange(17003L, 17022L, fileId, 0, 3, 22))); + + List ranges = listRangesForFile(fileId); + Assertions.assertEquals(2, ranges.size()); + assertRange(ranges.get(0), 17000L, 17003L, fileId, 0, 0, 3); + assertRange(ranges.get(1), 17022L, 17025L, fileId, 2, 2, 5); + assertNoInvalidRanges(fileId); + assertLocation(17002L, fileId, 0, 2); + assertLocationMissing(17010L); + assertLocationMissing(17021L); + assertLocation(17022L, fileId, 2, 2); + } + + @Test + public void testDeleteRowIdRangeLeftAlignedTrimsLeadingPortionOfSingleRange() throws Exception + { + long fileId = 60L; + putContiguousEntries(fileId, 0, 21000L, 21010L, 0); + Assertions.assertTrue(mainIndex.flushCache(fileId)); + + // Delete [21000, 21003) which shares its left edge with the existing range [21000, 21010). + // Expected to trim the leading portion and keep [21003, 21010). + Assertions.assertTrue(mainIndex.deleteRowIdRange(new RowIdRange(21000L, 21003L, fileId, 0, 0, 3))); + + List ranges = listRangesForFile(fileId); + Assertions.assertEquals(1, ranges.size()); + assertRange(ranges.get(0), 21003L, 21010L, fileId, 0, 3, 10); + assertNoInvalidRanges(fileId); + assertLocationMissing(21000L); + assertLocationMissing(21002L); + assertLocation(21003L, fileId, 0, 3); + assertLocation(21009L, fileId, 0, 9); + } + + @Test + public void testDeleteRowIdRangeRightAlignedTrimsTrailingPortionOfSingleRange() throws Exception + { + long fileId = 61L; + putContiguousEntries(fileId, 0, 22000L, 22010L, 0); + Assertions.assertTrue(mainIndex.flushCache(fileId)); + + // Delete [22007, 22010) which shares its right edge with the existing range [22000, 22010). + // Expected to trim the trailing portion and keep [22000, 22007). + Assertions.assertTrue(mainIndex.deleteRowIdRange(new RowIdRange(22007L, 22010L, fileId, 0, 7, 10))); + + List ranges = listRangesForFile(fileId); + Assertions.assertEquals(1, ranges.size()); + assertRange(ranges.get(0), 22000L, 22007L, fileId, 0, 0, 7); + assertNoInvalidRanges(fileId); + assertLocation(22000L, fileId, 0, 0); + assertLocation(22006L, fileId, 0, 6); + assertLocationMissing(22007L); + assertLocationMissing(22009L); + } + + @Test + public void testDeleteRowIdRangeFullyContainsSingleRangeRemovesItWithoutResidue() throws Exception + { + long fileId = 62L; + // Single committed range [23000, 23004) sitting in isolation. + putContiguousEntries(fileId, 0, 23000L, 23004L, 0); + Assertions.assertTrue(mainIndex.flushCache(fileId)); + + // Delete [22990, 23010) which strictly contains the entire range. + // No border range is partially overlapped, so the bulk DELETE clause should remove the range + // and leave no residue or split-out ranges. + Assertions.assertTrue(mainIndex.deleteRowIdRange(new RowIdRange(22990L, 23010L, fileId, 0, 0, 20))); + + Assertions.assertEquals(0, countRangesForFile(fileId)); + assertNoInvalidRanges(fileId); + for (long rowId = 23000L; rowId < 23004L; rowId++) + { + assertLocationMissing(rowId); + } + } + + @Test + public void testDeleteRowIdRangeMissingAllRangesIsNoop() throws Exception + { + long fileId = 63L; + // Persist a single range [24000, 24004) so the table is non-empty. + putContiguousEntries(fileId, 0, 24000L, 24004L, 0); + Assertions.assertTrue(mainIndex.flushCache(fileId)); + + // Delete a row id window that does not overlap any committed range; should be a no-op. + Assertions.assertTrue(mainIndex.deleteRowIdRange(new RowIdRange(30000L, 30010L, fileId, 0, 0, 10))); + + List ranges = listRangesForFile(fileId); + Assertions.assertEquals(1, ranges.size()); + assertRange(ranges.get(0), 24000L, 24004L, fileId, 0, 0, 4); + assertNoInvalidRanges(fileId); + assertLocation(24000L, fileId, 0, 0); + assertLocation(24003L, fileId, 0, 3); + // Row ids inside the deleted (but never committed) window remain unknown. + assertLocationMissing(30000L); + assertLocationMissing(30009L); + } + + @Test + public void testDeleteRowIdRangeRollsBackSplitWhenRightRangeInsertFails() throws Exception + { + long fileId = 57L; + putContiguousEntries(fileId, 0, 19000L, 19010L, 0); + Assertions.assertTrue(mainIndex.flushCache(fileId)); + + createFailingRangeInsertTrigger(19007L); + Assertions.assertThrows(MainIndexException.class, + () -> mainIndex.deleteRowIdRange(new RowIdRange(19003L, 19007L, fileId, 0, 3, 7))); + dropFailingRangeInsertTrigger(); + + List ranges = listRangesForFile(fileId); + Assertions.assertEquals(1, ranges.size()); + assertRange(ranges.get(0), 19000L, 19010L, fileId, 0, 0, 10); + assertNoInvalidRanges(fileId); + assertLocation(19003L, fileId, 0, 3); + assertLocation(19007L, fileId, 0, 7); + } + + @Test + public void testDeleteRowIdRangeRejectsEmptyOrReversedRange() throws Exception + { + Assertions.assertThrows(MainIndexException.class, + () -> mainIndex.deleteRowIdRange(new RowIdRange(20000L, 20000L, 58L, 0, 0, 0))); + Assertions.assertThrows(MainIndexException.class, + () -> mainIndex.deleteRowIdRange(new RowIdRange(20001L, 20000L, 58L, 0, 1, 0))); + } + + @Test + public void testCloseFlushesCacheWithMarkerAndReopenReadsRows() throws Exception + { + long fileId = 47L; + putMainIndexEntry(10000L, fileId, 0, 0); + putMainIndexEntry(10001L, fileId, 0, 1); + + MainIndexFactory.Instance().closeIndex(tableId, false); + mainIndex = MainIndexFactory.Instance().getMainIndex(tableId); + + Assertions.assertEquals(1, countRangesForFile(fileId)); + Assertions.assertEquals(1, countFlushMarkersForFile(fileId)); + assertLocation(10000L, fileId, 0, 0); + assertLocation(10001L, fileId, 0, 1); + } + @Test public void testPutAndGetLocation() throws MainIndexException { @@ -95,7 +529,7 @@ public void testPutAndGetLocation() throws MainIndexException } @Test - public void testFlushCacheAndDeleteEntry() throws MainIndexException + public void testFlushCacheAndDeleteEntry() throws Exception { long rowId = 2000L; IndexProto.RowLocation location = IndexProto.RowLocation.newBuilder() @@ -107,52 +541,107 @@ public void testFlushCacheAndDeleteEntry() throws MainIndexException Assertions.assertTrue(mainIndex.deleteRowIdRange(new RowIdRange(rowId, rowId + 1, 2, 2, 0, 1))); - Assertions.assertNull(mainIndex.getLocation(rowId)); + assertLocationMissing(rowId); + Assertions.assertEquals(0, countRangesForFile(2)); + location = location.toBuilder().setFileId(3).build(); Assertions.assertTrue(mainIndex.putEntry(rowId, location)); Assertions.assertNotNull(mainIndex.getLocation(rowId)); - Assertions.assertTrue(mainIndex.flushCache(2)); + Assertions.assertTrue(mainIndex.flushCache(3)); Assertions.assertTrue(mainIndex.deleteRowIdRange(new RowIdRange(rowId - 1, rowId + 1, - 2, 2, 0, 2))); - Assertions.assertNull(mainIndex.getLocation(rowId)); + 3, 2, 0, 2))); + assertLocationMissing(rowId); + Assertions.assertEquals(0, countRangesForFile(3)); + location = location.toBuilder().setFileId(4).build(); Assertions.assertTrue(mainIndex.putEntry(rowId, location)); Assertions.assertNotNull(mainIndex.getLocation(rowId)); - Assertions.assertTrue(mainIndex.flushCache(2)); + Assertions.assertTrue(mainIndex.flushCache(4)); Assertions.assertTrue(mainIndex.deleteRowIdRange(new RowIdRange(rowId - 1, rowId, - 2, 2, 0, 1))); + 4, 2, 0, 1))); Assertions.assertNotNull(mainIndex.getLocation(rowId)); + } - Assertions.assertTrue(mainIndex.deleteRowIdRange(new RowIdRange(rowId, rowId + 1, - 2, 2, 0, 1))); + @Test + @Tag("performance") + public void testFlushCachePerformanceSmoke() throws Exception + { + int entryCount = Integer.getInteger("sqlite.main.index.perf.smoke.entries", 50_000); + long timeoutSeconds = Long.getLong("sqlite.main.index.perf.smoke.timeout.sec", 30L); + long fileId = 56L; + long rowIdBase = 18000L; + long[] elapsedMs = new long[4]; + + Assertions.assertTimeout(Duration.ofSeconds(timeoutSeconds), () -> { + IndexProto.RowLocation.Builder locationBuilder = IndexProto.RowLocation.newBuilder() + .setFileId(fileId).setRgId(0); + long start = System.nanoTime(); + for (int i = 0; i < entryCount; i++) + { + Assertions.assertTrue(mainIndex.putEntry(rowIdBase + i, + locationBuilder.setRgRowOffset(i).build())); + } + elapsedMs[0] = nanosToMillis(System.nanoTime() - start); + + start = System.nanoTime(); + Assertions.assertTrue(mainIndex.flushCache(fileId)); + elapsedMs[1] = nanosToMillis(System.nanoTime() - start); + + start = System.nanoTime(); + int sampleStep = Math.max(1, entryCount / 100); + for (int i = 0; i < entryCount; i += sampleStep) + { + IndexProto.RowLocation location = mainIndex.getLocation(rowIdBase + i); + Assertions.assertEquals(fileId, location.getFileId()); + Assertions.assertEquals(i, location.getRgRowOffset()); + } + elapsedMs[2] = nanosToMillis(System.nanoTime() - start); + + start = System.nanoTime(); + Assertions.assertTrue(mainIndex.flushCache(fileId)); + Assertions.assertTrue(mainIndex.deleteRowIdRange(new RowIdRange( + rowIdBase, rowIdBase + entryCount, fileId, 0, 0, entryCount))); + elapsedMs[3] = nanosToMillis(System.nanoTime() - start); + }); + + Assertions.assertEquals(1, countFlushMarkersForFile(fileId)); + Assertions.assertEquals(0, countRangesForFile(fileId)); + System.out.println("sqlite main index perf smoke entries=" + entryCount + + ", putMs=" + elapsedMs[0] + + ", flushMs=" + elapsedMs[1] + + ", sampledGetMs=" + elapsedMs[2] + + ", idempotentFlushAndDeleteMs=" + elapsedMs[3]); } @Test + @Disabled("Manual performance smoke test; not a correctness gate.") + @Tag("performance") public void testPutAndGetPerformance() throws MainIndexException { final long rowIdBase = 0L; + final int entryCount = Integer.getInteger("sqlite.main.index.perf.entries", 10_000_000); IndexProto.RowLocation.Builder locationBuilder = IndexProto.RowLocation.newBuilder() .setFileId(1L).setRgId(0); long start = System.currentTimeMillis(); - for (int i = 0; i < 10000000; i++) + for (int i = 0; i < entryCount; i++) { mainIndex.putEntry(rowIdBase + i, locationBuilder.setRgRowOffset(i).build()); } - System.out.println("put 10M entries in " + (System.currentTimeMillis() - start) + " ms"); + System.out.println("put " + entryCount + " entries in " + (System.currentTimeMillis() - start) + " ms"); start = System.currentTimeMillis(); - for (int i = 0; i < 10000000; i++) + for (int i = 0; i < entryCount; i++) { mainIndex.getLocation(rowIdBase + i); } - System.out.println("get 10M entries in " + (System.currentTimeMillis() - start) + " ms"); + System.out.println("get " + entryCount + " entries in " + (System.currentTimeMillis() - start) + " ms"); start = System.currentTimeMillis(); mainIndex.flushCache(1); System.out.println("flush cache in " + (System.currentTimeMillis() - start) + " ms"); start = System.currentTimeMillis(); mainIndex.deleteRowIdRange(new RowIdRange( - 0L, 10_000_000L, 1L, 0, 0, 10_000_000)); + 0L, entryCount, 1L, 0, 0, entryCount)); System.out.println("delete all entries in " + (System.currentTimeMillis() - start) + " ms"); } @@ -261,10 +750,10 @@ public void testConcurrentPutAndDeleteRowIds() throws Exception { mainIndex.flushCache(threadNum); RowIdRange range = ranges.get(threadNum); - Assertions.assertTrue(mainIndex.deleteRowIdRange(range)); - for (long id = range.getRowIdStart(); id <= range.getRowIdEnd(); id++) + mainIndex.deleteRowIdRange(range); + for (long id = range.getRowIdStart(); id < range.getRowIdEnd(); id++) { - Assertions.assertNull(mainIndex.getLocation(id)); + assertLocationMissing(id); } } finally @@ -282,4 +771,302 @@ public void testConcurrentPutAndDeleteRowIds() throws Exception } executor.shutdown(); } -} \ No newline at end of file + + private void putMainIndexEntry(long rowId, long fileId, int rgId, int rgRowOffset) + { + Assertions.assertTrue(mainIndex.putEntry(rowId, IndexProto.RowLocation.newBuilder() + .setFileId(fileId).setRgId(rgId).setRgRowOffset(rgRowOffset).build())); + } + + private void putContiguousEntries(long fileId, int rgId, long rowIdStart, long rowIdEnd, int rgRowOffsetStart) + { + int offset = rgRowOffsetStart; + for (long rowId = rowIdStart; rowId < rowIdEnd; rowId++) + { + putMainIndexEntry(rowId, fileId, rgId, offset++); + } + } + + private IndexProto.PrimaryIndexEntry primaryEntry(long rowId, long fileId, int rgId, int rgRowOffset) + { + return IndexProto.PrimaryIndexEntry.newBuilder() + .setRowId(rowId) + .setRowLocation(IndexProto.RowLocation.newBuilder() + .setFileId(fileId).setRgId(rgId).setRgRowOffset(rgRowOffset).build()) + .build(); + } + + private void assertAllTrue(List results) + { + for (Boolean result : results) + { + Assertions.assertTrue(result); + } + } + + private void assertLocation(long rowId, long fileId, int rgId, int rgRowOffset) throws MainIndexException + { + IndexProto.RowLocation location = mainIndex.getLocation(rowId); + Assertions.assertNotNull(location); + Assertions.assertEquals(fileId, location.getFileId()); + Assertions.assertEquals(rgId, location.getRgId()); + Assertions.assertEquals(rgRowOffset, location.getRgRowOffset()); + } + + private void assertLocationMissing(long rowId) + { + Assertions.assertThrows(MainIndexException.class, () -> mainIndex.getLocation(rowId)); + } + + private void assertFlushFailsAndBufferSurvives(long fileId, long firstRowId, long secondRowId) throws Exception + { + Assertions.assertThrows(MainIndexException.class, () -> mainIndex.flushCache(fileId)); + Assertions.assertEquals(0, countRangesForFile(fileId)); + assertLocation(firstRowId, fileId, 0, 0); + assertLocation(secondRowId, fileId, 0, 1); + } + + private void assertRange(RowIdRange range, long rowIdStart, long rowIdEnd, long fileId, + int rgId, int rgRowOffsetStart, int rgRowOffsetEnd) + { + Assertions.assertEquals(rowIdStart, range.getRowIdStart()); + Assertions.assertEquals(rowIdEnd, range.getRowIdEnd()); + Assertions.assertEquals(fileId, range.getFileId()); + Assertions.assertEquals(rgId, range.getRgId()); + Assertions.assertEquals(rgRowOffsetStart, range.getRgRowOffsetStart()); + Assertions.assertEquals(rgRowOffsetEnd, range.getRgRowOffsetEnd()); + } + + private void assertNoInvalidRanges(long fileId) throws Exception + { + Assertions.assertEquals(0, countInvalidRangesForFile(fileId)); + } + + private Connection openMainIndexConnection() throws Exception + { + String path = sqlitePath.endsWith("/") ? sqlitePath : sqlitePath + "/"; + return DriverManager.getConnection("jdbc:sqlite:" + path + tableId + ".main.index.db"); + } + + private void insertRange(RowIdRange range) throws Exception + { + try (Connection connection = openMainIndexConnection(); + PreparedStatement pst = connection.prepareStatement("INSERT INTO row_id_ranges VALUES(?, ?, ?, ?, ?, ?)")) + { + pst.setLong(1, range.getRowIdStart()); + pst.setLong(2, range.getRowIdEnd()); + pst.setLong(3, range.getFileId()); + pst.setInt(4, range.getRgId()); + pst.setInt(5, range.getRgRowOffsetStart()); + pst.setInt(6, range.getRgRowOffsetEnd()); + pst.executeUpdate(); + } + } + + private void deleteExactRange(long rowIdStart, long rowIdEnd) throws Exception + { + try (Connection connection = openMainIndexConnection(); + PreparedStatement pst = connection.prepareStatement( + "DELETE FROM row_id_ranges WHERE row_id_start = ? AND row_id_end = ?")) + { + pst.setLong(1, rowIdStart); + pst.setLong(2, rowIdEnd); + pst.executeUpdate(); + } + } + + private void insertFlushMarker(long fileId, long entryCount, List ranges) throws Exception + { + insertFlushMarker(fileId, entryCount, ranges.size(), buildRangeHash(ranges)); + } + + private void insertFlushMarker(long fileId, long entryCount, long rangeCount, byte[] rangeHash) throws Exception + { + try (Connection connection = openMainIndexConnection(); + PreparedStatement pst = connection.prepareStatement( + "INSERT INTO row_id_range_flush_markers VALUES(?, ?, ?, ?, ?)")) + { + pst.setLong(1, fileId); + pst.setLong(2, entryCount); + pst.setLong(3, rangeCount); + pst.setBytes(4, rangeHash); + pst.setLong(5, System.currentTimeMillis()); + pst.executeUpdate(); + } + } + + private void deleteFlushMarker(long fileId) throws Exception + { + try (Connection connection = openMainIndexConnection(); + PreparedStatement pst = connection.prepareStatement( + "DELETE FROM row_id_range_flush_markers WHERE file_id = ?")) + { + pst.setLong(1, fileId); + pst.executeUpdate(); + } + } + + private void createFailingFlushMarkerTrigger(long fileId) throws Exception + { + try (Connection connection = openMainIndexConnection(); + Statement statement = connection.createStatement()) + { + statement.executeUpdate("DROP TRIGGER IF EXISTS fail_marker_insert"); + statement.executeUpdate("CREATE TRIGGER fail_marker_insert BEFORE INSERT ON row_id_range_flush_markers " + + "WHEN NEW.file_id = " + fileId + " BEGIN SELECT RAISE(ABORT, 'forced marker failure'); END"); + } + } + + private void dropFailingFlushMarkerTrigger() throws Exception + { + try (Connection connection = openMainIndexConnection(); + Statement statement = connection.createStatement()) + { + statement.executeUpdate("DROP TRIGGER IF EXISTS fail_marker_insert"); + } + } + + private void createFailingRangeInsertTrigger(long rowIdStart) throws Exception + { + try (Connection connection = openMainIndexConnection(); + Statement statement = connection.createStatement()) + { + statement.executeUpdate("DROP TRIGGER IF EXISTS fail_range_insert"); + statement.executeUpdate("CREATE TRIGGER fail_range_insert BEFORE INSERT ON row_id_ranges " + + "WHEN NEW.row_id_start = " + rowIdStart + " " + + "BEGIN SELECT RAISE(ABORT, 'forced range insert failure'); END"); + } + } + + private void dropFailingRangeInsertTrigger() throws Exception + { + try (Connection connection = openMainIndexConnection(); + Statement statement = connection.createStatement()) + { + statement.executeUpdate("DROP TRIGGER IF EXISTS fail_range_insert"); + } + } + + private List listRangesForFile(long fileId) throws Exception + { + try (Connection connection = openMainIndexConnection(); + PreparedStatement pst = connection.prepareStatement( + "SELECT * FROM row_id_ranges WHERE file_id = ? ORDER BY row_id_start")) + { + pst.setLong(1, fileId); + List ranges = new ArrayList<>(); + try (ResultSet rs = pst.executeQuery()) + { + while (rs.next()) + { + ranges.add(new RowIdRange( + rs.getLong("row_id_start"), + rs.getLong("row_id_end"), + rs.getLong("file_id"), + rs.getInt("rg_id"), + rs.getInt("rg_row_offset_start"), + rs.getInt("rg_row_offset_end"))); + } + } + return ranges; + } + } + + private int countRangesForFile(long fileId) throws Exception + { + try (Connection connection = openMainIndexConnection(); + PreparedStatement pst = connection.prepareStatement("SELECT COUNT(*) FROM row_id_ranges WHERE file_id = ?")) + { + pst.setLong(1, fileId); + try (ResultSet rs = pst.executeQuery()) + { + Assertions.assertTrue(rs.next()); + return rs.getInt(1); + } + } + } + + private int countFlushMarkersForFile(long fileId) throws Exception + { + try (Connection connection = openMainIndexConnection(); + PreparedStatement pst = connection.prepareStatement( + "SELECT COUNT(*) FROM row_id_range_flush_markers WHERE file_id = ?")) + { + pst.setLong(1, fileId); + try (ResultSet rs = pst.executeQuery()) + { + Assertions.assertTrue(rs.next()); + return rs.getInt(1); + } + } + } + + private int countExactRanges(long rowIdStart, long rowIdEnd) throws Exception + { + try (Connection connection = openMainIndexConnection(); + PreparedStatement pst = connection.prepareStatement( + "SELECT COUNT(*) FROM row_id_ranges WHERE row_id_start = ? AND row_id_end = ?")) + { + pst.setLong(1, rowIdStart); + pst.setLong(2, rowIdEnd); + try (ResultSet rs = pst.executeQuery()) + { + Assertions.assertTrue(rs.next()); + return rs.getInt(1); + } + } + } + + private int countInvalidRangesForFile(long fileId) throws Exception + { + try (Connection connection = openMainIndexConnection(); + PreparedStatement pst = connection.prepareStatement( + "SELECT COUNT(*) FROM row_id_ranges WHERE file_id = ? AND " + + "(row_id_end <= row_id_start OR " + + "(row_id_end - row_id_start) != (rg_row_offset_end - rg_row_offset_start))")) + { + pst.setLong(1, fileId); + try (ResultSet rs = pst.executeQuery()) + { + Assertions.assertTrue(rs.next()); + return rs.getInt(1); + } + } + } + + private byte[] buildRangeHash(List ranges) throws Exception + { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + for (RowIdRange range : ranges) + { + updateLong(digest, range.getRowIdStart()); + updateLong(digest, range.getRowIdEnd()); + updateLong(digest, range.getFileId()); + updateInt(digest, range.getRgId()); + updateInt(digest, range.getRgRowOffsetStart()); + updateInt(digest, range.getRgRowOffsetEnd()); + } + return digest.digest(); + } + + private static void updateLong(MessageDigest digest, long value) + { + for (int shift = 56; shift >= 0; shift -= 8) + { + digest.update((byte) (value >>> shift)); + } + } + + private static void updateInt(MessageDigest digest, int value) + { + for (int shift = 24; shift >= 0; shift -= 8) + { + digest.update((byte) (value >>> shift)); + } + } + + private long nanosToMillis(long nanos) + { + return nanos / 1_000_000L; + } +} diff --git a/pixels-index/pixels-index-main-sqlite/src/test/java/io/pixelsdb/pixels/index/main/sqlite/TestSqliteMainIndexBenchmark.java b/pixels-index/pixels-index-main-sqlite/src/test/java/io/pixelsdb/pixels/index/main/sqlite/TestSqliteMainIndexBenchmark.java new file mode 100644 index 0000000000..d4b07de060 --- /dev/null +++ b/pixels-index/pixels-index-main-sqlite/src/test/java/io/pixelsdb/pixels/index/main/sqlite/TestSqliteMainIndexBenchmark.java @@ -0,0 +1,462 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the Affero GNU General Public + * License along with Pixels. If not, see + * . + */ +package io.pixelsdb.pixels.index.main.sqlite; + +import io.pixelsdb.pixels.common.exception.MainIndexException; +import io.pixelsdb.pixels.common.index.MainIndex; +import io.pixelsdb.pixels.common.index.MainIndexFactory; +import io.pixelsdb.pixels.common.index.RowIdRange; +import io.pixelsdb.pixels.common.utils.ConfigFactory; +import io.pixelsdb.pixels.index.IndexProto; +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.io.IOException; +import java.security.MessageDigest; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; + +@Tag("benchmark") +public class TestSqliteMainIndexBenchmark +{ + private static final String ENABLE_PROPERTY = "pixels.sqlite.main.index.benchmark"; + private static final long NOT_APPLICABLE = -1L; + private static final int CONTIGUOUS_ROWS = Integer.getInteger( + "pixels.sqlite.main.index.benchmark.contiguousRows", 1_000_000); + private static final int FRAGMENTED_ROWS = Integer.getInteger( + "pixels.sqlite.main.index.benchmark.fragmentedRows", 100_000); + private static long nextTableId = 900_000L; + + private String sqlitePath; + private long tableId; + private MainIndex mainIndex; + + @BeforeEach + public void setUp() + { + Assumptions.assumeTrue(Boolean.getBoolean(ENABLE_PROPERTY), + "Set -D" + ENABLE_PROPERTY + "=true to run manual sqlite main-index benchmarks."); + } + + @AfterEach + public void tearDown() throws Exception + { + closeAndRemoveIndex(); + } + + @Test + public void benchmarkPutGetAndFlushPaths() throws Exception + { + System.out.println(); + printBenchmarkParameters(); + List results = new ArrayList<>(); + results.add(benchmarkHotPutGetPath()); + results.add(benchmarkContiguousFlush()); + results.add(benchmarkFragmentedFlush()); + results.add(benchmarkMarkerHitRetry()); + printBenchmarkSummary(results); + } + + private BenchmarkResult benchmarkHotPutGetPath() throws Exception + { + openFreshIndex(); + long fileId = 1L; + long rowIdBase = 1_000_000_000L; + + long putNs = elapsedNanos(() -> putContiguousEntries(CONTIGUOUS_ROWS, fileId, rowIdBase)); + long getNs = elapsedNanos(() -> getContiguousEntries(CONTIGUOUS_ROWS, rowIdBase)); + long cleanupFlushNs = elapsedNanos(() -> Assertions.assertTrue(mainIndex.flushCache(fileId))); + long ranges = countRangesForFile(fileId); + long markers = countFlushMarkersForFile(fileId); + + closeAndRemoveIndex(); + return new BenchmarkResult("hot put/get path", "contiguous, pre-flush get", + CONTIGUOUS_ROWS, ranges, markers, putNs, cleanupFlushNs, + NOT_APPLICABLE, NOT_APPLICABLE, getNs); + } + + private BenchmarkResult benchmarkContiguousFlush() throws Exception + { + openFreshIndex(); + long fileId = 2L; + long rowIdBase = 2_000_000_000L; + + long putNs = elapsedNanos(() -> putContiguousEntries(CONTIGUOUS_ROWS, fileId, rowIdBase)); + long flushNs = elapsedNanos(() -> Assertions.assertTrue(mainIndex.flushCache(fileId))); + long getNs = elapsedNanos(() -> getContiguousEntries(CONTIGUOUS_ROWS, rowIdBase)); + long ranges = countRangesForFile(fileId); + long markers = countFlushMarkersForFile(fileId); + + Assertions.assertEquals(1L, ranges); + Assertions.assertEquals(1L, markers); + closeAndRemoveIndex(); + return new BenchmarkResult("contiguous first flush", "contiguous rows -> 1 range", + CONTIGUOUS_ROWS, ranges, markers, putNs, flushNs, + NOT_APPLICABLE, NOT_APPLICABLE, getNs); + } + + private BenchmarkResult benchmarkFragmentedFlush() throws Exception + { + openFreshIndex(); + long fileId = 3L; + long rowIdBase = 3_000_000_000L; + + long putNs = elapsedNanos(() -> putFragmentedEntries(FRAGMENTED_ROWS, fileId, rowIdBase)); + long flushNs = elapsedNanos(() -> Assertions.assertTrue(mainIndex.flushCache(fileId))); + long getNs = elapsedNanos(() -> getFragmentedEntries(FRAGMENTED_ROWS, rowIdBase)); + long ranges = countRangesForFile(fileId); + long markers = countFlushMarkersForFile(fileId); + + Assertions.assertEquals(FRAGMENTED_ROWS, ranges); + Assertions.assertEquals(1L, markers); + closeAndRemoveIndex(); + return new BenchmarkResult("fragmented first flush", "1-row gaps -> many ranges", + FRAGMENTED_ROWS, ranges, markers, putNs, flushNs, + NOT_APPLICABLE, NOT_APPLICABLE, getNs); + } + + private BenchmarkResult benchmarkMarkerHitRetry() throws Exception + { + openFreshIndex(); + long fileId = 4L; + long rowIdBase = 4_000_000_000L; + List ranges = buildFragmentedRanges(FRAGMENTED_ROWS, fileId, rowIdBase); + + insertRangesAndMarker(fileId, FRAGMENTED_ROWS, ranges); + long putNs = elapsedNanos(() -> putFragmentedEntries(FRAGMENTED_ROWS, fileId, rowIdBase)); + long markerRetryNs = elapsedNanos(() -> Assertions.assertTrue(mainIndex.flushCache(fileId))); + long emptyRetryNs = elapsedNanos(() -> Assertions.assertTrue(mainIndex.flushCache(fileId))); + long getNs = elapsedNanos(() -> getFragmentedEntries(FRAGMENTED_ROWS, rowIdBase)); + long storedRanges = countRangesForFile(fileId); + long markers = countFlushMarkersForFile(fileId); + + Assertions.assertEquals(FRAGMENTED_ROWS, storedRanges); + Assertions.assertEquals(1L, markers); + closeAndRemoveIndex(); + return new BenchmarkResult("marker-hit retry flush", "matching marker already durable", + FRAGMENTED_ROWS, storedRanges, markers, putNs, NOT_APPLICABLE, + markerRetryNs, emptyRetryNs, getNs); + } + + private void openFreshIndex() throws Exception + { + closeAndRemoveIndex(); + this.tableId = nextTableId++; + this.sqlitePath = ConfigFactory.Instance().getProperty("index.sqlite.path"); + try + { + FileUtils.forceMkdir(new File(sqlitePath)); + } + catch (IOException e) + { + throw new MainIndexException("Failed to create SQLite benchmark directory", e); + } + this.mainIndex = MainIndexFactory.Instance().getMainIndex(tableId); + } + + private void closeAndRemoveIndex() throws Exception + { + if (this.mainIndex != null) + { + MainIndexFactory.Instance().closeIndex(this.tableId, true); + this.mainIndex = null; + } + if (this.sqlitePath != null) + { + FileUtils.deleteDirectory(new File(sqlitePath)); + } + } + + private void putContiguousEntries(int rowCount, long fileId, long rowIdBase) + { + IndexProto.RowLocation.Builder locationBuilder = IndexProto.RowLocation.newBuilder() + .setFileId(fileId).setRgId(0); + for (int i = 0; i < rowCount; i++) + { + Assertions.assertTrue(mainIndex.putEntry(rowIdBase + i, locationBuilder.setRgRowOffset(i).build())); + } + } + + private void putFragmentedEntries(int rowCount, long fileId, long rowIdBase) + { + IndexProto.RowLocation.Builder locationBuilder = IndexProto.RowLocation.newBuilder() + .setFileId(fileId).setRgId(0); + for (int i = 0; i < rowCount; i++) + { + Assertions.assertTrue(mainIndex.putEntry(rowIdBase + i * 2L, locationBuilder.setRgRowOffset(i).build())); + } + } + + private void getContiguousEntries(int rowCount, long rowIdBase) throws MainIndexException + { + for (int i = 0; i < rowCount; i++) + { + Assertions.assertNotNull(mainIndex.getLocation(rowIdBase + i)); + } + } + + private void getFragmentedEntries(int rowCount, long rowIdBase) throws MainIndexException + { + for (int i = 0; i < rowCount; i++) + { + Assertions.assertNotNull(mainIndex.getLocation(rowIdBase + i * 2L)); + } + } + + private List buildFragmentedRanges(int rowCount, long fileId, long rowIdBase) + { + List ranges = new ArrayList<>(rowCount); + for (int i = 0; i < rowCount; i++) + { + long rowId = rowIdBase + i * 2L; + ranges.add(new RowIdRange(rowId, rowId + 1, fileId, 0, i, i + 1)); + } + return ranges; + } + + private void insertRangesAndMarker(long fileId, long entryCount, List ranges) throws Exception + { + try (Connection connection = openMainIndexConnection()) + { + boolean originalAutoCommit = connection.getAutoCommit(); + connection.setAutoCommit(false); + try + { + try (PreparedStatement pst = connection.prepareStatement("INSERT INTO row_id_ranges VALUES(?, ?, ?, ?, ?, ?)")) + { + for (RowIdRange range : ranges) + { + pst.setLong(1, range.getRowIdStart()); + pst.setLong(2, range.getRowIdEnd()); + pst.setLong(3, range.getFileId()); + pst.setInt(4, range.getRgId()); + pst.setInt(5, range.getRgRowOffsetStart()); + pst.setInt(6, range.getRgRowOffsetEnd()); + pst.addBatch(); + } + pst.executeBatch(); + } + try (PreparedStatement pst = connection.prepareStatement( + "INSERT INTO row_id_range_flush_markers VALUES(?, ?, ?, ?, ?)")) + { + pst.setLong(1, fileId); + pst.setLong(2, entryCount); + pst.setLong(3, ranges.size()); + pst.setBytes(4, buildRangeHash(ranges)); + pst.setLong(5, System.currentTimeMillis()); + pst.executeUpdate(); + } + connection.commit(); + } + catch (Exception e) + { + connection.rollback(); + throw e; + } + finally + { + connection.setAutoCommit(originalAutoCommit); + } + } + } + + private Connection openMainIndexConnection() throws Exception + { + String path = sqlitePath.endsWith("/") ? sqlitePath : sqlitePath + "/"; + return DriverManager.getConnection("jdbc:sqlite:" + path + tableId + ".main.index.db"); + } + + private long countRangesForFile(long fileId) throws Exception + { + try (Connection connection = openMainIndexConnection(); + PreparedStatement pst = connection.prepareStatement("SELECT COUNT(*) FROM row_id_ranges WHERE file_id = ?")) + { + pst.setLong(1, fileId); + try (ResultSet rs = pst.executeQuery()) + { + Assertions.assertTrue(rs.next()); + return rs.getLong(1); + } + } + } + + private long countFlushMarkersForFile(long fileId) throws Exception + { + try (Connection connection = openMainIndexConnection(); + PreparedStatement pst = connection.prepareStatement( + "SELECT COUNT(*) FROM row_id_range_flush_markers WHERE file_id = ?")) + { + pst.setLong(1, fileId); + try (ResultSet rs = pst.executeQuery()) + { + Assertions.assertTrue(rs.next()); + return rs.getLong(1); + } + } + } + + private byte[] buildRangeHash(List ranges) throws Exception + { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + for (RowIdRange range : ranges) + { + updateLong(digest, range.getRowIdStart()); + updateLong(digest, range.getRowIdEnd()); + updateLong(digest, range.getFileId()); + updateInt(digest, range.getRgId()); + updateInt(digest, range.getRgRowOffsetStart()); + updateInt(digest, range.getRgRowOffsetEnd()); + } + return digest.digest(); + } + + private static void updateLong(MessageDigest digest, long value) + { + for (int shift = 56; shift >= 0; shift -= 8) + { + digest.update((byte) (value >>> shift)); + } + } + + private static void updateInt(MessageDigest digest, int value) + { + for (int shift = 24; shift >= 0; shift -= 8) + { + digest.update((byte) (value >>> shift)); + } + } + + private long elapsedNanos(ThrowingRunnable runnable) throws Exception + { + long start = System.nanoTime(); + runnable.run(); + return System.nanoTime() - start; + } + + private void printBenchmarkParameters() + { + System.out.println("SQLite MainIndex benchmark parameters"); + System.out.println(" -D" + ENABLE_PROPERTY + "=" + Boolean.getBoolean(ENABLE_PROPERTY)); + System.out.println(" -Dpixels.sqlite.main.index.benchmark.contiguousRows=" + CONTIGUOUS_ROWS); + System.out.println(" -Dpixels.sqlite.main.index.benchmark.fragmentedRows=" + FRAGMENTED_ROWS); + System.out.println(" index.sqlite.path=" + ConfigFactory.Instance().getProperty("index.sqlite.path")); + System.out.println(" java.version=" + System.getProperty("java.version")); + System.out.println(" os.name=" + System.getProperty("os.name")); + System.out.println(" os.arch=" + System.getProperty("os.arch")); + } + + private void printBenchmarkSummary(List results) + { + System.out.println(); + System.out.println("SQLite MainIndex benchmark summary"); + System.out.println("rows = logical MainIndex entries; ranges = persisted row_id_ranges."); + System.out.println("markerRetry = retry when a matching per-file durable marker already exists."); + System.out.println("emptyRetry = immediate second flush after marker retry discarded the buffer."); + System.out.println(String.format("%-27s %-31s %12s %10s %7s %10s %13s %10s %16s %15s %13s %10s %13s", + "workload", "shape", "rows", "ranges", "markers", "put(ms)", "put rows/s", + "flush(ms)", "flush ranges/s", "markerRetry(ms)", "emptyRetry(ms)", "get(ms)", "get rows/s")); + for (BenchmarkResult result : results) + { + System.out.println(String.format("%-27s %-31s %12s %10s %7s %10s %13s %10s %16s %15s %13s %10s %13s", + result.name, + result.shape, + formatLong(result.rows), + formatLong(result.ranges), + formatLong(result.markers), + formatMillis(result.putNs), + formatRate(result.rows, result.putNs), + formatMillis(result.flushNs), + formatRate(result.ranges, result.flushNs), + formatMillis(result.markerRetryNs), + formatMillis(result.emptyRetryNs), + formatMillis(result.getNs), + formatRate(result.rows, result.getNs))); + } + } + + private String formatLong(long value) + { + return String.format(Locale.US, "%,d", value); + } + + private String formatMillis(long nanos) + { + if (nanos < 0) + { + return "-"; + } + return String.format(Locale.US, "%,.3f", nanos / 1_000_000.0D); + } + + private String formatRate(long count, long nanos) + { + if (nanos <= 0) + { + return "-"; + } + double rate = count * 1_000_000_000.0D / nanos; + return String.format(Locale.US, "%,.0f", rate); + } + + private static final class BenchmarkResult + { + private final String name; + private final String shape; + private final long rows; + private final long ranges; + private final long markers; + private final long putNs; + private final long flushNs; + private final long markerRetryNs; + private final long emptyRetryNs; + private final long getNs; + + private BenchmarkResult(String name, String shape, long rows, long ranges, long markers, + long putNs, long flushNs, long markerRetryNs, long emptyRetryNs, long getNs) + { + this.name = name; + this.shape = shape; + this.rows = rows; + this.ranges = ranges; + this.markers = markers; + this.putNs = putNs; + this.flushNs = flushNs; + this.markerRetryNs = markerRetryNs; + this.emptyRetryNs = emptyRetryNs; + this.getNs = getNs; + } + } + + private interface ThrowingRunnable + { + void run() throws Exception; + } +} diff --git a/pixels-index/pixels-index-main-sqlite/src/test/java/io/pixelsdb/pixels/index/main/sqlite/TestSqliteMainIndexQuery.java b/pixels-index/pixels-index-main-sqlite/src/test/java/io/pixelsdb/pixels/index/main/sqlite/TestSqliteMainIndexQuery.java index 7847fcd34c..df5bbaaea0 100644 --- a/pixels-index/pixels-index-main-sqlite/src/test/java/io/pixelsdb/pixels/index/main/sqlite/TestSqliteMainIndexQuery.java +++ b/pixels-index/pixels-index-main-sqlite/src/test/java/io/pixelsdb/pixels/index/main/sqlite/TestSqliteMainIndexQuery.java @@ -19,8 +19,6 @@ */ package io.pixelsdb.pixels.index.main.sqlite; -import io.pixelsdb.pixels.common.exception.MainIndexException; -import io.pixelsdb.pixels.common.exception.RowIdException; import io.pixelsdb.pixels.common.index.MainIndex; import io.pixelsdb.pixels.common.index.MainIndexFactory; import io.pixelsdb.pixels.common.index.RowIdRange; @@ -39,63 +37,170 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; + public class TestSqliteMainIndexQuery { + private static long nextTableId = 3035L; + MainIndex mainIndex; - Long tableId =3035L; + long tableId; + String sqlitePath; Connection connection; + @BeforeEach public void setUp() throws Exception { - String sqlitePath = ConfigFactory.Instance().getProperty("index.sqlite.path"); - if (!sqlitePath.endsWith("/")) + tableId = nextTableId++; + sqlitePath = ConfigFactory.Instance().getProperty("index.sqlite.path"); + try { - sqlitePath += "/"; + FileUtils.forceMkdir(new File(sqlitePath)); } + catch (IOException e) + { + System.err.println("Failed to create SQLite test directory: " + e.getMessage()); + } + mainIndex = MainIndexFactory.Instance().getMainIndex(tableId); - connection = DriverManager.getConnection("jdbc:sqlite:" + sqlitePath + tableId + ".main.index.db"); + String path = sqlitePath.endsWith("/") ? sqlitePath : sqlitePath + "/"; + connection = DriverManager.getConnection("jdbc:sqlite:" + path + tableId + ".main.index.db"); + } + + @AfterEach + public void tearDown() throws Exception + { + if (connection != null) + { + connection.close(); + } + MainIndexFactory.Instance().closeIndex(tableId, true); + try + { + FileUtils.deleteDirectory(new File(sqlitePath)); + } + catch (IOException e) + { + System.err.println("Failed to clean up SQLite test directory: " + e.getMessage()); + } + } + + @Test + public void testQueryRowRangesFromCommittedFlush() throws Exception + { + putMainIndexEntry(11000L, 51L, 0, 0); + putMainIndexEntry(11001L, 51L, 0, 1); + putMainIndexEntry(11010L, 51L, 1, 0); + Assertions.assertTrue(mainIndex.flushCache(51L)); + + List rowIdRanges = queryRowRanges(); + Assertions.assertEquals(2, rowIdRanges.size()); + assertRange(rowIdRanges.get(0), 11000L, 11002L, 51L, 0, 0, 2); + assertRange(rowIdRanges.get(1), 11010L, 11011L, 51L, 1, 0, 1); + } + + @Test + public void testQueryRowRangesFromOutOfOrderBatchFlushesMultipleFiles() throws Exception + { + assertAllTrue(mainIndex.putEntries(Arrays.asList( + primaryEntry(11102L, 52L, 0, 2), + primaryEntry(11201L, 53L, 0, 1), + primaryEntry(11100L, 52L, 0, 0), + primaryEntry(11200L, 53L, 0, 0), + primaryEntry(11101L, 52L, 0, 1), + primaryEntry(11202L, 53L, 0, 2)))); + + Assertions.assertTrue(mainIndex.flushCache(53L)); + Assertions.assertTrue(mainIndex.flushCache(52L)); + + List rowIdRanges = queryRowRanges(); + Assertions.assertEquals(2, rowIdRanges.size()); + assertRange(rowIdRanges.get(0), 11100L, 11103L, 52L, 0, 0, 3); + assertRange(rowIdRanges.get(1), 11200L, 11203L, 53L, 0, 0, 3); } @Test - public void testQueryRowRanges() throws Exception + public void testQueryRowRangesReflectDeleteSplitForRecoveryCleanup() throws Exception { - String query = "SELECT * FROM row_id_ranges order by row_id_start"; - long fileid = 0; - try (PreparedStatement pst = this.connection.prepareStatement(query)) + putContiguousEntries(11300L, 11306L, 54L, 0, 0); + Assertions.assertTrue(mainIndex.flushCache(54L)); + + Assertions.assertTrue(mainIndex.deleteRowIdRange(new RowIdRange(11302L, 11305L, 54L, 0, 2, 5))); + + List rowIdRanges = queryRowRanges(); + Assertions.assertEquals(2, rowIdRanges.size()); + assertRange(rowIdRanges.get(0), 11300L, 11302L, 54L, 0, 0, 2); + assertRange(rowIdRanges.get(1), 11305L, 11306L, 54L, 0, 5, 6); + } + + private void putMainIndexEntry(long rowId, long fileId, int rgId, int rgRowOffset) + { + Assertions.assertTrue(mainIndex.putEntry(rowId, IndexProto.RowLocation.newBuilder() + .setFileId(fileId).setRgId(rgId).setRgRowOffset(rgRowOffset).build())); + } + + private void putContiguousEntries(long rowIdStart, long rowIdEnd, long fileId, int rgId, int rgRowOffsetStart) + { + int offset = rgRowOffsetStart; + for (long rowId = rowIdStart; rowId < rowIdEnd; rowId++) { -// pst.setLong(1, fileid); - try (ResultSet rs = pst.executeQuery()) + putMainIndexEntry(rowId, fileId, rgId, offset++); + } + } + + private IndexProto.PrimaryIndexEntry primaryEntry(long rowId, long fileId, int rgId, int rgRowOffset) + { + return IndexProto.PrimaryIndexEntry.newBuilder() + .setRowId(rowId) + .setRowLocation(IndexProto.RowLocation.newBuilder() + .setFileId(fileId).setRgId(rgId).setRgRowOffset(rgRowOffset).build()) + .build(); + } + + private void assertAllTrue(List results) + { + for (Boolean result : results) + { + Assertions.assertTrue(result); + } + } + + private List queryRowRanges() throws Exception + { + String query = "SELECT * FROM row_id_ranges ORDER BY row_id_start"; + List ranges = new ArrayList<>(); + try (PreparedStatement pst = this.connection.prepareStatement(query); + ResultSet rs = pst.executeQuery()) + { + while (rs.next()) { - while (rs.next()) - { - long rowIdStart = rs.getLong("row_id_start"); - long rowIdEnd = rs.getLong("row_id_end"); - long fileId = rs.getLong("file_id"); - int rgId = rs.getInt("rg_id"); - int rgRowOffsetStart = rs.getInt("rg_row_offset_start"); - int rgRowOffsetEnd = rs.getInt("rg_row_offset_end"); - if (rowIdEnd - rowIdStart != rgRowOffsetEnd - rgRowOffsetStart) - { - throw new RowIdException("The width of row id range (" + rowIdStart + ", " + - rgRowOffsetEnd + ") does not match the width of row group row offset range (" + - rgRowOffsetStart + ", " + rgRowOffsetEnd + ")"); - } - System.out.println( - "rowIdStart=" + rowIdStart + - ", rowIdEnd=" + rowIdEnd + - ", fileId=" + fileId + - ", rgId=" + rgId + - ", rgRowOffsetStart=" + rgRowOffsetStart + - ", rgRowOffsetEnd=" + rgRowOffsetEnd - ); - } + long rowIdStart = rs.getLong("row_id_start"); + long rowIdEnd = rs.getLong("row_id_end"); + int rgRowOffsetStart = rs.getInt("rg_row_offset_start"); + int rgRowOffsetEnd = rs.getInt("rg_row_offset_end"); + Assertions.assertEquals(rowIdEnd - rowIdStart, rgRowOffsetEnd - rgRowOffsetStart); + + ranges.add(new RowIdRange( + rowIdStart, + rowIdEnd, + rs.getLong("file_id"), + rs.getInt("rg_id"), + rgRowOffsetStart, + rgRowOffsetEnd)); } } + return ranges; + } + private void assertRange(RowIdRange range, long rowIdStart, long rowIdEnd, long fileId, + int rgId, int rgRowOffsetStart, int rgRowOffsetEnd) + { + Assertions.assertEquals(rowIdStart, range.getRowIdStart()); + Assertions.assertEquals(rowIdEnd, range.getRowIdEnd()); + Assertions.assertEquals(fileId, range.getFileId()); + Assertions.assertEquals(rgId, range.getRgId()); + Assertions.assertEquals(rgRowOffsetStart, range.getRgRowOffsetStart()); + Assertions.assertEquals(rgRowOffsetEnd, range.getRgRowOffsetEnd()); } } From 9931c83aa240016c2c33fb66fd5c5fef0d5ddbdd Mon Sep 17 00:00:00 2001 From: Dongyang Geng Date: Wed, 13 May 2026 10:34:53 +0800 Subject: [PATCH 04/17] fix: RowLocation fileId race in PixelsWriteBuffer --- .../main/java/io/pixelsdb/pixels/retina/PixelsWriteBuffer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/PixelsWriteBuffer.java b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/PixelsWriteBuffer.java index 0b9b47c80f..21b3b40ccd 100644 --- a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/PixelsWriteBuffer.java +++ b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/PixelsWriteBuffer.java @@ -236,7 +236,7 @@ public long addRow(byte[][] values, long timestamp, IndexProto.RowLocation.Build { throw new RetinaException("Expect rgRowOffset >= 0, get " + rgRowOffset); } - builder.setFileId(activeMemTable.getFileId()) + builder.setFileId(currentMemTable.getFileId()) .setRgId(0) .setRgRowOffset(rgRowOffset); return rowId; From 179cdd57a9687ea40e4f1e07ab8223c3068a2688 Mon Sep 17 00:00:00 2001 From: Dongyang Geng Date: Wed, 13 May 2026 13:37:02 +0800 Subject: [PATCH 05/17] fix: publish ingest files after index flush --- .../pixels/retina/FileWriterManager.java | 51 +- .../pixels/retina/PixelsWriteBuffer.java | 138 ++-- .../retina/TestIngestFilePublisher.java | 664 ++++++++++++++++++ 3 files changed, 793 insertions(+), 60 deletions(-) create mode 100644 pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestIngestFilePublisher.java diff --git a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/FileWriterManager.java b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/FileWriterManager.java index 0e6c5cec1f..c66c063177 100644 --- a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/FileWriterManager.java +++ b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/FileWriterManager.java @@ -50,6 +50,9 @@ public class FileWriterManager private final long firstBlockId; private long lastBlockId = -1; private final int virtualNodeId; + // Initialized by PixelsWriteBuffer's single-threaded file publisher. + private CompletableFuture physicalCloseFuture; + /** * Creating pixelsWriter by passing in parameters avoids the need to read * the configuration file for each call. @@ -125,6 +128,17 @@ public FileWriterManager(long tableId, TypeDescription schema, } } + FileWriterManager(long tableId, PixelsWriter writer, File file, + long firstBlockId, long lastBlockId, int virtualNodeId) + { + this.tableId = tableId; + this.writer = writer; + this.file = file; + this.firstBlockId = firstBlockId; + this.lastBlockId = lastBlockId; + this.virtualNodeId = virtualNodeId; + } + public long getFileId() { return this.file.getId(); @@ -145,6 +159,19 @@ public long getLastBlockId() return this.lastBlockId; } + File getFileSnapshot() + { + File snapshot = new File(); + snapshot.setId(this.file.getId()); + snapshot.setName(this.file.getName()); + snapshot.setType(this.file.getType()); + snapshot.setNumRowGroup(this.file.getNumRowGroup()); + snapshot.setMinRowId(this.file.getMinRowId()); + snapshot.setMaxRowId(this.file.getMaxRowId()); + snapshot.setPathId(this.file.getPathId()); + return snapshot; + } + public void addRowBatch(VectorizedRowBatch rowBatch) throws RetinaException { try @@ -158,13 +185,22 @@ public void addRowBatch(VectorizedRowBatch rowBatch) throws RetinaException /** * Create a background thread to write the block of data stored in shared storage to a file. + * Metadata publication is handled by {@link PixelsWriteBuffer} after the + * physical close and index flush barrier both complete. */ - public CompletableFuture finish() + CompletableFuture finish() { + if (physicalCloseFuture != null) + { + return physicalCloseFuture; + } + CompletableFuture future = new CompletableFuture<>(); + physicalCloseFuture = future; new Thread(() -> { - try { + try + { for (long blockId = firstBlockId; blockId <= lastBlockId; ++blockId) { ObjectStorageManager objectStorageManager = ObjectStorageManager.Instance(); @@ -176,21 +212,12 @@ public CompletableFuture finish() this.writer.addRowBatch(VectorizedRowBatch.deserialize(data)); } this.writer.close(); - - // Update the file's type. - this.file.setType(File.Type.REGULAR); - MetadataService metadataService = MetadataService.Instance(); - if (!metadataService.updateFile(this.file)) - { - throw new MetadataException("failed to publish ingest file " + this.file.getId() + " as REGULAR"); - } - future.complete(null); } catch (Exception e) { future.completeExceptionally(e); } - }).start(); + }, "pixels-retina-file-finish-" + this.file.getId()).start(); return future; } diff --git a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/PixelsWriteBuffer.java b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/PixelsWriteBuffer.java index 21b3b40ccd..1880a2be63 100644 --- a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/PixelsWriteBuffer.java +++ b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/PixelsWriteBuffer.java @@ -26,6 +26,7 @@ import io.pixelsdb.pixels.common.index.service.IndexServiceProvider; import io.pixelsdb.pixels.common.index.RowIdAllocator; import io.pixelsdb.pixels.common.metadata.MetadataService; +import io.pixelsdb.pixels.common.metadata.domain.File; import io.pixelsdb.pixels.common.metadata.domain.Path; import io.pixelsdb.pixels.common.metadata.domain.SinglePointIndex; import io.pixelsdb.pixels.common.physical.Storage; @@ -43,7 +44,6 @@ import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; -import java.util.stream.LongStream; import static com.google.common.base.Preconditions.checkArgument; @@ -102,6 +102,7 @@ public class PixelsWriteBuffer // backend flush thread private final ExecutorService flushObjectExecutor; + // Single-threaded by design: it serializes file publishing and FileWriterManager physical close initialization. private final ScheduledExecutorService flushFileExecutor; private ScheduledFuture flushFileFuture; @@ -160,6 +161,7 @@ public PixelsWriteBuffer(long tableId, TypeDescription schema, Path targetOrdere this.objectEntries = new ArrayList<>(); this.flushObjectExecutor = Executors.newFixedThreadPool(Integer.parseInt(configFactory.getProperty("retina.buffer.object.flush.threads"))); + // Keep file publishing serialized: physical close, index flush, metadata publish, and cleanup are ordered per stream. this.flushFileExecutor = Executors.newSingleThreadScheduledExecutor(); this.fileWriterManagers = new ConcurrentLinkedQueue<>(); @@ -368,6 +370,57 @@ public SuperVersion getCurrentVersion() } } + private void publishFinishedFile(FileWriterManager fileWriterManager) throws RetinaException + { + try + { + fileWriterManager.finish().get(); + + if (this.index == null) + { + this.index = MetadataService.Instance().getPrimaryIndex(tableId); + if (this.index == null) + { + throw new RetinaException("Primary index not found for table " + tableId); + } + } + + boolean flushed = IndexServiceProvider.getService(IndexServiceProvider.ServiceMode.local) + .flushIndexEntriesOfFile( + tableId, index.getId(), fileWriterManager.getFileId(), true, indexOption); + if (!flushed) + { + throw new RetinaException("Failed to flush main index for ingest file " + + fileWriterManager.getFileId()); + } + + File regularFile = fileWriterManager.getFileSnapshot(); + regularFile.setType(File.Type.REGULAR); + if (!MetadataService.Instance().updateFile(regularFile)) + { + throw new RetinaException("Failed to publish ingest file " + + fileWriterManager.getFileId() + " as REGULAR"); + } + } catch (InterruptedException e) + { + Thread.currentThread().interrupt(); + throw new RetinaException("Interrupted while publishing ingest file " + + fileWriterManager.getFileId(), e); + } catch (ExecutionException e) + { + throw new RetinaException("Failed to physically close ingest file " + + fileWriterManager.getFileId(), e.getCause()); + } catch (IndexException e) + { + throw new RetinaException("Failed to flush main index for ingest file " + + fileWriterManager.getFileId(), e); + } catch (MetadataException e) + { + throw new RetinaException("Failed to publish ingest file " + + fileWriterManager.getFileId() + " as REGULAR", e); + } + } + /** * Determine whether the last data block managed by fileWriterManager has * been written to Object. If it has been written, execute the file write @@ -378,47 +431,42 @@ private void startFlushObjectToFileScheduler(long intervalSeconds) this.flushFileFuture = this.flushFileExecutor.scheduleWithFixedDelay(() -> { try { - if(index == null) - { - try - { - index = MetadataService.Instance().getPrimaryIndex(tableId); - } catch (MetadataException ignored) - { - logger.warn("There isn't primary index on table {}", tableId); - } - } - Iterator iterator = this.fileWriterManagers.iterator(); while (iterator.hasNext()) { FileWriterManager fileWriterManager = iterator.next(); if (fileWriterManager.getLastBlockId() <= this.continuousFlushedId.get()) { - CompletableFuture finished = fileWriterManager.finish(); - iterator.remove(); - - // update super version + publishFinishedFile(fileWriterManager); + + /* + * Detach only the current write-buffer view while holding versionLock. + * Physical object deletion stays outside the lock so storage I/O does + * not run under the SuperVersion write lock. + */ + List toRemove; this.versionLock.writeLock().lock(); - Set idsToRemove = LongStream.rangeClosed(fileWriterManager.getFirstBlockId(), - fileWriterManager.getLastBlockId()).boxed().collect(Collectors.toSet()); - List toRemove = this.objectEntries.stream() - .filter(objectEntry -> idsToRemove.contains(objectEntry.getId())) - .collect(Collectors.toList()); - - this.objectEntries.removeAll(toRemove); - - SuperVersion oldVersion = this.currentVersion; - this.currentVersion = new SuperVersion(this.activeMemTable, this.immutableMemTables, this.objectEntries); - oldVersion.unref(); - this.versionLock.writeLock().unlock(); - - finished.get(); - if(index != null) + try + { + long firstBlockId = fileWriterManager.getFirstBlockId(); + long lastBlockId = fileWriterManager.getLastBlockId(); + toRemove = this.objectEntries.stream() + .filter(objectEntry -> + objectEntry.getId() >= firstBlockId && objectEntry.getId() <= lastBlockId) + .collect(Collectors.toList()); + + this.objectEntries.removeAll(toRemove); + + SuperVersion oldVersion = this.currentVersion; + this.currentVersion = new SuperVersion( + this.activeMemTable, this.immutableMemTables, this.objectEntries); + oldVersion.unref(); + } finally { - IndexServiceProvider.getService(IndexServiceProvider.ServiceMode.local) - .flushIndexEntriesOfFile(tableId, index.getId(), fileWriterManager.getFileId(), true, indexOption); + this.versionLock.writeLock().unlock(); } + + iterator.remove(); for (ObjectEntry objectEntry : toRemove) { if (objectEntry.unref()) @@ -473,7 +521,7 @@ public void close() throws RetinaException } SuperVersion sv = getCurrentVersion(); - List> futures = new ArrayList<>(); + boolean completed = false; try { long maxObjectKey = this.continuousFlushedId.get(); @@ -492,7 +540,7 @@ public void close() throws RetinaException iterator.remove(); } } - this.currentFileWriterManager.finish().get(); + publishFinishedFile(this.currentFileWriterManager); // process the remaining fileWriterManager for (FileWriterManager fileWriterManager : this.fileWriterManagers) @@ -503,7 +551,7 @@ public void close() throws RetinaException // all written to object if (lastBlockId <= maxObjectKey) { - futures.add(fileWriterManager.finish()); + publishFinishedFile(fileWriterManager); } else { // process elements in immutable memTable @@ -521,18 +569,10 @@ public void close() throws RetinaException // elements in object will be processed in finish() later fileWriterManager.setLastBlockId(maxObjectKey); - futures.add(fileWriterManager.finish()); + publishFinishedFile(fileWriterManager); } } - - CompletableFuture all = CompletableFuture.allOf( - futures.toArray(new CompletableFuture[0]) - ); - all.get(15, TimeUnit.SECONDS); - } catch (InterruptedException e) - { - Thread.currentThread().interrupt(); - throw new RetinaException("Data persistence was interrupted during close", e); + completed = true; } catch (Exception e) { throw new RetinaException("Failed to persist data during close operation. Data may be lost", e); @@ -548,8 +588,10 @@ public void close() throws RetinaException for (ObjectEntry objectEntry : sv.getObjectEntries()) { - objectEntry.unref(); - this.objectStorageManager.delete(this.tableId, virtualNodeId, objectEntry.getId()); + if (objectEntry.unref() && completed) + { + this.objectStorageManager.delete(this.tableId, virtualNodeId, objectEntry.getId()); + } } } } diff --git a/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestIngestFilePublisher.java b/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestIngestFilePublisher.java new file mode 100644 index 0000000000..c6f9069015 --- /dev/null +++ b/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestIngestFilePublisher.java @@ -0,0 +1,664 @@ +/* + * Copyright 2026 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the Affero GNU General Public + * License along with Pixels. If not, see + * . + */ +package io.pixelsdb.pixels.retina; + +import io.pixelsdb.pixels.common.metadata.domain.File; +import io.pixelsdb.pixels.common.exception.RetinaException; +import io.pixelsdb.pixels.core.PixelsWriter; +import io.pixelsdb.pixels.core.TypeDescription; +import io.pixelsdb.pixels.core.vector.VectorizedRowBatch; +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class TestIngestFilePublisher +{ + @Test + public void finishClosesPhysicalFileOnlyOnceAndLeavesMetadataTemporary() throws Exception + { + CountingPixelsWriter writer = new CountingPixelsWriter(); + File file = temporaryFile(101L); + FileWriterManager fileWriterManager = testFileWriterManager(writer, file); + + CompletableFuture firstFinish = fileWriterManager.finish(); + CompletableFuture secondFinish = fileWriterManager.finish(); + firstFinish.get(5, TimeUnit.SECONDS); + secondFinish.get(5, TimeUnit.SECONDS); + + assertSame(firstFinish, secondFinish); + assertEquals(1, writer.closeCount.get()); + assertEquals(File.Type.TEMPORARY, file.getType()); + assertEquals(File.Type.TEMPORARY, fileWriterManager.getFileSnapshot().getType()); + assertTrue(firstFinish.isDone()); + assertFalse(firstFinish.isCompletedExceptionally()); + } + + @Test + public void finishFailureIsPropagatedAndDoesNotPublishMetadata() throws Exception + { + IOException closeFailure = new IOException("close failed"); + CountingPixelsWriter writer = new CountingPixelsWriter(null, null, closeFailure, null); + File file = temporaryFile(103L); + FileWriterManager fileWriterManager = testFileWriterManager(writer, file); + + CompletableFuture firstFinish = fileWriterManager.finish(); + try + { + firstFinish.get(5, TimeUnit.SECONDS); + fail("Expected physical close failure"); + } catch (ExecutionException e) + { + assertSame(closeFailure, e.getCause()); + } + + CompletableFuture secondFinish = fileWriterManager.finish(); + assertSame(firstFinish, secondFinish); + assertTrue(secondFinish.isCompletedExceptionally()); + assertEquals(1, writer.closeCount.get()); + assertEquals(File.Type.TEMPORARY, file.getType()); + assertEquals(File.Type.TEMPORARY, fileWriterManager.getFileSnapshot().getType()); + } + + @Test + public void fileSnapshotCopiesCurrentFileMetadata() + { + File file = temporaryFile(202L); + CountingPixelsWriter writer = new CountingPixelsWriter(); + FileWriterManager fileWriterManager = testFileWriterManager(writer, file); + + File snapshot = fileWriterManager.getFileSnapshot(); + + assertEquals(file.getId(), snapshot.getId()); + assertEquals(file.getName(), snapshot.getName()); + assertEquals(file.getType(), snapshot.getType()); + assertEquals(file.getNumRowGroup(), snapshot.getNumRowGroup()); + assertEquals(file.getMinRowId(), snapshot.getMinRowId()); + assertEquals(file.getMaxRowId(), snapshot.getMaxRowId()); + assertEquals(file.getPathId(), snapshot.getPathId()); + } + + @Test + public void fileSnapshotDoesNotExposeInternalFileState() + { + File file = temporaryFile(203L); + CountingPixelsWriter writer = new CountingPixelsWriter(); + FileWriterManager fileWriterManager = testFileWriterManager(writer, file); + File snapshot = fileWriterManager.getFileSnapshot(); + + snapshot.setName("published.pxl"); + snapshot.setType(File.Type.REGULAR); + snapshot.setNumRowGroup(99); + snapshot.setMinRowId(1000); + snapshot.setMaxRowId(2000); + snapshot.setPathId(88L); + + File freshSnapshot = fileWriterManager.getFileSnapshot(); + assertEquals("ingest_203.pxl", file.getName()); + assertEquals(File.Type.TEMPORARY, file.getType()); + assertEquals(1, file.getNumRowGroup()); + assertEquals(0, file.getMinRowId()); + assertEquals(63, file.getMaxRowId()); + assertEquals(9L, file.getPathId()); + assertEquals(file.getName(), freshSnapshot.getName()); + assertEquals(file.getType(), freshSnapshot.getType()); + assertEquals(file.getNumRowGroup(), freshSnapshot.getNumRowGroup()); + assertEquals(file.getMinRowId(), freshSnapshot.getMinRowId()); + assertEquals(file.getMaxRowId(), freshSnapshot.getMaxRowId()); + assertEquals(file.getPathId(), freshSnapshot.getPathId()); + } + + @Test + public void fileSnapshotReflectsMutationsOnUnderlyingFile() + { + File file = temporaryFile(205L); + CountingPixelsWriter writer = new CountingPixelsWriter(); + FileWriterManager fileWriterManager = testFileWriterManager(writer, file); + + File before = fileWriterManager.getFileSnapshot(); + assertEquals(File.Type.TEMPORARY, before.getType()); + assertEquals(63L, before.getMaxRowId()); + + // Mutations on the underlying file (e.g. visibility/row id updates) must be observed + // by snapshots taken afterwards. Snapshots taken earlier must remain unchanged. + file.setMaxRowId(127L); + file.setNumRowGroup(2); + + File after = fileWriterManager.getFileSnapshot(); + assertEquals(127L, after.getMaxRowId()); + assertEquals(2, after.getNumRowGroup()); + // The previously taken snapshot must keep its original values. + assertEquals(63L, before.getMaxRowId()); + assertEquals(1, before.getNumRowGroup()); + } + + @Test + public void gettersExposeConstructorArguments() + { + File file = temporaryFile(301L); + CountingPixelsWriter writer = new CountingPixelsWriter(); + FileWriterManager fileWriterManager = new FileWriterManager(7L, writer, file, 5L, 10L, 0); + + assertEquals(file.getId(), fileWriterManager.getFileId()); + assertEquals(5L, fileWriterManager.getFirstBlockId()); + assertEquals(10L, fileWriterManager.getLastBlockId()); + } + + @Test + public void setLastBlockIdUpdatesGetter() + { + File file = temporaryFile(302L); + CountingPixelsWriter writer = new CountingPixelsWriter(); + FileWriterManager fileWriterManager = new FileWriterManager(1L, writer, file, 0L, 0L, 0); + + fileWriterManager.setLastBlockId(42L); + assertEquals(42L, fileWriterManager.getLastBlockId()); + + // Allow lowering as well, e.g. when shrinking the range during close(). + fileWriterManager.setLastBlockId(-1L); + assertEquals(-1L, fileWriterManager.getLastBlockId()); + } + + @Test + public void addRowBatchSucceedsAndForwardsToWriter() throws Exception + { + CountingPixelsWriter writer = new CountingPixelsWriter(); + File file = temporaryFile(401L); + FileWriterManager fileWriterManager = testFileWriterManager(writer, file); + + fileWriterManager.addRowBatch(null); + fileWriterManager.addRowBatch(null); + fileWriterManager.addRowBatch(null); + + assertEquals(3, writer.addRowBatchCount.get()); + assertEquals(0, writer.closeCount.get()); + assertEquals(File.Type.TEMPORARY, file.getType()); + } + + @Test + public void addRowBatchFailureLeavesManagerUsableForFinish() throws Exception + { + IOException writeFailure = new IOException("write failed"); + CountingPixelsWriter writer = new CountingPixelsWriter(null, null, null, writeFailure); + File file = temporaryFile(402L); + FileWriterManager fileWriterManager = testFileWriterManager(writer, file); + + try + { + fileWriterManager.addRowBatch(null); + fail("Expected row batch write failure"); + } catch (RetinaException e) + { + assertSame(writeFailure, e.getCause()); + } + + // After a failed addRowBatch, finish() must still close the underlying writer exactly once + // and keep the file in TEMPORARY state (publication is the buffer's responsibility). + fileWriterManager.finish().get(5, TimeUnit.SECONDS); + assertEquals(1, writer.closeCount.get()); + assertEquals(File.Type.TEMPORARY, file.getType()); + } + + @SuppressWarnings("unchecked") + @Test + public void finishIsIdempotentUnderConcurrentCallers() throws Exception + { + CountDownLatch closeStarted = new CountDownLatch(1); + CountDownLatch allowClose = new CountDownLatch(1); + CountingPixelsWriter writer = new CountingPixelsWriter(closeStarted, allowClose, null, null); + File file = temporaryFile(501L); + FileWriterManager fileWriterManager = testFileWriterManager(writer, file); + + // Start the first finish() so the close thread is parked inside writer.close(). + CompletableFuture firstFinish = fileWriterManager.finish(); + assertTrue(closeStarted.await(5, TimeUnit.SECONDS)); + + int callerCount = 8; + ExecutorService callers = Executors.newFixedThreadPool(callerCount); + try + { + CountDownLatch readyLatch = new CountDownLatch(callerCount); + CountDownLatch startLatch = new CountDownLatch(1); + Future>[] results = new Future[callerCount]; + for (int i = 0; i < callerCount; ++i) + { + results[i] = callers.submit(() -> { + readyLatch.countDown(); + startLatch.await(); + return fileWriterManager.finish(); + }); + } + assertTrue(readyLatch.await(5, TimeUnit.SECONDS)); + startLatch.countDown(); + + for (Future> result : results) + { + CompletableFuture observed = result.get(5, TimeUnit.SECONDS); + assertSame(firstFinish, observed); + assertFalse(observed.isDone()); + } + } finally + { + allowClose.countDown(); + callers.shutdownNow(); + } + + firstFinish.get(5, TimeUnit.SECONDS); + assertEquals("writer.close() must run at most once even under concurrent finish() calls", + 1, writer.closeCount.get()); + assertEquals(File.Type.TEMPORARY, file.getType()); + } + + @Test + public void finishRunsCloseOnDedicatedNamedThread() throws Exception + { + CountDownLatch closeStarted = new CountDownLatch(1); + CountDownLatch allowClose = new CountDownLatch(1); + ThreadCapturingPixelsWriter writer = new ThreadCapturingPixelsWriter(closeStarted, allowClose); + File file = temporaryFile(601L); + FileWriterManager fileWriterManager = new FileWriterManager(1L, writer, file, 1L, 0L, 0); + + Thread caller = Thread.currentThread(); + CompletableFuture finishFuture = fileWriterManager.finish(); + assertTrue(closeStarted.await(5, TimeUnit.SECONDS)); + + Thread closeThread = writer.closeThread; + assertNotSame("close() must run off the caller thread", caller, closeThread); + assertEquals("pixels-retina-file-finish-" + file.getId(), closeThread.getName()); + + allowClose.countDown(); + finishFuture.get(5, TimeUnit.SECONDS); + } + + @Test + public void finishPropagatesRuntimeExceptionFromClose() throws Exception + { + RuntimeException closeFailure = new RuntimeException("boom"); + CountingPixelsWriter writer = new CountingPixelsWriter(null, null, null, null, + closeFailure); + File file = temporaryFile(701L); + FileWriterManager fileWriterManager = testFileWriterManager(writer, file); + + CompletableFuture firstFinish = fileWriterManager.finish(); + try + { + firstFinish.get(5, TimeUnit.SECONDS); + fail("Expected runtime close failure"); + } catch (ExecutionException e) + { + assertSame(closeFailure, e.getCause()); + } + + // Subsequent calls must keep returning the same failed future and must not retry close(). + CompletableFuture secondFinish = fileWriterManager.finish(); + assertSame(firstFinish, secondFinish); + assertTrue(secondFinish.isCompletedExceptionally()); + assertEquals(1, writer.closeCount.get()); + assertEquals(File.Type.TEMPORARY, file.getType()); + } + + @Test(timeout = 10_000L) + public void finishDoesNotBlockCallerThread() throws Exception + { + CountDownLatch closeStarted = new CountDownLatch(1); + CountDownLatch allowClose = new CountDownLatch(1); + CountingPixelsWriter writer = new CountingPixelsWriter(closeStarted, allowClose, null, null); + File file = temporaryFile(801L); + FileWriterManager fileWriterManager = testFileWriterManager(writer, file); + + long start = System.nanoTime(); + CompletableFuture finishFuture = fileWriterManager.finish(); + long elapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start); + + // The caller thread must return promptly; the actual close() runs on the named thread. + assertTrue("finish() must not block on writer.close(); elapsedMillis=" + elapsedMillis, + elapsedMillis < 2_000L); + assertTrue(closeStarted.await(5, TimeUnit.SECONDS)); + assertFalse(finishFuture.isDone()); + try + { + finishFuture.get(200, TimeUnit.MILLISECONDS); + fail("finish() future must not complete before writer.close() returns"); + } catch (TimeoutException expected) + { + // expected: still in progress + } + allowClose.countDown(); + finishFuture.get(5, TimeUnit.SECONDS); + } + + @Test + public void concurrentAddRowBatchesAreAllForwardedToWriter() throws Exception + { + // FileWriterManager does not perform internal locking around addRowBatch; verify it does + // not lose calls or throw NPEs when several threads forward row batches concurrently. + CountingPixelsWriter writer = new CountingPixelsWriter(); + File file = temporaryFile(1601L); + FileWriterManager fileWriterManager = testFileWriterManager(writer, file); + + int callerCount = 16; + int callsPerCaller = 25; + ExecutorService callers = Executors.newFixedThreadPool(callerCount); + try + { + CountDownLatch startLatch = new CountDownLatch(1); + List> results = new ArrayList<>(callerCount); + for (int i = 0; i < callerCount; ++i) + { + results.add(callers.submit(() -> { + startLatch.await(); + for (int j = 0; j < callsPerCaller; ++j) + { + fileWriterManager.addRowBatch(null); + } + return null; + })); + } + startLatch.countDown(); + for (Future result : results) + { + result.get(10, TimeUnit.SECONDS); + } + } finally + { + callers.shutdownNow(); + } + + assertEquals(callerCount * callsPerCaller, writer.addRowBatchCount.get()); + assertEquals(0, writer.closeCount.get()); + assertEquals(File.Type.TEMPORARY, file.getType()); + } + + @Test + public void finishIsRobustAgainstFileMetadataMutationsBeforeReturn() throws Exception + { + // Mutations on the underlying file (e.g. visibility/row id updates by other components) + // performed while finish() is in progress must not affect the success of physical close, + // and the post-close snapshot must reflect the mutated state because publication has + // not yet rewritten file.type. + CountDownLatch closeStarted = new CountDownLatch(1); + CountDownLatch allowClose = new CountDownLatch(1); + CountingPixelsWriter writer = new CountingPixelsWriter(closeStarted, allowClose, null, null); + File file = temporaryFile(2001L); + FileWriterManager fileWriterManager = testFileWriterManager(writer, file); + + CompletableFuture finishFuture = fileWriterManager.finish(); + assertTrue(closeStarted.await(5, TimeUnit.SECONDS)); + + // Concurrently update row id bookkeeping; this is what the visibility layer does. + file.setMaxRowId(255L); + file.setNumRowGroup(3); + + allowClose.countDown(); + finishFuture.get(5, TimeUnit.SECONDS); + + File snapshot = fileWriterManager.getFileSnapshot(); + assertEquals(255L, snapshot.getMaxRowId()); + assertEquals(3, snapshot.getNumRowGroup()); + assertEquals(File.Type.TEMPORARY, snapshot.getType()); + assertEquals(1, writer.closeCount.get()); + } + + @Test + public void addRowBatchPropagatesWriterRuntimeExceptionWithoutWrapping() throws Exception + { + // FileWriterManager only wraps IOException into RetinaException; unchecked exceptions + // (e.g. format-corruption indicators thrown by the underlying writer as RuntimeException) + // must propagate to the caller as-is so they are not silently masked. After such a failure + // the manager must remain usable and finish() must still close the writer exactly once. + RuntimeException formatFailure = new IllegalStateException("corrupted column vector"); + CountingPixelsWriter writer = new CountingPixelsWriter() + { + @Override + public boolean addRowBatch(VectorizedRowBatch rowBatch) throws IOException + { + addRowBatchCount.incrementAndGet(); + throw formatFailure; + } + }; + File file = temporaryFile(2101L); + FileWriterManager fileWriterManager = testFileWriterManager(writer, file); + + try + { + fileWriterManager.addRowBatch(null); + fail("Runtime exception from writer must propagate without being wrapped"); + } catch (RetinaException e) + { + fail("Runtime exception must not be wrapped as RetinaException, got: " + e); + } catch (IllegalStateException expected) + { + assertSame(formatFailure, expected); + } + assertEquals(1, writer.addRowBatchCount.get()); + + // After a runtime failure inside the writer, finish() must still be able to close it. + fileWriterManager.finish().get(5, TimeUnit.SECONDS); + assertEquals(1, writer.closeCount.get()); + assertEquals(File.Type.TEMPORARY, file.getType()); + } + + private static File temporaryFile(long id) + { + File file = new File(); + file.setId(id); + file.setName("ingest_" + id + ".pxl"); + file.setType(File.Type.TEMPORARY); + file.setNumRowGroup(1); + file.setMinRowId(0); + file.setMaxRowId(63); + file.setPathId(9L); + return file; + } + + private static FileWriterManager testFileWriterManager(CountingPixelsWriter writer, File file) + { + return new FileWriterManager(1L, writer, file, 1L, 0L, 0); + } + + private static class CountingPixelsWriter implements PixelsWriter + { + // Package-private so anonymous subclasses defined inside this test can observe call counts. + final AtomicInteger closeCount = new AtomicInteger(0); + final AtomicInteger addRowBatchCount = new AtomicInteger(0); + private final CountDownLatch closeStarted; + private final CountDownLatch allowClose; + private final IOException closeFailure; + private final IOException addRowBatchFailure; + private final RuntimeException closeRuntimeFailure; + + private CountingPixelsWriter() + { + this(null, null, null, null, null); + } + + private CountingPixelsWriter(CountDownLatch closeStarted, CountDownLatch allowClose, + IOException closeFailure, IOException addRowBatchFailure) + { + this(closeStarted, allowClose, closeFailure, addRowBatchFailure, null); + } + + private CountingPixelsWriter(CountDownLatch closeStarted, CountDownLatch allowClose, + IOException closeFailure, IOException addRowBatchFailure, + RuntimeException closeRuntimeFailure) + { + this.closeStarted = closeStarted; + this.allowClose = allowClose; + this.closeFailure = closeFailure; + this.addRowBatchFailure = addRowBatchFailure; + this.closeRuntimeFailure = closeRuntimeFailure; + } + + @Override + public boolean addRowBatch(VectorizedRowBatch rowBatch) throws IOException + { + addRowBatchCount.incrementAndGet(); + if (addRowBatchFailure != null) + { + throw addRowBatchFailure; + } + return true; + } + + @Override + public void addRowBatch(VectorizedRowBatch rowBatch, int hashValue) throws IOException + { + } + + @Override + public TypeDescription getSchema() + { + return null; + } + + @Override + public int getNumRowGroup() + { + return 0; + } + + @Override + public int getNumWriteRequests() + { + return 0; + } + + @Override + public long getCompletedBytes() + { + return 0; + } + + @Override + public void close() throws IOException + { + closeCount.incrementAndGet(); + if (closeStarted != null) + { + closeStarted.countDown(); + } + if (allowClose != null) + { + try + { + assertTrue(allowClose.await(5, TimeUnit.SECONDS)); + } catch (InterruptedException e) + { + Thread.currentThread().interrupt(); + throw new IOException("Interrupted while waiting to close", e); + } + } + if (closeFailure != null) + { + throw closeFailure; + } + if (closeRuntimeFailure != null) + { + throw closeRuntimeFailure; + } + } + } + + private static class ThreadCapturingPixelsWriter implements PixelsWriter + { + private final CountDownLatch closeStarted; + private final CountDownLatch allowClose; + private volatile Thread closeThread; + + private ThreadCapturingPixelsWriter(CountDownLatch closeStarted, CountDownLatch allowClose) + { + this.closeStarted = closeStarted; + this.allowClose = allowClose; + } + + @Override + public boolean addRowBatch(VectorizedRowBatch rowBatch) + { + return true; + } + + @Override + public void addRowBatch(VectorizedRowBatch rowBatch, int hashValue) + { + } + + @Override + public TypeDescription getSchema() + { + return null; + } + + @Override + public int getNumRowGroup() + { + return 0; + } + + @Override + public int getNumWriteRequests() + { + return 0; + } + + @Override + public long getCompletedBytes() + { + return 0; + } + + @Override + public void close() throws IOException + { + this.closeThread = Thread.currentThread(); + if (closeStarted != null) + { + closeStarted.countDown(); + } + if (allowClose != null) + { + try + { + assertTrue(allowClose.await(5, TimeUnit.SECONDS)); + } catch (InterruptedException e) + { + Thread.currentThread().interrupt(); + throw new IOException("Interrupted while waiting to close", e); + } + } + } + } +} From 32678215ad747a7ec48acda31d5bbc111581913a Mon Sep 17 00:00:00 2001 From: Dongyang Geng Date: Wed, 13 May 2026 14:21:07 +0800 Subject: [PATCH 06/17] fix: restrict metadata getFiles to regular files --- .../common/metadata/MetadataService.java | 3 + .../pixels/daemon/metadata/dao/FileDao.java | 3 + .../daemon/metadata/dao/impl/RdbFileDao.java | 9 +- .../retina/TestStorageGarbageCollector.java | 242 +++++++++++++++++- proto/metadata.proto | 3 +- 5 files changed, 255 insertions(+), 5 deletions(-) diff --git a/pixels-common/src/main/java/io/pixelsdb/pixels/common/metadata/MetadataService.java b/pixels-common/src/main/java/io/pixelsdb/pixels/common/metadata/MetadataService.java index 3b4b2d6479..615127bf1c 100644 --- a/pixels-common/src/main/java/io/pixelsdb/pixels/common/metadata/MetadataService.java +++ b/pixels-common/src/main/java/io/pixelsdb/pixels/common/metadata/MetadataService.java @@ -1428,6 +1428,9 @@ public File.Type getFileType(String filePathUri) throws MetadataException } } + /** + * Return query-visible REGULAR files under the path. + */ public List getFiles(long pathId) throws MetadataException { String token = UUID.randomUUID().toString(); diff --git a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/dao/FileDao.java b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/dao/FileDao.java index 73b921008b..b5ae2b9d1c 100644 --- a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/dao/FileDao.java +++ b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/dao/FileDao.java @@ -38,6 +38,9 @@ public List getAll() throw new UnsupportedOperationException("getAll is not supported."); } + /** + * Return query-visible REGULAR files under a path. + */ public abstract List getAllByPathId(long pathId); public abstract MetadataProto.File getByPathIdAndFileName(long pathId, String fileName); diff --git a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/dao/impl/RdbFileDao.java b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/dao/impl/RdbFileDao.java index 1af30d564b..ccae356b8c 100644 --- a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/dao/impl/RdbFileDao.java +++ b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/dao/impl/RdbFileDao.java @@ -71,10 +71,13 @@ public MetadataProto.File getById(long id) public List getAllByPathId(long pathId) { Connection conn = db.getConnection(); - try (Statement st = conn.createStatement()) + String sql = "SELECT * FROM FILES WHERE FILE_TYPE = ? AND PATHS_PATH_ID = ?"; + try (PreparedStatement st = conn.prepareStatement(sql)) { - // Issue #932: Add empty file markers and ignore empty files when retrieving file lists. - ResultSet rs = st.executeQuery("SELECT * FROM FILES WHERE FILE_TYPE <> 0 AND PATHS_PATH_ID=" + pathId); + // Query-visible file enumeration is REGULAR-only. + st.setInt(1, MetadataProto.File.Type.REGULAR.getNumber()); + st.setLong(2, pathId); + ResultSet rs = st.executeQuery(); List files = new ArrayList<>(); while (rs.next()) { diff --git a/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestStorageGarbageCollector.java b/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestStorageGarbageCollector.java index 6281626267..1a09060d6f 100644 --- a/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestStorageGarbageCollector.java +++ b/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestStorageGarbageCollector.java @@ -21,6 +21,7 @@ import io.pixelsdb.pixels.common.metadata.MetadataService; import io.pixelsdb.pixels.common.utils.CheckpointFileIO; +import io.pixelsdb.pixels.common.utils.MetaDBUtil; import io.pixelsdb.pixels.common.utils.PixelsFileNameUtils; import io.pixelsdb.pixels.common.utils.RetinaUtils; import io.pixelsdb.pixels.common.metadata.domain.Column; @@ -52,6 +53,7 @@ import java.lang.reflect.Method; import java.nio.file.Files; import java.nio.file.Path; +import java.sql.PreparedStatement; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -1866,7 +1868,7 @@ public void testAtomicSwap_idempotent() throws Exception /** * TEMPORARY visibility semantics: before the swap, {@code getFiles(pathId)} must - * not return the TEMPORARY new file (the DAO filters {@code FILE_TYPE <> 0}). + * not return the TEMPORARY new file (the DAO filters {@code FILE_TYPE = REGULAR}). * After the swap the promoted file is visible and the old file disappears. */ @Test @@ -1905,6 +1907,223 @@ public void testAtomicSwap_temporaryInvisibleViaGetFiles() throws Exception afterIds.contains(oldFileId)); } + // ----------------------------------------------------------------------- + // Coverage for getFiles(pathId) REGULAR-only enumeration. + // ----------------------------------------------------------------------- + + /** + * A path containing REGULAR and non-REGULAR FILE_TYPE values returns only REGULAR entries. + */ + @Test + public void testGetFiles_mixedAllFileTypes_onlyRegular() throws Exception + { + long regularId = -1L; + long tempId = -1L; + long nonRegularPositiveId = -1L; + long negativeId = -1L; + long extremeId = -1L; + try + { + String suffix = Long.toString(System.nanoTime()); + regularId = registerTestFile("mix_regular_" + suffix + ".pxl", + File.Type.REGULAR, 1, 0L, 1L); + tempId = registerTestFile("mix_temp_" + suffix + ".pxl", + File.Type.TEMPORARY, 1, 0L, 1L); + nonRegularPositiveId = insertRawFileWithType("mix_non_regular_" + suffix + ".pxl", + File.Type.REGULAR.ordinal() + 1, 1, 0L, 1L); + negativeId = insertRawFileWithType("mix_negative_" + suffix + ".pxl", + -2, 1, 0L, 1L); + extremeId = insertRawFileWithType("mix_extreme_max_" + suffix + ".pxl", + Integer.MAX_VALUE, 1, 0L, 1L); + + List files = metadataService.getFiles(testPathId); + Set visible = new HashSet<>(); + for (File f : files) + { + assertEquals("getFiles must only emit REGULAR", + File.Type.REGULAR, f.getType()); + visible.add(f.getId()); + } + assertTrue("REGULAR member of the mix must be visible", + visible.contains(regularId)); + assertFalse("TEMPORARY (FILE_TYPE=0) must be hidden", + visible.contains(tempId)); + assertFalse("non-REGULAR positive FILE_TYPE must be hidden", + visible.contains(nonRegularPositiveId)); + assertFalse("negative FILE_TYPE must be hidden", + visible.contains(negativeId)); + assertFalse("Integer.MAX_VALUE FILE_TYPE must be hidden", + visible.contains(extremeId)); + } + finally + { + List cleanup = new ArrayList<>(); + if (regularId > 0) cleanup.add(regularId); + if (tempId > 0) cleanup.add(tempId); + if (nonRegularPositiveId > 0) cleanup.add(nonRegularPositiveId); + if (negativeId > 0) cleanup.add(negativeId); + if (extremeId > 0) cleanup.add(extremeId); + if (!cleanup.isEmpty()) metadataService.deleteFiles(cleanup); + } + } + + /** + * A minimum-size REGULAR file is returned with its catalog fields intact. + */ + @Test + public void testGetFiles_singleRegularMinimumData() throws Exception + { + long fileId = -1L; + try + { + fileId = registerTestFile("min_single_regular_" + System.nanoTime() + ".pxl", + File.Type.REGULAR, 1, 0L, 0L); + List files = metadataService.getFiles(testPathId); + File found = null; + for (File f : files) + { + if (f.getId() == fileId) + { + found = f; + } + assertEquals("every returned entry must be REGULAR", + File.Type.REGULAR, f.getType()); + } + assertNotNull("the single REGULAR minimum-data file must be visible", found); + assertEquals("type must be REGULAR", File.Type.REGULAR, found.getType()); + assertEquals("numRowGroup of minimum file must be 1", 1, found.getNumRowGroup()); + assertEquals("minRowId of minimum file must be 0", 0L, found.getMinRowId()); + assertEquals("maxRowId of minimum file must be 0", 0L, found.getMaxRowId()); + } + finally + { + if (fileId > 0) + { + metadataService.deleteFiles(Collections.singletonList(fileId)); + } + } + } + + /** + * A deleted REGULAR file is no longer returned by {@code getFiles}. + */ + @Test + public void testGetFiles_deletedRegular_notVisible() throws Exception + { + long regularId = registerTestFile("delete_visibility_" + System.nanoTime() + ".pxl", + File.Type.REGULAR, 1, 0L, 1L); + + List beforeDelete = metadataService.getFiles(testPathId); + Set beforeIds = new HashSet<>(); + for (File f : beforeDelete) beforeIds.add(f.getId()); + assertTrue("REGULAR file must be visible before delete", + beforeIds.contains(regularId)); + + metadataService.deleteFiles(Collections.singletonList(regularId)); + + List afterDelete = metadataService.getFiles(testPathId); + for (File f : afterDelete) + { + assertFalse("deleted REGULAR file must no longer be visible", + f.getId() == regularId); + } + } + + /** + * Concurrent readers observe a consistent REGULAR-only result. + */ + @Test + public void testGetFiles_concurrentReaders_consistentRegularOnly() throws Exception + { + long regularId = -1L; + long tempId = -1L; + long nonRegularPositiveId = -1L; + ExecutorService pool = null; + try + { + String suffix = Long.toString(System.nanoTime()); + regularId = registerTestFile("conc_regular_" + suffix + ".pxl", + File.Type.REGULAR, 1, 0L, 1L); + tempId = registerTestFile("conc_temp_" + suffix + ".pxl", + File.Type.TEMPORARY, 1, 0L, 1L); + nonRegularPositiveId = insertRawFileWithType("conc_non_regular_" + suffix + ".pxl", + File.Type.REGULAR.ordinal() + 1, 1, 0L, 1L); + + final int threads = 8; + final int iterations = 16; + pool = Executors.newFixedThreadPool(threads); + CyclicBarrier startGate = new CyclicBarrier(threads); + AtomicInteger leakedTemporary = new AtomicInteger(); + AtomicInteger leakedNonRegular = new AtomicInteger(); + AtomicInteger missingRegular = new AtomicInteger(); + + List> futures = new ArrayList<>(); + final long pinnedRegular = regularId; + final long pinnedTemp = tempId; + final long pinnedNonRegular = nonRegularPositiveId; + for (int t = 0; t < threads; t++) + { + futures.add(CompletableFuture.runAsync(() -> + { + try + { + startGate.await(); + for (int i = 0; i < iterations; i++) + { + List snapshot = metadataService.getFiles(testPathId); + boolean sawRegular = false; + for (File f : snapshot) + { + if (f.getType() != File.Type.REGULAR) + { + leakedNonRegular.incrementAndGet(); + } + if (f.getId() == pinnedRegular) sawRegular = true; + if (f.getId() == pinnedTemp) leakedTemporary.incrementAndGet(); + if (f.getId() == pinnedNonRegular) leakedNonRegular.incrementAndGet(); + } + if (!sawRegular) missingRegular.incrementAndGet(); + } + } + catch (Exception e) + { + throw new RuntimeException(e); + } + }, pool)); + } + CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])) + .get(30, java.util.concurrent.TimeUnit.SECONDS); + + assertEquals("no concurrent reader may observe a TEMPORARY file", + 0, leakedTemporary.get()); + assertEquals("no concurrent reader may observe a non-REGULAR file", + 0, leakedNonRegular.get()); + assertEquals("every concurrent reader must observe the REGULAR file", + 0, missingRegular.get()); + + // A follow-up call should remain REGULAR-only after the concurrent burst. + List followUp = metadataService.getFiles(testPathId); + assertNotNull("follow-up getFiles must not return null", followUp); + for (File f : followUp) + { + assertEquals("follow-up entries must all be REGULAR", + File.Type.REGULAR, f.getType()); + } + } + finally + { + if (pool != null) + { + pool.shutdownNow(); + } + List cleanup = new ArrayList<>(); + if (regularId > 0) cleanup.add(regularId); + if (tempId > 0) cleanup.add(tempId); + if (nonRegularPositiveId > 0) cleanup.add(nonRegularPositiveId); + if (!cleanup.isEmpty()) metadataService.deleteFiles(cleanup); + } + } + /** * Multiple serial swaps: Storage GC processes FileGroups serially on a single * thread, so {@code atomicSwapFiles} is never called concurrently in production. @@ -2952,6 +3171,27 @@ private long registerTestFile(String name, File.Type type, return id; } + private long insertRawFileWithType(String name, int fileType, + int numRg, long minRow, long maxRow) + throws Exception + { + String sql = "INSERT INTO FILES(FILE_NAME, FILE_TYPE, FILE_NUM_RG, FILE_MIN_ROW_ID, FILE_MAX_ROW_ID, PATHS_PATH_ID) " + + "VALUES (?, ?, ?, ?, ?, ?)"; + try (PreparedStatement pst = MetaDBUtil.Instance().getConnection().prepareStatement(sql)) + { + pst.setString(1, name); + pst.setInt(2, fileType); + pst.setInt(3, numRg); + pst.setLong(4, minRow); + pst.setLong(5, maxRow); + pst.setLong(6, testPathId); + assertEquals("raw test file insert should affect one row", 1, pst.executeUpdate()); + } + long id = metadataService.getFileId(testOrderedPathUri + "/" + name); + assertTrue(name + " must have valid id", id > 0); + return id; + } + private long[] registerTestFiles(String[] names, File.Type[] types, int[] numRgs, long[] minRows, long[] maxRows) throws Exception diff --git a/proto/metadata.proto b/proto/metadata.proto index 575b868918..b10c30194c 100644 --- a/proto/metadata.proto +++ b/proto/metadata.proto @@ -680,6 +680,7 @@ message AddFilesResponse { } message GetFilesRequest { + // Query-visible REGULAR file enumeration. RequestHeader header = 1; uint64 pathId = 2; } @@ -844,4 +845,4 @@ message DropViewRequest { message DropViewResponse { ResponseHeader header = 1; } -// end request/response definition for rpc services \ No newline at end of file +// end request/response definition for rpc services From 8ed122bebd0216990052f6b30d4cb4edff705848 Mon Sep 17 00:00:00 2001 From: Dongyang Geng Date: Wed, 13 May 2026 18:03:43 +0800 Subject: [PATCH 07/17] fix: start Retina background GC after initialization --- .../daemon/retina/RetinaServerImpl.java | 157 ++++++------ .../daemon/retina/TestRetinaServer.java | 141 +++++++++++ .../pixels/retina/RetinaResourceManager.java | 88 +++++-- .../retina/TestRetinaResourceManager.java | 162 +++++++++++++ .../retina/TestStorageGarbageCollector.java | 223 ++++++++++++++---- 5 files changed, 622 insertions(+), 149 deletions(-) diff --git a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/retina/RetinaServerImpl.java b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/retina/RetinaServerImpl.java index 09218beef5..d2763698b9 100644 --- a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/retina/RetinaServerImpl.java +++ b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/retina/RetinaServerImpl.java @@ -75,9 +75,18 @@ public class RetinaServerImpl extends RetinaWorkerServiceGrpc.RetinaWorkerServic */ public RetinaServerImpl() { - this.metadataService = MetadataService.Instance(); - this.indexService = IndexServiceProvider.getService(IndexServiceProvider.ServiceMode.local); - this.retinaResourceManager = RetinaResourceManager.Instance(); + this(MetadataService.Instance(), + IndexServiceProvider.getService(IndexServiceProvider.ServiceMode.local), + RetinaResourceManager.Instance()); + } + + RetinaServerImpl(MetadataService metadataService, IndexService indexService, + RetinaResourceManager retinaResourceManager) + { + this.metadataService = requireNonNull(metadataService, "metadataService is null"); + this.indexService = requireNonNull(indexService, "indexService is null"); + this.retinaResourceManager = requireNonNull(retinaResourceManager, "retinaResourceManager is null"); + int totalBuckets = Integer.parseInt(ConfigFactory.Instance().getProperty("index.bucket.num")); this.indexOptionPool = new IndexOption[totalBuckets]; for (int i = 0; i < totalBuckets; i++) @@ -86,91 +95,95 @@ public RetinaServerImpl() this.indexOptionPool[i].setVNodeId(i); } - startRetinaMetricsLogThread(); try { - logger.info("Pre-loading checkpoints..."); - this.retinaResourceManager.recoverCheckpoints(); + initializeRetinaResources(); + this.retinaResourceManager.startBackgroundGc(); + startRetinaMetricsLogThread(); + logger.info("Retina service is ready"); + } + catch (Exception e) + { + logger.error("Error while initializing RetinaServerImpl", e); + throw new IllegalStateException("Failed to initialize RetinaServerImpl", e); + } + } + + private void initializeRetinaResources() throws Exception + { + logger.info("Pre-loading checkpoints..."); + this.retinaResourceManager.recoverCheckpoints(); - List schemas = this.metadataService.getSchemas(); - for (Schema schema : schemas) + List schemas = this.metadataService.getSchemas(); + for (Schema schema : schemas) + { + List tables = this.metadataService.getTables(schema.getName()); + for (Table table : tables) { - List
tables = this.metadataService.getTables(schema.getName()); - for (Table table : tables) + List layouts = this.metadataService.getLayouts(schema.getName(), table.getName()); + List files = new LinkedList<>(); + for (Layout layout : layouts) { - List layouts = this.metadataService.getLayouts(schema.getName(), table.getName()); - List files = new LinkedList<>(); - for (Layout layout : layouts) + if (layout.isReadable()) { - if (layout.isReadable()) - { - /* - * Issue #946: always add visibility to all files - */ - // add visibility for ordered files - List orderedPaths = layout.getOrderedPaths(); - validateOrderedOrCompactPaths(orderedPaths); - List orderedFiles = this.metadataService.getFiles(orderedPaths.get(0).getId()); - files.addAll(orderedFiles.stream() - .map(file -> orderedPaths.get(0).getUri() + "/" + file.getName()) - .collect(Collectors.toList())); - - // add visibility for compact files - List compactPaths = layout.getCompactPaths(); - validateOrderedOrCompactPaths(compactPaths); - List compactFiles = this.metadataService.getFiles(compactPaths.get(0).getId()); - files.addAll(compactFiles.stream() - .map(file -> compactPaths.get(0).getUri() + "/" + file.getName()) - .collect(Collectors.toList())); - } + /* + * Issue #946: always add visibility to all files + */ + // add visibility for ordered files + List orderedPaths = layout.getOrderedPaths(); + validateOrderedOrCompactPaths(orderedPaths); + List orderedFiles = this.metadataService.getFiles(orderedPaths.get(0).getId()); + files.addAll(orderedFiles.stream() + .map(file -> orderedPaths.get(0).getUri() + "/" + file.getName()) + .collect(Collectors.toList())); + + // add visibility for compact files + List compactPaths = layout.getCompactPaths(); + validateOrderedOrCompactPaths(compactPaths); + List compactFiles = this.metadataService.getFiles(compactPaths.get(0).getId()); + files.addAll(compactFiles.stream() + .map(file -> compactPaths.get(0).getUri() + "/" + file.getName()) + .collect(Collectors.toList())); } + } - int threadNum = Integer.parseInt - (ConfigFactory.Instance().getProperty("retina.service.init.threads")); - ExecutorService executorService = Executors.newFixedThreadPool(threadNum); - AtomicBoolean success = new AtomicBoolean(true); - AtomicReference e = new AtomicReference<>(); - try + int threadNum = Integer.parseInt + (ConfigFactory.Instance().getProperty("retina.service.init.threads")); + ExecutorService executorService = Executors.newFixedThreadPool(threadNum); + AtomicBoolean success = new AtomicBoolean(true); + AtomicReference e = new AtomicReference<>(); + try + { + for (String filePath : files) { - for (String filePath : files) + executorService.submit(() -> { - executorService.submit(() -> + try { - try - { - this.retinaResourceManager.addVisibility(filePath); - } - catch (Exception ex) - { - success.set(false); - e.set(ex); - } - }); - } - } - finally - { - executorService.shutdown(); - } - - if (success.get()) - { - executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); + this.retinaResourceManager.addVisibility(filePath); + } + catch (Exception ex) + { + success.set(false); + e.set(ex); + } + }); } + } + finally + { + executorService.shutdown(); + } - if (!success.get()) - { - throw new RetinaException("Can't add visibility", e.get()); - } + executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); - this.retinaResourceManager.addWriteBuffer(schema.getName(), table.getName()); + if (!success.get()) + { + throw new RetinaException("Can't add visibility", e.get()); } + + this.retinaResourceManager.addWriteBuffer(schema.getName(), table.getName()); } - logger.info("Retina service is ready"); - } - catch (Exception e) - { - logger.error("Error while initializing RetinaServerImpl", e); } } diff --git a/pixels-daemon/src/test/java/io/pixelsdb/pixels/daemon/retina/TestRetinaServer.java b/pixels-daemon/src/test/java/io/pixelsdb/pixels/daemon/retina/TestRetinaServer.java index 1167cf6e86..cf4d0e526a 100644 --- a/pixels-daemon/src/test/java/io/pixelsdb/pixels/daemon/retina/TestRetinaServer.java +++ b/pixels-daemon/src/test/java/io/pixelsdb/pixels/daemon/retina/TestRetinaServer.java @@ -19,12 +19,40 @@ */ package io.pixelsdb.pixels.daemon.retina; +import io.pixelsdb.pixels.common.exception.MetadataException; +import io.pixelsdb.pixels.common.exception.RetinaException; +import io.pixelsdb.pixels.common.index.service.IndexService; +import io.pixelsdb.pixels.common.metadata.MetadataService; +import io.pixelsdb.pixels.common.metadata.domain.File; +import io.pixelsdb.pixels.common.metadata.domain.Layout; +import io.pixelsdb.pixels.common.metadata.domain.Path; +import io.pixelsdb.pixels.common.metadata.domain.Permission; +import io.pixelsdb.pixels.common.metadata.domain.Schema; +import io.pixelsdb.pixels.common.metadata.domain.Table; import io.pixelsdb.pixels.daemon.ServerContainer; import io.pixelsdb.pixels.daemon.metadata.MetadataServer; +import io.pixelsdb.pixels.retina.RetinaResourceManager; +import org.junit.Ignore; import org.junit.Test; +import org.mockito.InOrder; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; public class TestRetinaServer { + @Ignore("Integration test requires real metadata server, metadata DB, and fixed local ports.") @Test public void testRetinaServer() { @@ -34,4 +62,117 @@ public void testRetinaServer() RetinaServer retinaServer = new RetinaServer(18890); container.addServer("retina server", retinaServer); } + + @Test + public void testRetinaServerImplInitializationFailureIsFailClosed() throws Exception + { + MetadataService metadataService = mock(MetadataService.class); + IndexService indexService = mock(IndexService.class); + RetinaResourceManager resourceManager = mock(RetinaResourceManager.class); + + when(metadataService.getSchemas()).thenThrow(new MetadataException("metadata unavailable")); + + try + { + RetinaServerImpl server = new RetinaServerImpl(metadataService, indexService, resourceManager); + fail("RetinaServerImpl must fail closed when initialization fails: " + server); + } + catch (IllegalStateException e) + { + assertTrue(e.getMessage().contains("Failed to initialize RetinaServerImpl")); + } + + verify(resourceManager).recoverCheckpoints(); + verify(resourceManager, never()).startBackgroundGc(); + } + + @Test + public void testRetinaServerImplStartsBackgroundGcAfterSuccessfulInitialization() throws Exception + { + MetadataService metadataService = mock(MetadataService.class); + IndexService indexService = mock(IndexService.class); + RetinaResourceManager resourceManager = mock(RetinaResourceManager.class); + + Schema schema = new Schema(); + schema.setName("gc_schema"); + Table table = new Table(); + table.setName("gc_table"); + Path orderedPath = new Path(); + orderedPath.setId(11L); + orderedPath.setUri("file:///tmp/pixels/ordered"); + Path compactPath = new Path(); + compactPath.setId(12L); + compactPath.setUri("file:///tmp/pixels/compact"); + Layout layout = new Layout(); + layout.setPermission(Permission.READ_WRITE); + layout.setOrderedPaths(Collections.singletonList(orderedPath)); + layout.setCompactPaths(Collections.singletonList(compactPath)); + File orderedFile = new File(); + orderedFile.setName("ordered.pxl"); + File compactFile = new File(); + compactFile.setName("compact.pxl"); + List lifecycleEvents = Collections.synchronizedList(new ArrayList<>()); + + when(metadataService.getSchemas()).thenReturn(Collections.singletonList(schema)); + when(metadataService.getTables(schema.getName())).thenReturn(Collections.singletonList(table)); + when(metadataService.getLayouts(schema.getName(), table.getName())).thenReturn(Collections.singletonList(layout)); + when(metadataService.getFiles(orderedPath.getId())).thenReturn(Collections.singletonList(orderedFile)); + when(metadataService.getFiles(compactPath.getId())).thenReturn(Collections.singletonList(compactFile)); + doAnswer(invocation -> { + lifecycleEvents.add("recover"); + return null; + }).when(resourceManager).recoverCheckpoints(); + doAnswer(invocation -> { + lifecycleEvents.add("visibility:" + invocation.getArgument(0)); + return null; + }).when(resourceManager).addVisibility(org.mockito.ArgumentMatchers.anyString()); + doAnswer(invocation -> { + lifecycleEvents.add("writeBuffer"); + return null; + }).when(resourceManager).addWriteBuffer(schema.getName(), table.getName()); + doAnswer(invocation -> { + lifecycleEvents.add("startGc"); + return null; + }).when(resourceManager).startBackgroundGc(); + + new RetinaServerImpl(metadataService, indexService, resourceManager); + + assertTrue(lifecycleEvents.indexOf("recover") >= 0); + assertTrue(lifecycleEvents.contains("visibility:file:///tmp/pixels/ordered/ordered.pxl")); + assertTrue(lifecycleEvents.contains("visibility:file:///tmp/pixels/compact/compact.pxl")); + int writeBufferIndex = lifecycleEvents.indexOf("writeBuffer"); + assertTrue(writeBufferIndex > lifecycleEvents.indexOf("recover")); + assertTrue(writeBufferIndex > lifecycleEvents.indexOf("visibility:file:///tmp/pixels/ordered/ordered.pxl")); + assertTrue(writeBufferIndex > lifecycleEvents.indexOf("visibility:file:///tmp/pixels/compact/compact.pxl")); + assertTrue(lifecycleEvents.indexOf("startGc") > writeBufferIndex); + verify(resourceManager).addVisibility("file:///tmp/pixels/ordered/ordered.pxl"); + verify(resourceManager).addVisibility("file:///tmp/pixels/compact/compact.pxl"); + verify(resourceManager).startBackgroundGc(); + } + + @Test + public void testRetinaServerImplBackgroundGcStartFailureIsFailClosed() throws Exception + { + MetadataService metadataService = mock(MetadataService.class); + IndexService indexService = mock(IndexService.class); + RetinaResourceManager resourceManager = mock(RetinaResourceManager.class); + + when(metadataService.getSchemas()).thenReturn(Collections.emptyList()); + doThrow(new RetinaException("gc disabled by invalid lifecycle")) + .when(resourceManager).startBackgroundGc(); + + try + { + RetinaServerImpl server = new RetinaServerImpl(metadataService, indexService, resourceManager); + fail("RetinaServerImpl must fail closed when background GC cannot start: " + server); + } + catch (IllegalStateException e) + { + assertTrue(e.getMessage().contains("Failed to initialize RetinaServerImpl")); + } + + InOrder inOrder = inOrder(resourceManager); + inOrder.verify(resourceManager).recoverCheckpoints(); + inOrder.verify(resourceManager).startBackgroundGc(); + } } diff --git a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/RetinaResourceManager.java b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/RetinaResourceManager.java index 2eeb97f015..900d907b63 100644 --- a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/RetinaResourceManager.java +++ b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/RetinaResourceManager.java @@ -48,6 +48,7 @@ import java.nio.file.Paths; import java.util.*; import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -66,7 +67,7 @@ public class RetinaResourceManager // GC related fields private final ScheduledExecutorService gcExecutor; - private final boolean storageGcEnabled; + private final AtomicBoolean gcScheduled; private final StorageGarbageCollector storageGarbageCollector; // Checkpoint related fields @@ -150,37 +151,20 @@ private RetinaResourceManager() return t; }); - ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(r -> { + this.gcExecutor = Executors.newSingleThreadScheduledExecutor(r -> { Thread t = new Thread(r, "retina-gc-thread"); t.setDaemon(true); return t; }); - try - { - long interval = Long.parseLong(config.getProperty("retina.gc.interval")); - if (interval > 0) - { - executor.scheduleAtFixedRate( - this::runGC, - interval, - interval, - TimeUnit.SECONDS - ); - } - } catch (Exception e) - { - logger.error("Failed to start retina background gc", e); - } - this.gcExecutor = executor; + this.gcScheduled = new AtomicBoolean(false); totalVirtualNodeNum = Integer.parseInt(ConfigFactory.Instance().getProperty("node.virtual.num")); this.retinaHostName = NetUtils.getLocalHostName(); - boolean gcEnabled = false; StorageGarbageCollector gc = null; try { - gcEnabled = Boolean.parseBoolean(config.getProperty("retina.storage.gc.enabled")); - if (gcEnabled) + boolean storageGcEnabled = Boolean.parseBoolean(config.getProperty("retina.storage.gc.enabled")); + if (storageGcEnabled) { double threshold = Double.parseDouble(config.getProperty("retina.storage.gc.threshold")); long targetFileSize = Long.parseLong(config.getProperty("retina.storage.gc.target.file.size")); @@ -200,10 +184,8 @@ private RetinaResourceManager() catch (Exception e) { logger.error("Failed to initialise StorageGarbageCollector, Storage GC will be disabled", e); - gcEnabled = false; gc = null; } - this.storageGcEnabled = gcEnabled; this.storageGarbageCollector = gc; } @@ -217,6 +199,62 @@ public static RetinaResourceManager Instance() return InstanceHolder.instance; } + /** + * Starts the periodic Retina GC scheduler after the service has reached the + * lifecycle point where background cleanup is safe to run. + * + *

The constructor intentionally does not schedule GC: recovery-capable + * startup must stay fail-closed until initialization succeeds. This method is + * idempotent so future lifecycle READY hooks can call it safely.

+ * + * @throws RetinaException if GC configuration is invalid or the scheduler cannot be started. + */ + public void startBackgroundGc() throws RetinaException + { + long interval; + try + { + interval = Long.parseLong(ConfigFactory.Instance().getProperty("retina.gc.interval")); + } + catch (Exception e) + { + throw new RetinaException("Invalid retina GC interval configuration", e); + } + + if (interval <= 0) + { + logger.info("Retina background GC is disabled"); + return; + } + + if (!this.gcScheduled.compareAndSet(false, true)) + { + logger.debug("Retina background GC scheduler has already been started"); + return; + } + + try + { + this.gcExecutor.scheduleAtFixedRate( + this::runGC, + interval, + interval, + TimeUnit.SECONDS + ); + logger.info("Retina background GC scheduler started with interval {} seconds", interval); + } + catch (RuntimeException e) + { + this.gcScheduled.set(false); + throw new RetinaException("Failed to start retina background GC", e); + } + } + + public boolean isBackgroundGcStarted() + { + return this.gcScheduled.get(); + } + public void addVisibility(long fileId, int rgId, int recordNum, long timestamp, long[] bitmap, boolean overwrite) { @@ -1000,7 +1038,7 @@ private void runGC() // Step 3: Storage GC — pass file-level stats so that candidate selection // uses O(1) lookups instead of per-RG aggregation loops. - if (storageGcEnabled && storageGarbageCollector != null) + if (storageGarbageCollector != null) { try { diff --git a/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestRetinaResourceManager.java b/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestRetinaResourceManager.java index 6edb341693..48986a7468 100644 --- a/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestRetinaResourceManager.java +++ b/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestRetinaResourceManager.java @@ -20,10 +20,27 @@ package io.pixelsdb.pixels.retina; import io.pixelsdb.pixels.common.exception.RetinaException; +import io.pixelsdb.pixels.common.utils.ConfigFactory; import io.pixelsdb.pixels.core.vector.VectorizedRowBatch; +import org.junit.Ignore; import org.junit.Test; +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.lang.reflect.Method; import java.nio.ByteBuffer; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; public class TestRetinaResourceManager { @@ -45,6 +62,150 @@ private boolean checkVisibility(long[] visibility, int rowId) return (targetLong & (1L << (rowId % 64))) != 0; } + private RetinaResourceManager newIsolatedManager() throws Exception + { + Constructor constructor = RetinaResourceManager.class.getDeclaredConstructor(); + constructor.setAccessible(true); + return constructor.newInstance(); + } + + private void setGcExecutor(RetinaResourceManager manager, + ScheduledExecutorService executor) throws Exception + { + Field field = RetinaResourceManager.class.getDeclaredField("gcExecutor"); + field.setAccessible(true); + field.set(manager, executor); + } + + @Test + public void testBackgroundGcIsNotStartedByConstructor() throws Exception + { + Constructor constructor = RetinaResourceManager.class.getDeclaredConstructor(); + constructor.setAccessible(true); + RetinaResourceManager manager = constructor.newInstance(); + + assertFalse("background GC must be started by lifecycle only", + manager.isBackgroundGcStarted()); + } + + @Test + public void testStartBackgroundGcIsExplicitAndIdempotent() throws Exception + { + String originalInterval = ConfigFactory.Instance().getProperty("retina.gc.interval"); + RetinaResourceManager manager = newIsolatedManager(); + ScheduledExecutorService executor = mock(ScheduledExecutorService.class); + setGcExecutor(manager, executor); + try + { + ConfigFactory.Instance().addProperty("retina.gc.interval", "300"); + + manager.startBackgroundGc(); + manager.startBackgroundGc(); + + assertTrue("explicit lifecycle start must mark background GC as started", + manager.isBackgroundGcStarted()); + verify(executor).scheduleAtFixedRate(any(Runnable.class), eq(300L), eq(300L), eq(TimeUnit.SECONDS)); + verifyNoMoreInteractions(executor); + } + finally + { + ConfigFactory.Instance().addProperty("retina.gc.interval", originalInterval); + } + } + + @Test + public void testStartBackgroundGcDisabledByNonPositiveInterval() throws Exception + { + String originalInterval = ConfigFactory.Instance().getProperty("retina.gc.interval"); + RetinaResourceManager manager = newIsolatedManager(); + try + { + ConfigFactory.Instance().addProperty("retina.gc.interval", "0"); + + manager.startBackgroundGc(); + + assertFalse("disabled interval must not mark background GC as started", + manager.isBackgroundGcStarted()); + } + finally + { + ConfigFactory.Instance().addProperty("retina.gc.interval", originalInterval); + } + } + + @Test + public void testStartBackgroundGcInvalidIntervalFailsWithoutStarting() throws Exception + { + String originalInterval = ConfigFactory.Instance().getProperty("retina.gc.interval"); + RetinaResourceManager manager = newIsolatedManager(); + try + { + ConfigFactory.Instance().addProperty("retina.gc.interval", "not-a-number"); + + try + { + manager.startBackgroundGc(); + fail("invalid GC interval must fail closed"); + } + catch (RetinaException e) + { + assertTrue(e.getMessage().contains("Invalid retina GC interval configuration")); + } + + assertFalse("failed lifecycle start must not mark GC as started", + manager.isBackgroundGcStarted()); + } + finally + { + ConfigFactory.Instance().addProperty("retina.gc.interval", originalInterval); + } + } + + @Test + public void testStartBackgroundGcSchedulerFailureRollsBackStartedFlag() throws Exception + { + String originalInterval = ConfigFactory.Instance().getProperty("retina.gc.interval"); + RetinaResourceManager manager = newIsolatedManager(); + ScheduledExecutorService executor = mock(ScheduledExecutorService.class); + setGcExecutor(manager, executor); + try + { + ConfigFactory.Instance().addProperty("retina.gc.interval", "300"); + when(executor.scheduleAtFixedRate(any(Runnable.class), eq(300L), eq(300L), eq(TimeUnit.SECONDS))) + .thenThrow(new RuntimeException("scheduler rejected")); + + try + { + manager.startBackgroundGc(); + fail("scheduler failure must fail closed"); + } + catch (RetinaException e) + { + assertTrue(e.getMessage().contains("Failed to start retina background GC")); + } + + assertFalse("scheduler failure must roll back started flag", + manager.isBackgroundGcStarted()); + } + finally + { + ConfigFactory.Instance().addProperty("retina.gc.interval", originalInterval); + } + } + + @Test + public void testRunGcBeforeLifecycleStartIsRejected() throws Exception + { + RetinaResourceManager manager = newIsolatedManager(); + Method runGc = RetinaResourceManager.class.getDeclaredMethod("runGC"); + runGc.setAccessible(true); + + runGc.invoke(manager); + + assertFalse("manual GC invocation before lifecycle start must be ignored", + manager.isBackgroundGcStarted()); + } + @Test public void TestVisibility() { @@ -80,6 +241,7 @@ private byte[][] createTpchNationRow(long nationKey, String name, long regionKey return row; } + @Ignore("Integration test requires real tpch.nation metadata and storage state.") @Test public void testWriteBuffer() { diff --git a/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestStorageGarbageCollector.java b/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestStorageGarbageCollector.java index 1a09060d6f..bc2e14a21a 100644 --- a/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestStorageGarbageCollector.java +++ b/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestStorageGarbageCollector.java @@ -72,6 +72,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * Tests for {@link StorageGarbageCollector}, covering scan/grouping, data rewrite, @@ -95,6 +96,7 @@ * * Legacy test names (pre-convention) are preserved for CI stability. */ +@Ignore("Integration suite requires a running metadata server and external metadata DB state.") public class TestStorageGarbageCollector { // ----------------------------------------------------------------------- @@ -704,96 +706,166 @@ public void testScanAndGroupFiles_skipsFilesWithNoVisibility() // ======================================================================= /** - * After {@code runStorageGC}, the {@code gcSnapshotBitmaps} map must have had - * non-candidate entries removed. Candidate bitmaps must be retained for the rewrite phase. + * When no file crosses the strict deletion-ratio threshold, + * {@code runStorageGC} must return before metadata scan and keep the bitmap + * snapshot intact for the already-written GC checkpoint. */ @Test - public void testRunStorageGC_trimsBitmapMapToCandidate() + public void testRunStorageGC_noCandidateDoesNotScanOrTrim() { - long candidateFileId = 66001L; - long otherFileId = 66002L; + long belowThresholdFileId = 66101L; + long exactlyThresholdFileId = 66102L; - Map bitmaps = new HashMap<>(); - bitmaps.put(candidateFileId + "_0", makeBitmap(100, 60)); - bitmaps.put(otherFileId + "_0", makeBitmap(100, 20)); - - // File-level stats: candidateFileId has 60% deletion, otherFileId has 20% Map fileStats = new HashMap<>(); - fileStats.put(candidateFileId, makeRgStats(100, 60)); - fileStats.put(otherFileId, makeRgStats(100, 20)); + fileStats.put(belowThresholdFileId, makeRgStats(100, 40)); + fileStats.put(exactlyThresholdFileId, makeRgStats(100, 50)); - List fakeFiles = Arrays.asList( - new FakeFileEntry(candidateFileId, 1, 1L, 0), - new FakeFileEntry(otherFileId, 1, 1L, 0)); + Map bitmaps = new HashMap<>(); + bitmaps.put(RetinaUtils.buildRgKey(belowThresholdFileId, 0), makeBitmap(100, 40)); + bitmaps.put(RetinaUtils.buildRgKey(exactlyThresholdFileId, 0), makeBitmap(100, 50)); - DirectScanStorageGC gc = new DirectScanStorageGC( - retinaManager, 0.5, 10, fakeFiles); + TrackingRunStorageGC trackingGc = new TrackingRunStorageGC(Collections.emptyList()); - gc.runStorageGC(300L, fileStats, bitmaps); + trackingGc.runStorageGC(301L, fileStats, bitmaps); - assertTrue("candidate RG key must be retained", - bitmaps.containsKey(candidateFileId + "_0")); - assertFalse("non-candidate RG key must be removed", - bitmaps.containsKey(otherFileId + "_0")); + assertFalse("no candidate means metadata scan must not run", trackingGc.scanCalled); + assertFalse("no candidate means process phase must not run", trackingGc.processCalled); + assertTrue("below-threshold bitmap must remain for checkpoint recovery", + bitmaps.containsKey(RetinaUtils.buildRgKey(belowThresholdFileId, 0))); + assertTrue("exact-threshold bitmap must remain because threshold is strict >", + bitmaps.containsKey(RetinaUtils.buildRgKey(exactlyThresholdFileId, 0))); + assertEquals("bitmap snapshot must remain unchanged", 2, bitmaps.size()); } - // ======================================================================= - // Section 4: runStorageGC end-to-end scan → process - // ======================================================================= - /** - * A file whose invalidRatio is exactly equal to the threshold (0.5) must NOT - * be selected as a candidate. The design uses strict {@code >}, not {@code >=}. + * Candidate selection must be driven by file-level stats only. Files at the + * threshold, with zero rows, or below threshold must not be passed to scan; + * their bitmap entries are released before rewrite processing starts. */ @Test - public void testRunStorageGC_thresholdExactlyEqual() + public void testRunStorageGC_passesOnlyStrictFileLevelCandidatesToScan() { - long fileId = 57001L; + long candidateA = 66201L; + long candidateB = 66202L; + long exactlyThreshold = 66203L; + long zeroRows = 66204L; + long belowThreshold = 66205L; Map fileStats = new HashMap<>(); - fileStats.put(fileId, makeRgStats(100, 50)); // exactly 50% = threshold + fileStats.put(candidateA, makeRgStats(100, 51)); + fileStats.put(candidateB, makeRgStats(200, 120)); + fileStats.put(exactlyThreshold, makeRgStats(100, 50)); + fileStats.put(zeroRows, new long[]{0, 10}); + fileStats.put(belowThreshold, makeRgStats(100, 49)); Map bitmaps = new HashMap<>(); - bitmaps.put(fileId + "_0", makeBitmap(100, 50)); + for (long fileId : Arrays.asList(candidateA, candidateB, exactlyThreshold, zeroRows, belowThreshold)) + { + bitmaps.put(RetinaUtils.buildRgKey(fileId, 0), makeBitmap(100, 1)); + } + bitmaps.put(RetinaUtils.buildRgKey(candidateB, 1), makeBitmap(100, 1)); - DirectScanStorageGC gc = new DirectScanStorageGC( - retinaManager, 0.5, 10, - Collections.singletonList(new FakeFileEntry(fileId, 1, 1L, 0))); + TrackingRunStorageGC trackingGc = new TrackingRunStorageGC(Collections.emptyList()); - gc.runStorageGC(400L, fileStats, bitmaps); + trackingGc.runStorageGC(302L, fileStats, bitmaps); - assertTrue("file at exactly threshold must NOT be trimmed (no candidates)", - bitmaps.containsKey(fileId + "_0")); - assertEquals(1, bitmaps.size()); + assertTrue("candidate scan must run when at least one file qualifies", trackingGc.scanCalled); + assertEquals(new HashSet<>(Arrays.asList(candidateA, candidateB)), trackingGc.capturedCandidateFileIds); + assertEquals("only candidate RG bitmaps should remain", 3, bitmaps.size()); + assertTrue(bitmaps.containsKey(RetinaUtils.buildRgKey(candidateA, 0))); + assertTrue(bitmaps.containsKey(RetinaUtils.buildRgKey(candidateB, 0))); + assertTrue(bitmaps.containsKey(RetinaUtils.buildRgKey(candidateB, 1))); + assertFalse(bitmaps.containsKey(RetinaUtils.buildRgKey(exactlyThreshold, 0))); + assertFalse(bitmaps.containsKey(RetinaUtils.buildRgKey(zeroRows, 0))); + assertFalse(bitmaps.containsKey(RetinaUtils.buildRgKey(belowThreshold, 0))); + assertFalse("empty scan result must skip process phase", trackingGc.processCalled); } /** - * A file whose {@code fileStats} entry has {@code totalRows=0} must not - * produce a candidate even if invalidCount is also 0 (division by zero guard). + * The process phase must see the safe GC timestamp, the groups returned from + * scan, and a bitmap map already trimmed to candidate files. This protects + * the Storage GC rewrite path from accidentally consuming non-candidate RGs. */ @Test - public void testRunStorageGC_skipsTotalRowsZero() + public void testRunStorageGC_processSeesTrimmedCandidateBitmapsAndSafeTs() { - long fileId = 58001L; + long candidateFileId = 66301L; + long otherFileId = 66302L; + long safeGcTs = 303L; + + StorageGarbageCollector.FileGroup group = new StorageGarbageCollector.FileGroup( + 7L, 4, Collections.singletonList( + new StorageGarbageCollector.FileCandidate( + makeFile(candidateFileId, 2), "fake_candidate", candidateFileId, 2, 7L, 4, 0.75, 0L))); + TrackingRunStorageGC trackingGc = new TrackingRunStorageGC(Collections.singletonList(group)); Map fileStats = new HashMap<>(); - fileStats.put(fileId, new long[]{0, 0}); // totalRows=0 + fileStats.put(candidateFileId, makeRgStats(100, 75)); + fileStats.put(otherFileId, makeRgStats(100, 10)); Map bitmaps = new HashMap<>(); - bitmaps.put(fileId + "_0", new long[]{0L}); + bitmaps.put(RetinaUtils.buildRgKey(candidateFileId, 0), makeBitmap(100, 75)); + bitmaps.put(RetinaUtils.buildRgKey(candidateFileId, 1), makeBitmap(100, 60)); + bitmaps.put(RetinaUtils.buildRgKey(otherFileId, 0), makeBitmap(100, 10)); + + trackingGc.runStorageGC(safeGcTs, fileStats, bitmaps); + + assertTrue("process phase must run for non-empty groups", trackingGc.processCalled); + assertEquals("safeGcTs must be forwarded to process phase", safeGcTs, trackingGc.capturedSafeGcTs); + assertEquals("scan groups must be forwarded unchanged", 1, trackingGc.capturedFileGroups.size()); + assertEquals(candidateFileId, trackingGc.capturedFileGroups.get(0).files.get(0).fileId); + assertEquals(new HashSet<>(Arrays.asList( + RetinaUtils.buildRgKey(candidateFileId, 0), + RetinaUtils.buildRgKey(candidateFileId, 1))), trackingGc.bitmapKeysSeenByProcess); + assertFalse("non-candidate bitmap must be trimmed before process", + bitmaps.containsKey(RetinaUtils.buildRgKey(otherFileId, 0))); + } - DirectScanStorageGC gc = new DirectScanStorageGC( - retinaManager, 0.5, 10, - Collections.singletonList(new FakeFileEntry(fileId, 1, 1L, 0))); + /** + * If the downstream process phase fails, {@code runStorageGC} must already + * have released non-candidate bitmaps. This mirrors the real GC ordering: + * checkpoint is complete, then candidate-only rewrite state is retained. + */ + @Test + public void testRunStorageGC_processFailureKeepsOnlyCandidateBitmaps() + { + long candidateFileId = 66401L; + long otherFileId = 66402L; - gc.runStorageGC(500L, fileStats, bitmaps); + StorageGarbageCollector.FileGroup group = new StorageGarbageCollector.FileGroup( + 8L, 0, Collections.singletonList( + new StorageGarbageCollector.FileCandidate( + makeFile(candidateFileId, 1), "fake_candidate", candidateFileId, 1, 8L, 0, 0.80, 0L))); + TrackingRunStorageGC trackingGc = new TrackingRunStorageGC(Collections.singletonList(group)); + trackingGc.processFailure = new RuntimeException("simulated process failure"); - assertTrue("totalRows=0 file must remain untouched (no candidates)", - bitmaps.containsKey(fileId + "_0")); + Map fileStats = new HashMap<>(); + fileStats.put(candidateFileId, makeRgStats(100, 80)); + fileStats.put(otherFileId, makeRgStats(100, 20)); + + Map bitmaps = new HashMap<>(); + bitmaps.put(RetinaUtils.buildRgKey(candidateFileId, 0), makeBitmap(100, 80)); + bitmaps.put(RetinaUtils.buildRgKey(otherFileId, 0), makeBitmap(100, 20)); + + try + { + trackingGc.runStorageGC(304L, fileStats, bitmaps); + fail("process failure should propagate to the caller"); + } + catch (RuntimeException e) + { + assertEquals("simulated process failure", e.getMessage()); + } + + assertTrue("process phase should have been entered", trackingGc.processCalled); + assertTrue("candidate bitmap remains available for failure handling", + bitmaps.containsKey(RetinaUtils.buildRgKey(candidateFileId, 0))); + assertFalse("non-candidate bitmap must remain released after failure", + bitmaps.containsKey(RetinaUtils.buildRgKey(otherFileId, 0))); } // ======================================================================= - // Section 4b: processFileGroups error handling + // Section 4: processFileGroups error handling // ======================================================================= /** @@ -3959,6 +4031,53 @@ void processFileGroups(List fileGroups, long safeGcTs, } } + /** + * StorageGarbageCollector subclass that records the boundaries between + * {@code runStorageGC}'s candidate calculation, scan, bitmap trimming, and + * process phases without touching real metadata or Pixels files. + */ + static class TrackingRunStorageGC extends StorageGarbageCollector + { + private final List groupsToReturn; + boolean scanCalled; + boolean processCalled; + RuntimeException processFailure; + Set capturedCandidateFileIds = Collections.emptySet(); + List capturedFileGroups = Collections.emptyList(); + long capturedSafeGcTs = Long.MIN_VALUE; + Set bitmapKeysSeenByProcess = Collections.emptySet(); + + TrackingRunStorageGC(List groupsToReturn) + { + super(null, null, 0.5, 0L, Integer.MAX_VALUE, 10, + 1048576, EncodingLevel.EL2, 86_400_000L); + this.groupsToReturn = groupsToReturn; + } + + @Override + List scanAndGroupFiles(Set candidateFileIds, + Map fileStats) + { + this.scanCalled = true; + this.capturedCandidateFileIds = new HashSet<>(candidateFileIds); + return groupsToReturn; + } + + @Override + void processFileGroups(List fileGroups, long safeGcTs, + Map gcSnapshotBitmaps) + { + this.processCalled = true; + this.capturedFileGroups = new ArrayList<>(fileGroups); + this.capturedSafeGcTs = safeGcTs; + this.bitmapKeysSeenByProcess = new HashSet<>(gcSnapshotBitmaps.keySet()); + if (processFailure != null) + { + throw processFailure; + } + } + } + /** * StorageGarbageCollector subclass where {@code rewriteFileGroup} throws on * the first call and succeeds (cleaning up bitmaps) on subsequent calls. From 9515a36149e346ce8ca92dddc144df1a4d2990a0 Mon Sep 17 00:00:00 2001 From: Dongyang Geng Date: Wed, 13 May 2026 21:01:37 +0800 Subject: [PATCH 08/17] feat(metadata)!: track cleanup deadlines for retired files --- .../cli/load/AbstractPixelsConsumer.java | 4 +- .../common/metadata/MetadataService.java | 6 +- .../pixels/common/metadata/domain/File.java | 53 ++- .../metadata/domain/TestFileDomain.java | 336 +++++++++++++++ .../pixels/daemon/metadata/dao/FileDao.java | 4 +- .../daemon/metadata/dao/impl/RdbFileDao.java | 75 ++-- .../src/main/resources/pixels_metadata.mwb | Bin 23880 -> 24473 bytes .../dao/impl/TestRdbFileDaoCleanupAt.java | 406 ++++++++++++++++++ .../pixels/retina/FileWriterManager.java | 2 +- .../retina/StorageGarbageCollector.java | 10 +- .../retina/TestIngestFilePublisher.java | 30 +- .../retina/TestStorageGarbageCollector.java | 167 ++++++- proto/metadata.proto | 7 +- scripts/sql/metadata_schema.sql | 4 +- 14 files changed, 1014 insertions(+), 90 deletions(-) create mode 100644 pixels-common/src/test/java/io/pixelsdb/pixels/common/metadata/domain/TestFileDomain.java create mode 100644 pixels-daemon/src/test/java/io/pixelsdb/pixels/daemon/metadata/dao/impl/TestRdbFileDaoCleanupAt.java diff --git a/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/load/AbstractPixelsConsumer.java b/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/load/AbstractPixelsConsumer.java index f80459c25b..373ca3b83c 100644 --- a/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/load/AbstractPixelsConsumer.java +++ b/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/load/AbstractPixelsConsumer.java @@ -163,7 +163,7 @@ private void cleanupTemporaryFiles() { for (File tmpFile : tmpFiles) { - if (tmpFile.getType() == File.Type.TEMPORARY) + if (tmpFile.getType() == File.Type.TEMPORARY_INGEST) { try { @@ -210,7 +210,7 @@ protected File openTmpFile(String fileName, Path filePath) throws MetadataExcept { File file = new File(); file.setName(fileName); - file.setType(File.Type.TEMPORARY); + file.setType(File.Type.TEMPORARY_INGEST); file.setNumRowGroup(1); file.setPathId(filePath.getId()); String tmpFilePath = filePath.getUri() + "/" + fileName; diff --git a/pixels-common/src/main/java/io/pixelsdb/pixels/common/metadata/MetadataService.java b/pixels-common/src/main/java/io/pixelsdb/pixels/common/metadata/MetadataService.java index 615127bf1c..f840c21ce7 100644 --- a/pixels-common/src/main/java/io/pixelsdb/pixels/common/metadata/MetadataService.java +++ b/pixels-common/src/main/java/io/pixelsdb/pixels/common/metadata/MetadataService.java @@ -1420,7 +1420,7 @@ public File.Type getFileType(String filePathUri) throws MetadataException { throw new MetadataException("response token does not match."); } - return File.Type.valueOf(response.getFileType().getNumber()); + return File.Type.valueOf(response.getFileTypeValue()); } catch (Exception e) { @@ -1540,8 +1540,8 @@ public File getFileById(long fileId) throws MetadataException } /** - * Atomically promote a TEMPORARY file to REGULAR and delete the old files. - * @param newFileId the id of the new TEMPORARY file to promote + * Atomically promote a temporary file to REGULAR and delete the old files. + * @param newFileId the id of the new temporary file to promote * @param oldFileIds the ids of old files to delete * @throws MetadataException if the request fails */ diff --git a/pixels-common/src/main/java/io/pixelsdb/pixels/common/metadata/domain/File.java b/pixels-common/src/main/java/io/pixelsdb/pixels/common/metadata/domain/File.java index 7dd46ecdc3..a567b82939 100644 --- a/pixels-common/src/main/java/io/pixelsdb/pixels/common/metadata/domain/File.java +++ b/pixels-common/src/main/java/io/pixelsdb/pixels/common/metadata/domain/File.java @@ -33,22 +33,37 @@ */ public class File extends Base { - /** - * Files such as loaded and compacted are marked as REGULAR, while file - * created by pixelsWriterImpl during build are marked as TEMPORARY. - */ public enum Type { - TEMPORARY, REGULAR; + TEMPORARY_INGEST(0), + REGULAR(1), + TEMPORARY_GC(2), + RETIRED(3); + + private final int number; + + Type(int number) + { + this.number = number; + } + + public int getNumber() + { + return number; + } public static Type valueOf(int number) { switch (number) { case 0: - return TEMPORARY; + return TEMPORARY_INGEST; case 1: return REGULAR; + case 2: + return TEMPORARY_GC; + case 3: + return RETIRED; default: throw new InvalidArgumentException("invalid number for File.Type"); } @@ -61,6 +76,7 @@ public static Type valueOf(int number) private long minRowId; private long maxRowId; private long pathId; + private Long cleanupAt; public File() { @@ -70,11 +86,12 @@ public File(MetadataProto.File file) { this.setId(file.getId()); this.name = file.getName(); - this.type = Type.valueOf(file.getType().getNumber()); + this.type = Type.valueOf(file.getTypeValue()); this.numRowGroup = file.getNumRowGroup(); this.minRowId = file.getMinRowId(); this.maxRowId = file.getMaxRowId(); this.pathId = file.getPathId(); + this.cleanupAt = file.hasCleanupAt() ? file.getCleanupAt() : null; } public String getName() @@ -137,6 +154,16 @@ public void setPathId(long pathId) this.pathId = pathId; } + public Long getCleanupAt() + { + return cleanupAt; + } + + public void setCleanupAt(Long cleanupAt) + { + this.cleanupAt = cleanupAt; + } + public static List convertFiles(List protoFiles) { requireNonNull(protoFiles, "protoFiles is null"); @@ -182,8 +209,14 @@ public static String getFilePath(Path path, File file) @Override public MetadataProto.File toProto() { - return MetadataProto.File.newBuilder().setId(this.getId()).setName(this.name) - .setTypeValue(this.type.ordinal()).setNumRowGroup(this.numRowGroup) - .setMinRowId(this.minRowId).setMaxRowId(this.maxRowId).setPathId(this.pathId).build(); + MetadataProto.File.Builder builder = MetadataProto.File.newBuilder() + .setId(this.getId()).setName(this.name) + .setTypeValue(this.type.getNumber()).setNumRowGroup(this.numRowGroup) + .setMinRowId(this.minRowId).setMaxRowId(this.maxRowId).setPathId(this.pathId); + if (this.cleanupAt != null) + { + builder.setCleanupAt(this.cleanupAt); + } + return builder.build(); } } diff --git a/pixels-common/src/test/java/io/pixelsdb/pixels/common/metadata/domain/TestFileDomain.java b/pixels-common/src/test/java/io/pixelsdb/pixels/common/metadata/domain/TestFileDomain.java new file mode 100644 index 0000000000..5e66d66e29 --- /dev/null +++ b/pixels-common/src/test/java/io/pixelsdb/pixels/common/metadata/domain/TestFileDomain.java @@ -0,0 +1,336 @@ +/* + * Copyright 2026 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the Affero GNU General Public + * License along with Pixels. If not, see + * . + */ +package io.pixelsdb.pixels.common.metadata.domain; + +import io.pixelsdb.pixels.common.exception.InvalidArgumentException; +import io.pixelsdb.pixels.daemon.MetadataProto; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Unit tests for {@link File} that exercise the c01.1 contract: + *
    + *
  • {@link File.Type} now carries an explicit numeric tag (no longer relies on {@code ordinal()}).
    • + *
  • The four enum constants — {@code TEMPORARY_INGEST(0)}, {@code REGULAR(1)}, + * {@code TEMPORARY_GC(2)}, {@code RETIRED(3)} — must round-trip cleanly through both + * {@link MetadataProto.File} and the domain object.
    • + *
  • {@link File#getCleanupAt()} is an optional field: it must be preserved across + * {@link File#toProto()} / {@code new File(MetadataProto.File)} when present and absent.
    • + *
+ * + * @author tdd-guide + * @create 2026-05-13 + */ +public class TestFileDomain +{ + // ------------------------------------------------------------------------- + // File.Type — numeric tags + // ------------------------------------------------------------------------- + + /** + * The domain {@link File.Type#getNumber()} must agree with the proto-generated + * {@link MetadataProto.File.Type#getNumber()} for every constant we publish. + * This guards against the previous implementation that relied on + * {@code ordinal()} and would silently re-number constants when the enum order changed. + */ + @Test + public void typeNumber_isConsistentWithProtoEnum() + { + assertEquals(MetadataProto.File.Type.TEMPORARY_INGEST.getNumber(), + File.Type.TEMPORARY_INGEST.getNumber()); + assertEquals(MetadataProto.File.Type.REGULAR.getNumber(), + File.Type.REGULAR.getNumber()); + assertEquals(MetadataProto.File.Type.TEMPORARY_GC.getNumber(), + File.Type.TEMPORARY_GC.getNumber()); + assertEquals(MetadataProto.File.Type.RETIRED.getNumber(), + File.Type.RETIRED.getNumber()); + } + + // ------------------------------------------------------------------------- + // File.Type.valueOf(int) — happy path + boundaries + // ------------------------------------------------------------------------- + + @Test + public void typeValueOf_resolvesAllKnownNumbers() + { + assertSame(File.Type.TEMPORARY_INGEST, File.Type.valueOf(0)); + assertSame(File.Type.REGULAR, File.Type.valueOf(1)); + assertSame(File.Type.TEMPORARY_GC, File.Type.valueOf(2)); + assertSame(File.Type.RETIRED, File.Type.valueOf(3)); + } + + @Test + public void typeValueOf_rejectsInvalidNumbers() + { + // Test various boundary cases for invalid type numbers + int[] invalidNumbers = {-1, 4, Integer.MAX_VALUE, Integer.MIN_VALUE}; + + for (int invalidNumber : invalidNumbers) + { + try + { + File.Type.valueOf(invalidNumber); + fail("expected InvalidArgumentException for number: " + invalidNumber); + } + catch (InvalidArgumentException expected) + { + assertNotNull("Exception message should not be null for number: " + invalidNumber, + expected.getMessage()); + } + } + } + + /** + * Round-trip: every constant survives {@code num -> valueOf -> getNumber}. + */ + @Test + public void typeValueOf_roundTripForAllConstants() + { + for (File.Type t : File.Type.values()) + { + assertSame("round-trip failed for " + t, + t, File.Type.valueOf(t.getNumber())); + } + } + + // ------------------------------------------------------------------------- + // cleanupAt — getter / setter + // ------------------------------------------------------------------------- + + @Test + public void cleanupAt_defaultsToNullOnNoArgConstructor() + { + File f = new File(); + assertNull("a freshly constructed File must have a null cleanupAt", f.getCleanupAt()); + } + + @Test + public void cleanupAt_setterAcceptsValueAndNull() + { + File f = new File(); + f.setCleanupAt(123_456_789L); + assertEquals(Long.valueOf(123_456_789L), f.getCleanupAt()); + + // explicit clear must be supported (used after promote-to-REGULAR) + f.setCleanupAt(null); + assertNull(f.getCleanupAt()); + } + + // ------------------------------------------------------------------------- + // toProto / fromProto round-trip + // ------------------------------------------------------------------------- + + /** + * When {@code cleanupAt == null}, {@link File#toProto()} must NOT set the optional + * field on the wire. Otherwise downstream consumers calling {@code hasCleanupAt()} + * would see a spurious zero deadline. + */ + @Test + public void toProto_omitsCleanupAt_whenDomainValueIsNull() + { + File f = makeFile(1L, "n.pxl", File.Type.TEMPORARY_INGEST, 1, 0L, 0L, 1L, null); + + MetadataProto.File proto = f.toProto(); + + assertFalse("cleanupAt must be absent on the wire when domain value is null", + proto.hasCleanupAt()); + } + + /** + * cleanupAt = 0L is a legitimate value (epoch start); it must NOT be confused with "absent". + * Without this guard, a naïve {@code if (cleanupAt != 0)} check would silently drop the field. + */ + @Test + public void toProto_includesCleanupAt_whenValueIsZero() + { + File f = makeFile(1L, "z.pxl", File.Type.RETIRED, 1, 0L, 0L, 1L, 0L); + + MetadataProto.File proto = f.toProto(); + + assertTrue("cleanupAt = 0L must be carried on the wire (zero != absent)", + proto.hasCleanupAt()); + assertEquals(0L, proto.getCleanupAt()); + } + + @Test + public void fromProto_preservesCleanupAt_whenSet() + { + long deadline = 1_700_000_123_456L; + MetadataProto.File proto = MetadataProto.File.newBuilder() + .setId(42L) + .setName("retired.pxl") + .setTypeValue(File.Type.RETIRED.getNumber()) + .setNumRowGroup(2) + .setMinRowId(0L) + .setMaxRowId(127L) + .setPathId(9L) + .setCleanupAt(deadline) + .build(); + + File f = new File(proto); + + assertEquals(42L, f.getId()); + assertEquals("retired.pxl", f.getName()); + assertSame(File.Type.RETIRED, f.getType()); + assertEquals(2, f.getNumRowGroup()); + assertEquals(0L, f.getMinRowId()); + assertEquals(127L, f.getMaxRowId()); + assertEquals(9L, f.getPathId()); + assertNotNull("cleanupAt must be retained from the proto", f.getCleanupAt()); + assertEquals(Long.valueOf(deadline), f.getCleanupAt()); + } + + /** + * If the proto omits the optional cleanupAt, the domain object MUST observe {@code null} + * (not 0L). This is the reciprocal of {@link #toProto_omitsCleanupAt_whenDomainValueIsNull()}. + */ + @Test + public void fromProto_returnsNullCleanupAt_whenAbsent() + { + MetadataProto.File proto = MetadataProto.File.newBuilder() + .setId(1L) + .setName("tmp.pxl") + .setTypeValue(File.Type.TEMPORARY_GC.getNumber()) + .setNumRowGroup(1) + .setMinRowId(0L) + .setMaxRowId(0L) + .setPathId(1L) + .build(); + + File f = new File(proto); + + assertNull("absent cleanupAt on the wire must materialise as null in the domain", + f.getCleanupAt()); + } + + /** + * End-to-end round-trip — domain → proto → domain — must be lossless for every {@link File.Type}. + */ + @Test + public void roundTrip_domainProtoDomain_isLossless_forEveryType() + { + for (File.Type t : File.Type.values()) + { + // RETIRED carries cleanupAt; the others should not. We deliberately set cleanupAt + // independently of type to verify the domain object preserves whatever it is given. + Long cleanup = (t == File.Type.RETIRED) ? 1_700_000_000_999L : null; + File original = makeFile(7L, "x_" + t + ".pxl", t, 1, 0L, 63L, 3L, cleanup); + + File restored = new File(original.toProto()); + + assertEquals("id mismatch for " + t, original.getId(), restored.getId()); + assertEquals("name mismatch for " + t, original.getName(), restored.getName()); + assertSame("type mismatch for " + t, original.getType(), restored.getType()); + assertEquals("numRowGroup mismatch for " + t, + original.getNumRowGroup(), restored.getNumRowGroup()); + assertEquals("minRowId mismatch for " + t, + original.getMinRowId(), restored.getMinRowId()); + assertEquals("maxRowId mismatch for " + t, + original.getMaxRowId(), restored.getMaxRowId()); + assertEquals("pathId mismatch for " + t, + original.getPathId(), restored.getPathId()); + assertEquals("cleanupAt mismatch for " + t, + original.getCleanupAt(), restored.getCleanupAt()); + } + } + + // ------------------------------------------------------------------------- + // convertFiles / revertFiles + // ------------------------------------------------------------------------- + + @Test + public void convertFiles_handlesEmptyList() + { + List result = File.convertFiles(Collections.emptyList()); + assertNotNull(result); + assertTrue(result.isEmpty()); + } + + @Test(expected = NullPointerException.class) + public void convertFiles_rejectsNullInput() + { + File.convertFiles(null); + } + + @Test + public void convertFiles_thenRevertFiles_isLossless() + { + MetadataProto.File p1 = MetadataProto.File.newBuilder() + .setId(10L).setName("a.pxl") + .setTypeValue(File.Type.REGULAR.getNumber()) + .setNumRowGroup(1).setMinRowId(0L).setMaxRowId(63L).setPathId(1L) + .build(); + MetadataProto.File p2 = MetadataProto.File.newBuilder() + .setId(11L).setName("b.pxl") + .setTypeValue(File.Type.RETIRED.getNumber()) + .setNumRowGroup(2).setMinRowId(64L).setMaxRowId(127L).setPathId(1L) + .setCleanupAt(1_700_000_000_000L) + .build(); + + List domain = File.convertFiles(Arrays.asList(p1, p2)); + assertEquals(2, domain.size()); + assertSame(File.Type.REGULAR, domain.get(0).getType()); + assertNull(domain.get(0).getCleanupAt()); + assertSame(File.Type.RETIRED, domain.get(1).getType()); + assertEquals(Long.valueOf(1_700_000_000_000L), domain.get(1).getCleanupAt()); + + List back = File.revertFiles(domain); + assertEquals(2, back.size()); + assertEquals(p1, back.get(0)); + assertEquals(p2, back.get(1)); + } + + @Test(expected = NullPointerException.class) + public void revertFiles_rejectsNullInput() + { + File.revertFiles(null); + } + + // ------------------------------------------------------------------------- + // helpers + // ------------------------------------------------------------------------- + + private static File makeFile(long id, String name, File.Type type, + int numRowGroup, long minRowId, long maxRowId, + long pathId, Long cleanupAt) + { + File f = new File(); + f.setId(id); + f.setName(name); + f.setType(type); + f.setNumRowGroup(numRowGroup); + f.setMinRowId(minRowId); + f.setMaxRowId(maxRowId); + f.setPathId(pathId); + f.setCleanupAt(cleanupAt); + return f; + } +} diff --git a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/dao/FileDao.java b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/dao/FileDao.java index b5ae2b9d1c..d400256e7a 100644 --- a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/dao/FileDao.java +++ b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/dao/FileDao.java @@ -78,8 +78,8 @@ public boolean save (MetadataProto.File file) abstract public boolean deleteByIds (List ids); /** - * Atomically promote a TEMPORARY file to REGULAR and delete the old files in a single transaction. - * @param newFileId the id of the new TEMPORARY file to promote + * Atomically promote a temporary file to REGULAR and delete the old files in a single transaction. + * @param newFileId the id of the new temporary file to promote * @param oldFileIds the ids of old files to delete * @return true if the transaction committed successfully */ diff --git a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/dao/impl/RdbFileDao.java b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/dao/impl/RdbFileDao.java index ccae356b8c..8db4d04783 100644 --- a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/dao/impl/RdbFileDao.java +++ b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/dao/impl/RdbFileDao.java @@ -42,6 +42,36 @@ public RdbFileDao() { } private static final MetaDBUtil db = MetaDBUtil.Instance(); + private static MetadataProto.File buildFile(ResultSet rs) throws SQLException + { + MetadataProto.File.Builder builder = MetadataProto.File.newBuilder() + .setId(rs.getLong("FILE_ID")) + .setName(rs.getString("FILE_NAME")) + .setTypeValue(rs.getInt("FILE_TYPE")) + .setNumRowGroup(rs.getInt("FILE_NUM_RG")) + .setMinRowId(rs.getLong("FILE_MIN_ROW_ID")) + .setMaxRowId(rs.getLong("FILE_MAX_ROW_ID")) + .setPathId(rs.getLong("PATHS_PATH_ID")); + long cleanupAt = rs.getLong("FILE_CLEANUP_AT"); + if (!rs.wasNull()) + { + builder.setCleanupAt(cleanupAt); + } + return builder.build(); + } + + private static void setCleanupAt(PreparedStatement pst, int index, MetadataProto.File file) throws SQLException + { + if (file.getTypeValue() == MetadataProto.File.Type.RETIRED.getNumber() && file.hasCleanupAt()) + { + pst.setLong(index, file.getCleanupAt()); + } + else + { + pst.setNull(index, Types.BIGINT); + } + } + @Override public MetadataProto.File getById(long id) { @@ -51,13 +81,7 @@ public MetadataProto.File getById(long id) ResultSet rs = st.executeQuery("SELECT * FROM FILES WHERE FILE_ID=" + id); if (rs.next()) { - return MetadataProto.File.newBuilder().setId(id) - .setName(rs.getString("FILE_NAME")) - .setTypeValue(rs.getInt("FILE_TYPE")) - .setNumRowGroup(rs.getInt("FILE_NUM_RG")) - .setMinRowId(rs.getLong("FILE_MIN_ROW_ID")) - .setMaxRowId(rs.getLong("FILE_MAX_ROW_ID")) - .setPathId(rs.getLong("PATHS_PATH_ID")).build(); + return buildFile(rs); } } catch (SQLException e) { @@ -81,15 +105,7 @@ public List getAllByPathId(long pathId) List files = new ArrayList<>(); while (rs.next()) { - MetadataProto.File.Builder builder = MetadataProto.File.newBuilder() - .setId(rs.getLong("FILE_ID")) - .setTypeValue(rs.getInt("FILE_TYPE")) - .setName(rs.getString("FILE_NAME")) - .setNumRowGroup(rs.getInt("FILE_NUM_RG")) - .setMinRowId(rs.getLong("FILE_MIN_ROW_ID")) - .setMaxRowId(rs.getLong("FILE_MAX_ROW_ID")) - .setPathId(rs.getLong("PATHS_PATH_ID")); - files.add(builder.build()); + files.add(buildFile(rs)); } return files; } catch (SQLException e) @@ -104,7 +120,7 @@ public List getAllByPathId(long pathId) public MetadataProto.File getByPathIdAndFileName(long pathId, String fileName) { Connection conn = db.getConnection(); - String sql = "SELECT FILE_ID, FILE_TYPE, FILE_NUM_RG, FILE_MIN_ROW_ID, FILE_MAX_ROW_ID FROM FILES WHERE PATHS_PATH_ID=? AND FILE_NAME=?"; + String sql = "SELECT * FROM FILES WHERE PATHS_PATH_ID=? AND FILE_NAME=?"; try (PreparedStatement st = conn.prepareStatement(sql)) { st.setLong(1, pathId); @@ -112,14 +128,7 @@ public MetadataProto.File getByPathIdAndFileName(long pathId, String fileName) ResultSet rs = st.executeQuery(); if (rs.next()) { - return MetadataProto.File.newBuilder() - .setId(rs.getLong("FILE_ID")) - .setName(fileName) - .setTypeValue(rs.getInt("FILE_TYPE")) - .setNumRowGroup(rs.getInt("FILE_NUM_RG")) - .setMinRowId(rs.getLong("FILE_MIN_ROW_ID")) - .setMaxRowId(rs.getLong("FILE_MAX_ROW_ID")) - .setPathId(pathId).build(); + return buildFile(rs); } } catch (SQLException e) { @@ -159,7 +168,8 @@ public long insert(MetadataProto.File file) "`FILE_NUM_RG`," + "`FILE_MIN_ROW_ID`," + "`FILE_MAX_ROW_ID`," + - "`PATHS_PATH_ID`) VALUES (?,?,?,?,?,?)"; + "`PATHS_PATH_ID`," + + "`FILE_CLEANUP_AT`) VALUES (?,?,?,?,?,?,?)"; try (PreparedStatement pst = conn.prepareStatement(sql)) { pst.setString(1, file.getName()); @@ -168,6 +178,7 @@ public long insert(MetadataProto.File file) pst.setLong(4, file.getMinRowId()); pst.setLong(5, file.getMaxRowId()); pst.setLong(6, file.getPathId()); + setCleanupAt(pst, 7, file); if (pst.executeUpdate() == 1) { ResultSet rs = pst.executeQuery("SELECT LAST_INSERT_ID()"); @@ -202,7 +213,8 @@ public boolean insertBatch(List files) "`FILE_NUM_RG`," + "`FILE_MIN_ROW_ID`," + "`FILE_MAX_ROW_ID`," + - "`PATHS_PATH_ID`) VALUES (?,?,?,?,?,?)"; + "`PATHS_PATH_ID`," + + "`FILE_CLEANUP_AT`) VALUES (?,?,?,?,?,?,?)"; try (PreparedStatement pst = conn.prepareStatement(sql)) { for (MetadataProto.File file : files) @@ -213,6 +225,7 @@ public boolean insertBatch(List files) pst.setLong(4, file.getMinRowId()); pst.setLong(5, file.getMaxRowId()); pst.setLong(6, file.getPathId()); + setCleanupAt(pst, 7, file); pst.addBatch(); } pst.executeBatch(); @@ -233,7 +246,8 @@ public boolean update(MetadataProto.File file) "`FILE_TYPE` = ?," + "`FILE_NUM_RG` = ?," + "`FILE_MIN_ROW_ID` = ?," + - "`FILE_MAX_ROW_ID` = ?\n" + + "`FILE_MAX_ROW_ID` = ?," + + "`FILE_CLEANUP_AT` = ?\n" + "WHERE `FILE_ID` = ?"; try (PreparedStatement pst = conn.prepareStatement(sql)) { @@ -242,7 +256,8 @@ public boolean update(MetadataProto.File file) pst.setInt(3, file.getNumRowGroup()); pst.setLong(4, file.getMinRowId()); pst.setLong(5, file.getMaxRowId()); - pst.setLong(6, file.getId()); + setCleanupAt(pst, 6, file); + pst.setLong(7, file.getId()); return pst.executeUpdate() == 1; } catch (SQLException e) { @@ -282,7 +297,7 @@ public boolean atomicSwapFiles(long newFileId, List oldFileIds) { conn.setAutoCommit(false); try (PreparedStatement pst = conn.prepareStatement( - "UPDATE FILES SET FILE_TYPE=? WHERE FILE_ID=?")) + "UPDATE FILES SET FILE_TYPE=?, FILE_CLEANUP_AT=NULL WHERE FILE_ID=?")) { pst.setInt(1, MetadataProto.File.Type.REGULAR.getNumber()); pst.setLong(2, newFileId); diff --git a/pixels-daemon/src/main/resources/pixels_metadata.mwb b/pixels-daemon/src/main/resources/pixels_metadata.mwb index 3a9176fa93ecd3a52b1228e692646422f9c7aa7f..0874f98600210b37d47f7d4859eaf149277bf89e 100644 GIT binary patch delta 23717 zcmZ^~Wl$bn8-$6wySqCCcXxMpCwOof;KAM9-Q5YnT@u_qxC99VXL-N1s;$~zbEayh z=GUCQ?(RP0-QZb$;CQMEkWg4)5MXd%C#iLYooZn+CI(<&!Z%5HOn964Mlo&K&YH>L zrM^BUwsK^0MKMskV7f_$WI&(E>`$1!jFu;F{-*?;M=sxcVCS^~AjOU`ci}f6%2TvC z9@!T$PWl@&V9Wnqkd-#3WMn1gR>b)2`F^)=8|9rl@6FRtIlKU-Pg&-0ttMx5)#2f6 z=7Rv~{q5iWMKe(?OAg6L3lWPk@=?s%p$mA*zF@%P)B20ZRxD9c#Sxb4DVBc27PrR0o237;7uYF?`5AwTZ8s%_}hGU*vQ@O#9$3TYRv z2#d1K$dlF==^C#)9U!m}KvpM49qHhX9bM{RZXhw9#a(bO2pwV zji0{Sx4GyT6_aBgn~i4t6;c+FoWAb6C?Sq0=Z!k+ri}O-IyRSZPBsZTx*>W=%4V7Y}_54_yp0|8O&jH6Em%=cMG%RF?p1uSapQCYefbz=u=%xhjCqbXSWy-t?g_QN%~dv}oc=!N`ukJo&l&d~UJ**R zi!M=tk=IDEuhGZ36s*a|N~o>Qj&bnDn@J!nE?ye#$l=;x0E6|2i`p#8V5a=SVS)Y6LNiSe%$uf z$XG*t&)tyqV?tZ9x*Jr!T=q%PP~tv6B!{{R9ugy< zf13yhFa(Nh6$AzvC@bR@T-{rv79D9}UdhD5=1buw8Say09H%%tb@_hjYe8L(ac!v8 zP6)MNs`JDlvWEAiYBtT|5DmlHoIpeG+DIV5l~030{bM)V8=!BBOskCsB0X7+ zAa!~=G}oT3S(<_+y@XWKYwDDYCJFq3%5wnZB!Nc0Y3HV&a~;WSzD0E`fAMhmed8HF zu7$d$M0qBR|`F)$8NeZA#02>s8C+_VUf^*0&~m3MbDF8=ZVC0^TRP$XlHetXVUnm@=(}4|M-WhjPPo0_^$|Rn*mqU%+hdnaX4FaE=gy>qsd=r z<1|I6Rww5K&!MZSrq+TNq04pN6W2FT zs0Y&}8jd|7@5(Be7d$w=9I{3(1Dv*u9kF)&X>3^Uptu|d&I18r*`#h}QzJVL$@_tX zjIQq3Jc=2)PBM8lv`ac#qLzW%BmxrMeqpY=pm$nRqlvEW5%t_X__fKeK0zJ;h9O-? z$`2;E?OHk-JzAGN_^D|tjuxm@zZ$0VV2Pd^V5nxi3Yotl9`di z+Qg>hPT}ba3oNiIGRck(8=IRKanKy4lqjvp&ivT%v)R5PelIDJntz7zI(J^Sn%Pb&&{*6KXO*HCNhR4QALFeBX7O`(lrN-fa8J^qd(cuVAY)YW#eH zax&ryTYCVRt`6_+vPGt%I^{y4pl-k`P@a?pOC2|xpvPFE6#skYym2H!JVhVN(Rj14`4~;$%EhLhqhv?Z|_Jsk_$Rmt?-K_=OZ!$9}|Tt zquJuTG}qq_^xfHba~LxFJM^{tyDax|juK&o1#KU3*FQ##rGZIFM~6DwImgI9;y*6Z zRR~b0P!iuJZK2OnDebe`&>}JVzsjP-kN%zf!cZ(aoY|vuD>J6cC`8n&{{b{%*;FhH z=LqNV#7@hU*2Ej(X0KfA=P*i6nh#Xb7KWzv6MZr5Jr455;`V6fy1K%qNHT=yFCT(k zzYjvOUJ7!MY8(>`;Q8_*_E>;AHBsrPu&mo$jcAHGC|(TCWjRFR`PakW826u>N~f+( zwumQW!pu^+v9Da)ySrgrj5Xkr8MxYVdffrK`AButwX!;RQ;+VmQwSveIX0zQ%7+Ii z$KOb>=jf%!WVzT*gOd+J{X$geAdP9@vH9i0PKCHaIT-x| zwRL6GoNM&V^6a*gvD= zZ78yOA3s+Oc~KVuNe(vL0h!3XgjYUS?OGB4i`6EQ)mOlnOk-3L-#~S=*Gu^IZ4Lim z=Ix>6=42|y*s0%|DgSxvQ?yeMVe+dhV^CP=N(Y4lhLx^=A3M}UmO>f zkTNFt0tqvkg2Tf`-^Vn~z)mA3(K0lMJ96Ez^t5)b0&RlTGSWH7#nbE0#CV)`1r>2k4jW5Ky`fr8k9qU*aFw=x`l~S= zr9rN;MvSaA#-*Vie$A)ZwfKGY<8Sulu$Wp|zdxOF4W(aYaMIKP49a0WVYgZ3R%x$Y zRg90{f^m#%OYMdpJH1GL8#_8pg;v!nk6+5f;>M9@x;xNkWq^W#wse-TQK8(U+sc<4 zr=o)C+GGFo%!s(O6JLpli6wy$`SsiB3g`VSp|O7^+}^hbTcSI*5<>!BJmnSq6eaNl zUo%*-P)PNlIj}q6;A^j%nw1-;etkV@VC_zSc#&*eZ#0Pe@y&GWmO!dAN7Snhu6&C^ zve)DG#Sgf8H{q;~B`Ou$hJqllfR2U9_nr4uU1blgOFrE(s){viHrA9zkz{D7f@RF0 zZc2o-SfJc9CNSs17S@rz$_N=ivsznur~ljum;GlDV3csSdM_n=3e&T~H!M6{KB~(6 z60yJyEXDB5gWwfK^_+s$9WD1~1y^gmwpn+Fx&R!@K6bQY{ls3#0p=>`tlgTN=5a3M zOEe7q(oTtdnvfn8(Cdzs;^<824vU#R6-cuR&Yf~5eCo^ z!bF|QyN&Thm!Kgg8=Wzow$v8J7^SJTx2`}7WmVKruTIougJDrq@*q12QRta|S5muc z2tY5SgNfW!HkyP6$^{g*H!h(mU8`GH-HO;tlZzglw!%&e_0>*XG|#B&QW>t2ea2l} zR2go3P=J*jb2q&F6>$d2R^hLgp?AavS!gpn6`p%PT8e3uOjOhA@Ux&28(w=Hw)&!k zHXakTAqRr?!Q$MUJsU5Lxmj9_aY?_fH4wHjiE#dP&54JxmVM?6<8-adN}5#t!)kM} z>grw5>9F zFEM~Lf+&Z6&utUQ9qX6RB$nz;pmxS33ZrK_Y|9pce@*pxLNPymLM(?ytUeb51*nbG zajx@Uua?T}cS2dgfmcIB1v5~0O~M!V@)K{WJJQic3M;rFvATuWDx*0zu9N)R4p&x} zbRe+rn426y_2vI1ph`&_*5A zW**6|DrMU`YV?GL562f2 zak)x)3wYhTIeUIuiO7#D2wl{Vn<7TjIt%-VptS44;sq$#LcunAG*v@YATb<)7-}@> z45_vkLc5Bi@c~VD3aM5Ia@kSOG~nOOO39l^Hv*Fb{suV|XR^1QQQEl!G8-$GZXNHi zEp(v?Q*=aot_6e~Q4YKZ00D_>(sWgXZIJ?G1~W8jYWu#;jPm%PWn? zHlvY)mazL>0-bzXr~)6!4>H`cQbGCiZA7G+iqY`5q>>raRcgF&b5CxTvp|BjU2?0oshQESnz0hj*-zgQQjHAI%+gFs@LJ)+TJ>&<8Q&Usw?l zY@LMq_qS*Z{BrWyP>5D=VA+R8Qgw7P!K$; zD7<8_uX&zTlEaaGSN2Bd{4Lge_F%4v*Jutu)?Jc-Lvgj|!RSV1wFm2HM!88B-9Ofv zimT~hg@a~DwjQlEvfjW0S4$dQcw+~o*r!DPZGzoNvpPe7Y`MOq+EXIFD3b~;x1Jg0 z4HSlh>s2YA;d^z|dI48>l^1K}m2%V0d)B>xVK$uQMPgga`S1I%X$@qhCjRhz?Kg}R zmh!{PihsRZecVLSBGctlF_sp+Ypt&Gp0^unZ8S~AYOt4lYXlGaY9$6GX-OoS8Uq8m zr`7!EQ)aaQJou2(pb}Bke*EL^Y5S=)X$#4jHEnumX9m_F$NiKp6gOMGF+58h*DzYD zM@x@!cQvAU!in`tlFOR?)&>89;JLNU)`g+cVUg#BJXacFOc|>tdru3T6;DmoB78tc zJoZD=Xg;N4b0j(&2?om-@d58cQS$v-ngO&dy}fpzdGLYJ{pU2oMuUrx{L8u$DnTBu zTI3?r1?Hbo0;XH^5b8{-{j;p;k??j8-^HYN-_r$3SS3E&^=ESlot`zB9jqL;wIVFZ zEneDGP~{~`adzyElBCwAOBg=%g{$lvwK_172sTIOT zP~43KUXO6eT)|`(y9?_MQMg3VHx^o)D2g9u@?x(mpwVRNK;aExPB<#{ku{s#*-#foGDMUJIqb~05D)XPje5pCn%02*Ik zt~<1h&13W*?x()(%ns2Bz$uCt@(g9r9!%UWUBU!@%=m95_YtR|{b>#K#vK`AjedH) z`}S~c@BW7LD`$cYm(jA0L(?GNsju&;#az&>fQ8m^bBu@Z4hKeGFl6nod+zLPO!<|~ z4?3K?N$`t|q5ZF`qmvPC+$xwZbwJ1@Pbne;Cae@S>va)t-FzW|CR>3C36Be%*VZp& ziA$PGB?Y7DjhqMl*c_>}Wj(e38)FXSq)KR0syb#anT`Xe+KH&hs+UMR&?P;B$Rb*9 zPVLN{bT0LJYd}~$2T`DrL)Vhv0+zSz@@=1?7}N%b9Zur3o#7FOI1$av?5Z6&F_? z{_L{s^D{}oAB)j|+6V6!2|Ag&tYTcen3JX-woO21{r%n~#re<@N2k`a3oYH}3SPQd zc$4||Zu#}#ZEQ%nbzSBe-_Mvq#k`1RFcRb9Z+}s#Wn`^V{k!OVT3}?L z(~cL;yg*NYH-{o_g-B;^Y@gxMxvoD04o}m!MB}=4TxsHg>J8K*wr}c57Dh1wP<5$< zQIRxcOw0X=V}%(6%evAk%p(+;NOf-^%N!C8ABEU0NOeM>bXw-(k_)d$5hJcaT0@Ydy;T;={5SyyaZsFL_h~s?kWYf#f!b;#NxBYlQc#)q` za&!1p{5r3s>Bp50v#?f~yJZYca=}cwGUdZZz*K(foHEASfS{qhcXcK^y%csZQCKOy zx6lG$STFo^;mA@b?4a~%sPyZq62(fOQ5E;Yt(fXz^@kQx zcw5D);RpML^-EepUGYF1=dJu8zh2|?kimU4C^}hscHsiYJ+woxKb~l#>SR6w0kah; zFXWF4fA7f1FHNIb@5I<^(4Jn4Z|O^*#!dm}n1P$eZ(##Q%=~>H_aIkd7QU614hgi_ zQm$|p6#D%UtsE>mN@m_Kj?u}OSmg8sD&GyyoW znv*NGh_0XQ7_4WPnPORwoCEFw0m?^N_mUFs$Z-KFL`n3z|Bd?9zP?2kH6G=C9iE7r zqbHz$RTbHRLl61|wk9EG!rX`t_FsxYpT4t$RqF)8cCYe9ZDp#|2R9+>Bn@z;NjV?P zua!GHd1F@w?{j(hBj8uC{F2i7RLh28Nk7nPqSt8y83D{jv4MA`EKObkvhsvsU=$YeCYz{)Y z`9w~Z@jl9Nm@sw0);yuOF>(+icGMYbXv-=8;}x-j|Cd*)dnUYlC?>lX<{id}#bIjq zgq0NXN24@w#0i*~OfBw9{;i&z1~~s~K#rILP+LxKDb{qIxgB$h^Ikn$_3czR&t&wM zX(%e~ zQ$zkqLaR_E4{&8q9=adc=e@-$?K1eBb~5md4x8qUs+!v|NhiZ8Nzu8A8~!BD$hI^D z0^|+TzWTXnIS|qx5fP2S>qsar%)=w{!cfq01xLl!)Va3l_x{0BB;{JQpWE{UtLtW< zdEWXG7hwhKdKbq~r%BOK%f_mUkcvM-?{uI{%M+2+nlEj)mab(-JW-sWr7y&>$pA$+ zGL$HOfc0FV3?o%pxI8Cmrph9*c2A=Q7~%PqB3Ti`|E*(@(6h;Q7Q>%340h9DR1I9+ zd0D#nDDgm+-%K`YwI8e3%r70vF!pI~!Lu4lqdiAjsn-s0Y8>Pr_|x7b9`O6tb_xI#@!#qa)QNzcnts)5salB~|x}gAK!V7?Xz?8jEde z*RQ_&FS<97wF}#%c9|bV_Y7k@la>dbwkG8^DiZq15Uvo97?DnW%-wTyxElMf6$Iw< z^mgawrTeA)X2Er2^3N4Kx+i4=*z#LDgXNyA%(UUX8@8{dX;`B7oNFkb&S?n@|MOZx z1bKRMnX)YjQfnc(-}k!le+c5Mq&GP6T)#LUoiDe2SW_-q#3$TqoqfF%3Kv5rT~BMm z7UphcF97mqNobVsQSG(`>)e_`HqX%PrY?_Pazq$MDgT|i&5!!|x3B}wW`!e^nMxvK zrx)LC&jOBO%-I``J_7leX2Tt8{VN%mW^og4e!Mh4{>Lb3cK<#ZWtP4OM*7ng;Z5yv zItefGOnE3TV>!B1@?##IY4(jQjsG3Z_8rED5qJNqL3>B<#>BzeTegkvc03d8CUd0vmzpyK`!6yZK>6RO^K6fvC{@3<#h)09I10yN zT<29V@~48nz0;CldG9sIqMtyL(bcL>f>J_5*G5|9;H9MT(s%UB8opzvOOqAT;=UvJ_1`QIHU{ZB^x^IOZ;Ll0cMXIG$s60Oe6E<#n)-4X5( zfsGO=vzef}6c8oLH^54p7yNyO@+LkHy{&UDOox_N3-8iF--zeb4 zdvjzol(NxqY%Ddt!I~la%3q}Nc0)ch32d_X+j2`tQU8B8mm622=sdn|hZ;_b#YoWt zkkKzkTdaG#woe7hWUfk}4cN#oPYL3&rFzdH*?$e5kT?2la>#JT)Z^Kil_)VJlk3o^VicIO4#=4nc_Oj&$1?cA4I z_JEAhZw5i#UK}VLG8WyvC$`I(KSW}B^ey|0^-iQMyMK%*E!j0nFBIxQ6@2$}`#o9F z1ftiiOWPkfBin2-r@Kp%55*tXJp`A|BO(C?F+$_WO{XZoluon?`8+(!c%(aU2New! z!{Yg(p}=|+KU$L!#riFKtVMlVX8LIa$BZ86=*?%DwtH2l28fbz=L4)#SS=2PQ z*^(HubbGMLh}6WSC!&tO_KDiORH0No1#)4p0sQN10#ivsRo*US=bDy)5L+zobiH$5+DevKA&M4l z+BD}HtK~DrJEoR&t7W&M@QC#$#%TMovogkN51f(<+aj2n5YgPAd#!*K)ji@QA_y@- z0^-~f-zivy^dc&@BAb2yzu!~XW>RFFqw@mX&pX1T6qvo_Inzm;BkP}%N4jSB` zAVhq4fNi8-8q2$1H|)f#)Lth_inZF)E|kt%Tw8?_;!ZGxJW0ofHS|Cj&Cb7meTg>s zdvY^Vq7G|UU<{Upg1+U*D*u2i&RCC9p!n*aa(f*i2_+2<2h$G?pk@f+XDSC%(BOCe zT4^G@7&^Q%>Wb``6L0N^OYBGX)Eduw0@$n*RFu!u70uJIt;T-Q)VI0zmxo$8 z>v2pq(uBSHqX(|b)9fn&Kg2YDxhEV4A2K&uZID4(kX-@;wF=U@q?3BA6XyiBv@cB7 z5K>lexT3NzR90jo5SvA8ab(9Ql!uL5Fgi4__N_BlkcYe#bKL)BJ8P)--yi;vneHP` zIDYiK0ReF_Jcpn#H1!xwKYKPHG;yZboDiE@E$JVMQku(qDS^kwbP0akoiK1s77Aje&%YT{}iPSQU(2}fN*MenY zgOQ4&0cqG^EfBjZQw#fLbYL{^`Pg8D5yC3vV6xB>ng%csr;;F|c6uyMqSR;#-#<+c zPC8g%X=h;&rkskyCs>j+#B7)Il8eGd*Z)qv)}WuS;F~h%57}I zI%L6J^oYB@;P+=Q6I`V)nSrb131OlqC-H%g0x}u{g%Dw3&?;Go=xWhu6Kh|3Lk298 zTWoyeBF7kPe*V75@=SdElurf4S~I5z(K(r6C>;zoZm7&gIHY6*#s2E`7PNuG)cO2i z^DMEZl|o)iKE(@sy8mY)j)NHzIbN7By+0q=*J#p>58@;|RwydIImxk}Tk-|Wo;U2M zSK0FYSM2c&v0_vj1SItSy=1v8YUNV8LR-h!r=}(WPcnumVGl|Qh9p;J56kqz zvuVA=UPp=fjJ)A&$wb14Ripr)rvH@r%lQcNsRA z*KG2ro9PW=P;EQ;f)eeBc>+l>REcsZ)X*rMrpEC<&zKQs$yJ{jJ4Un1Zlw5sma&zP zAq9JaGKi8z6yb&*uMY25P!&1Mi|MaCL78IlJWa%4N5L7?h&w0hqZB7Z7{albiZw}vjdC4jH>=?{C_fBerM zUH~@lL@BlVKMmqM8qqW{rWplV_S65L1l{s8K@Tu6G+}1=)@ayQKUc4f=tN1c&5eeS z<~`z{O(+>+2HznG!A4JXt9ET8H9w0{^gly#&h$IMt|6##7gr^V0RrQW9q_IV==Jyi_`Hhex) zx_j{ALY84P99K!&rcm++iF=8>(1>V_77z=jrPo>5PV|gl`!?=BXQQ`zHS4e}>~_;I zzd!zuFD+B&+f7?;oFoB0YeY%K)95b)=s8F`$S$M>P`@_)@5RB?&kJQ}n+?)nzCr zf&NTMB5Da3Kb1Ca2KQ0s!`RDz9tBxn+IL82a@pNSjSga`QWmqmL{H=UP2f8q?-&-6 zSCgguCrFSMe9V~U(Pr)R%iU$F-3vx(!mTC^c6Fd}aLL$Nl3{b#T3$i6p;n_pO z$!I33xfT1cEd8Q2O2tfu9$Z!Wt3`Ph zX=BR#Ee{W3k~N_&&s-z|xyU*;i>Xm&CNoTh+#|f(rbC}ozFO%uZV&9N{RKKPZ%*%S zEE}Jv7?V!_(&~sW0GS^*9<+~-=i5E?V9nuG#Nx6eu~rTtrNdTO8JR_1j=TZ(-^d{k~Qa78d7E!H5z6)HqaJ97$lqsPP0c+<;2p1*(;a~Tr zvEkUsL$G;$kLp-(TTPEkh}@qu7PflK(>*9WT6DZ+EY3M_!6jvlX*nSBN^hwc{T{=8 zvR{v(+QT0znjJiezLY{t*J4Ao@If;2SUaX(f`_LNC%^*xt*lzg;4<$PhfRmKYmIC` zJ};0>ybe1Q{i;R(uaW(nBly|&1n(wEo2`7)h?ZzHF-sw5jDTe7rG+sWwD ziUNm+%U{TfGpaA1jzd6Bbzcytm?CunMv{vbH6?tsjyXCB;WTsc@S}~?qLPyN(?weB z*@f*_2sJHKw0klllryGpJfOwh+<|;FIJ!9h@+JjN44t7JUIskv%tLi)6@)6c7I-$wcab@3}THk#O{(fka(vNmo(Sw&_S(m>Xzi*4O4sJx^1Rc6G^UG# zh(aq?tdYi}fgI)=79P>OB*jqGn2(gJL6^l#-ImQ!{H2%d7q96}*Og8Snz(4(EY62y z&ZB+AuE#-4ToYb$CR~%%%YDR4&wTfrFhhTGT$6EN1eb)@O6UK4NJVNloVE0Eu*62+ zeJ?Bmv#6lO>`2N2 z*ky|ey+?+`Jn+Osp5}^i`@0H- zfO@$BD*0QIg6D>8ApjLJhv!)Au`5vqbkKpo&RfBU<*sSi{@6~Ftumq!9X3k}K@F&} zD5Q!t!o!ht_Jx|v%$W?m4}4m+urjc~B@_x_)|Rm8H2;D}uC79!9kXx67soOrUO0qR zhqMqt+mToYmWbYFjvc+&$2iY!kgq~azMlcjbZ&i7G^ZTykiwL<6ssu;Wv3GGchZ{X zPj!c^&g6k=H>sTEyvOR<#qy@o7~A+OuVsgl=ASi>BSm3^GxA*l%Uz@R5A_C#nqmX7 z7aV?LIYy@hQLyWzc%))uP~1|cwc50{ob}Fpw{ud>m3lBiz)*Q=MIMc&_877S8J4cw zRiTiCXx=Yz)rJu5pV3XyO|p#uGFpfdRlj6v;yCoRmNkeLV`7&;>3qM7y?u6jl!ndo zah`qIU4qH1oIuuY+9LNm-6#oX!n|ITR4KQ`g@*YB{5nbO(zPDc^x&${`$=`cNN%4#Oipst zInQ)D>ZHh25d!rqMhK{pW)2&hnQjIG4y*E;u55(`PVA}gT_sDY5Cz`Ck60CA)=sg3 z%>RDEQOki$Cb2hF0Z%Dv>Ew)hj0L^#%{ds09uRw%=3!$Jp`T7;o|5h~WcaubK?IzP ztgItA`()HT5~cVT#!W!|^9Rhk>e4eizpt0EGwC@k;=@$7hotd_n3jeKHvl)Ds;o@p z|6OS3P$BbDx4%_D@)_n17M_~CYokm~yt<{o-}&*q@IMZzQyr^-eB9WNN9O*d87_Rg z9UG`H7tW+<&JIik>5T-yP95Jqo3zthJwH1A=@SmO(K$0Do&k*WBn0di%Foe|{s}>8 z@Ijr+z_ovlwW7S%yyfCW!$JPgk_b|4!n&%d3MD0J!OwMwW6zzkq*j%W)Zu7RP)C(8 zQx*i7h_&+|4S(_hOxThwO)i0<$4|?lH=~?U#Aa-uTe!|^2kJ(sf zVrQa&8;ZIx)uVQixkyp&>J!4x%rZm~1^xY7O*T zs%NaRdRw7`ctH#d7ktc_huK&g=q&Q-Li)1sfSjfms3g{uxuRM@IZL`eyh$u(^4FW% z#ep{|ub^qZw#wV3LNwXN`Een8;szQ_ca(bNE_#fuyB&~(%|?e&bVH_D&E(BT2iGg- zd(gY-GU?};ZUgzo1;<(u->EhfONvyW zpSa;dx~^1-BbKVKe%+weY%dYt#v%;HL0E+}oT145%2#O2mkp!$ieDqHjdBshI_Ywt z_-w-xRz?-hDKtc(MY)EI!l|8tDYC=b>AEK7BOI1CYeyPl?$u|*<c>)~8pqTL zL>l`Tg?K6a1c<8o^ojGXc1e~9ce(t1WmR{VQ?o+uXa0G1`5MQc*ZP=ijN5}Y@6)H7 zyb-1zJ=`K?w*Ry-nWR~qf%(P|pj0I)u_spmsf7l?>*9w*mTOyScb0u(t^U@Rmca)H z<1B1z?bQ%$r?|p-BSa}Z89CH1Di^RSU-?5@{O7f&Xm7SpX^BzglI~MH4MeK8%*~fo zCZFZ>MCi|Hg@VYK&S|aQN(&^7VsMKKzQv5&K$l5R&~phiSs+u7LJkN6_r)#XkeExp zPD{N+? z=euk9K?g9mg|$S_yO-<&l!wq92u`F3p5+DKB*^LCZhPeC)0T(aEM8Tro<57z@_)2_ z6=yZ8+p(4sp z1DUbo6J>x-e_uE}yPhY^(}K(!HwVukJPAvQHc`8{;x5rPeJ9Zq_#uk1%E+_{8WHfX zY1EHE6m*G59laqN2NXk&AG@BB9Glji0dP*3mG$w$Cc%!dyH7V;+du%)mZ^=9fdEwA z047K^ESsF))EJOb#Un4&)-R=(V_w4N_k+%pNLm42;);dVq06XBY{lnxV)VHDdh{TS z%@5U=I$jqF{R5t#RX}T;Ge72TO!*CZnQ(#U`VT+o)*Y92L{+15qdE*}{*Yx4rYp35 zisQI1p+Q(75JPjI`FL^FSYByt4NirVe~6|sRAc3D62Ow$w$_YIOm@H7&*7ks+tefq zc6EgTTY_Ld{CMqb0RjhIPr3fF2a3r*&6tbH^e|`Ak0o+1+ zqTygzzxwx7vlqh?IweaSZXpP@zjm>Zu46Y$k`4h=@%m?Vav#0fRkOEK+UtLM&+x=E zd-X#VNb{fOm6D{~6;m)(lxCA2XQ&f}vMyC9VXFDPQQ(RllWw0^Y5QYDEs{S>zH2iGTpok_brxaO5w*Wm%UsG_gJP5E2ZxumWA01?TuAt4 zu_mzX+taYyP1^iEH;2G`#{UbTFRN7DckIn^#erSM7 zuuyyw;2_A`V68);8B4$tC~E-87EMd0mT_jQq)Qb8p)WCC0ekwE@o5ULbg6COino3E zL*%j?MLIt@x)tL2E|h)k6){brT2Q50J*acT18u`m@0dTb3&p%1lzC1j>>}7O?t$h` zJX{bzoRXmqdxK(L5JS!_KWtnKp9AL)}+@_ z1~d|fFjHo0P1%Pr=rtbNEH&3dO>xss%yZ*=V(MEd+v8ib7II00c z5l@R7qGPFkr7KgS?>h0=UheYx+C+KO=CDuDD|Fm_rZfW~QfsEP zTCFBT*2?~PzRgfewfnu5pzQAO&*Yq`AU>9jP0f1W$o|5s0l-|Q%}y2)&AWRjqKH(3B&rDBRt+?w4;Zb_PZSQcrJ)QIZ#Bjw!RRXCbza#BG?9 zEQ{;I?Q3VQmJ44_V7?CTI`h}7zK@M0zmJt1OXiDSD@Td#oNoNz_mMJ#tylcX%kyQ9I>d{Th&Uuwld!s|;E1 zj${lX$v=%Xs!-ZCMZ+TYuFG`1O$p4xHNnQ+yAm61bfQcdF)007$wh^ zK#;Pxbt@Mc6<2?#qU8uBay;)(L9v0SON+3f{{~*T4*gq6IE{ZCIBED1UUP*fu3OR>RFoo z_7RQl46WoO?hs$ZIdVj%M>uJ!ahgzA+*B{^J4L)NW}8T6Ujc10%uz)=1GLe)D!|gM zcP+mrEp~Dm>tz6I1;4$EVnSPoYVWN^mV;u*Fv+rmF}YPBG>I|!cQjQOg2gPP!j563NZ+}G9Z?1*uqEU>5rkSrGXV~ z#w$|P!9dz#Xf9-~P0||Bn@ar}2-2Nh_gdc9RFc1ug$nAorY*krvolWTIqoB}+;Z9_ z7Yd42J>NVJ*(OHZJ#~TFE&Q}CE@RaIOpbVdZXN4cO zO^nRaG|qvbCtb8x{4$_cVUEx?!?np;GX$B>Zxn8hOC<&_Ysms$xG9wI=GwL%DPnY&E_EK=1Jz|l*h5R(g{$JfY7gkVh6 z*mRd*o&Z~A1fybrG(`n(oCaI{0`}jbf(|J)*1hG3#*g#u|0-+5#Bq9KzaPlWXYm3; zV&^d1PGjemEk8T!Rsj+tp*P9Fe=8YoFsbOM)O=&uQnuf{TNZEs%Oqm{mq}n| z4Jj*IdIG6DN_}o>S0>sjTUn4c{wH8dPKSaOE6yQ@!ElF|eHBr^Eh0qy!h(G-VqF*z zelCkP5ejsTP!;WcyIgE0Woqs09Kx)tI(7n-r{cjrO-tQxJowHP+O-1AkV%scX9JU! z!RV=1%QO)~y`01L41%skic=dZfBbp?N8Z@LOr;FmiBaDcyMx7x-yJ1Olg}0Afvx(+ zyeFm48@}2-`n-?u3QEtkW;D9!Ol-=o5X`vbK3KG9o+vUmwYuwTYJUwDGK!8z=lKDF z$1MxH=h2M#GH(4^MS|0);nMgF?GSEwg*YAXJ5flV)`KtUp3!BBUs2QBGrGrh^>GzQ z1wK}=lt?=^?AP`@;fx+ck1OLBHkxA}lK+o3zA`F~Xz4b%OOOPY;O>$^LI~~%AY}S?jE>)m^85o$9W$YuCOK$jdO| zbCwepIJG0mvnk)}QZeQ65}I0_PS% zi;dagFgi?i9c7COSsAqHZd8q5q)3i)aE@U1rBL%2E)rnDN7gA?QvWdncnp92m!8gP z?(M8X7p?dKD6S0Q7E(UMQg#v!5bLIDU=aQhCKJ(bmrRWQT|Ekul9C)3@XLAyFkgK4 zgDSdjZtum37ubgZ0c7O1`d&PJb`P;tsg;%F@_kSw0BA<_)i-y<268A{YOpbg?3xMUnv zXbQR?g%8Q891^Fh$*DRKf3Kar!@|;}`ZsZ&L!Mj9t%%4;-VcRV64%5a{}Xbh!g}%8 zTB5-rYCZ%bR$0KBZ6l^arQKeW2ibm9Yq>*^zu`8O>1VXQI{W>Y^HHzz@8C~%0NK8O zL;`|f*jogHKcHuqE#%UysD9v?7FZau!;*1favmIdoz9pS2Ed}hnHXK!3w`L$Uv0t z2J8#ag9ILRGRI>lM&-3>=es6VxHJohpR@Ca6$4D8UZKA8ea69~@Ro%BQYy1FILYBS z$HAuvkl{5H>iyXhp3fE2_H#m#-4?0J(o|wng>&3Qwip z(*n|B=E|wxlQ*log*GaQHST7qE=CtvO4G!)34m@V&)V~r55}g8&VBR)jtA#0C12*} z^#Fl{kTeKLY=sZ$15Q7M6qUun! zK1)QUT&86Yxsy5*?YQRh&>*zX=bU0FU>)RKE z2ywWLl@?eymhSgaxNFTSGW2`IvWLH2F$1LT|0KeApDI>NfK@loUYUc?>vRRyIMvf=`e)x{h%a!;-V#6&JJs@S(yjO=FevYR`mS3 zSW9QyVEnav(874dZ|U2+qjURRy=4HTtPzxC566^njOQBAMIlmpb`b%6Vb95!0*I7T zDs{S;^NL!C?g@ajWjb6Zkx?}or43nH-pL~Iq3L#n^4YF>rFPFe} z{UUKlurimDCxPqj5W3;g^7#?~8n8<}{J7$$zcSNH?NqM{#b8Td7sf2W#&U)<&;IZk z1F^kb!|{_|k>I$4`mG2$D1%XhJkYq62vWK5jPo|)UQR1_A)38|e%5WRNx_dgB(M<# zl@crKh@GVBb?WSjEix}MyU`4^r8B_?*1p5K-tyuY57zZe!?NmM8GDqYTyvg5o@_nZKAgDZ(Xx@F1$nXZ}ekg*v>3QzL*db`_TMf0MB_mxJ0ruQIA|SD$ekb!-UcI6O zhrdpGJ|8|mol7Z{!1W&>L|JzNw@_$~z;}{7d|KFru0i3S8z$Ksbgw^W;|^bij=u+V z&T%SlZ68Pts8AWEWxceuT}@H7az%pf86C&ms~BMA?WYNVphzl^^>HPk z#zS2Cq}=`{!Nl3z0WzIafrZ;6YH!rE+bWLjf1x4KV9=6-D8&>NfM=8>5bgimc!xnk zeZ6wFc-dU0J)KdaHwm(<-ftlsfA}Pw;kmO)GHDVuIT|xgo8f6!U3ccSR!}nGa^|?0 zQdGVq@`3F4OS9#JLmSP}J>Ql;V+f#H8)?*0}K7 z+t0Ll?4gN!)Ow^Y{#W-<(;S38$)oXJt9sS<-sg?+?7we39U5dYX zpPY~6vKzi`g&c>jS4Bxzi#~j|*!k;`Zu!u!RC*QZha0?Rl6jalvv^hDP{B0>J2A)w zRi-D{^Nx)kI8vJZ-c-Rl@c#YWB;xd(Lg zRfBJ~mX@}7j~UxWncogYv7mpcUO~%?&?8DLO360G*U(Zwk$$7XPD!F*B*i8snKp64 zXBsdGkyeVd${L;952gS8#jys2INufH7OSZjOSZz&AjKPFBue$c$|Nc0Ru?Df%~&mo zjXd`$9=4)(c%zZFsUq}*%)W_sPx-JEUB}<7K_vZU8jcDp9JG%WRUr_F$9cB+9uxct ziEzNlD%wukMu&eiI6Zok@*HR1*Cvx!`jkVWdODn&-F+{pTP(KEOiCDm_cFIz2CiZ7 zAY}&`H%d}YIh+W2%ouQR^qFoL;%4W#<;%sHou@wC_o6ww;^S;n;k~eaVVlUt4&Rv{ zV6-g-`B0;?+l@`9cDpNP{YUN?O+MS58v(zo$LJR!OJM@+JS5>g_+PyGq3%DxX_644 z=$m~l(c69NE>lQL5e;ctXA(bGAWlX1V1Igns5l@r>Y{un=_%|he7H{JTiN`ENm-au z-|7hf0Z|uVWB={WkD4o}7oJ|)iyupuyQN|=*xbx^RU@+7-_i8TJXn&_U2szpWs_kv ziYWQ5c^q@V0Jhx#F|lLNw;1eTp-l+#nIv?ki{3q#-2Zk*g&#|e11*B>W-`}O?m~%1 zfzn+xOgzX@K36WA3>NN+jT)iv>#D@{{S~dE_^Q@-(%whCVweS6-^BZ#7$%a7Kf=uV z8Q5M_+jEP$n8M}`VLj3m-z{v1EU;9eJRsa=jl)!V>S=~=a}3{JdAy>G<%IN2U|J|T z*e4|HrEp~cWF=5I#qnleG3$L_SzrsyxD@Qu2+1f4q%C)15M(%wC_u^5!p8rkSIDF) zJ;X?lFkbFfFsduI?w95{W=}8ky0r zve-CKY+1>XX3B-df3zVi1DKQ=2ks67ZqM|m?#fagn46jv#MTOGrNM5YqctWCsgSxMp9@V zL=Nc>J0MArDYm@H_t~+Qg7T6PL7sGMSlP72fv>fW+`H-Ii!SUN*<|Qhf-FX~H`a8_ zCz{Cuanl3%tKwa4ZpfYv(LZNyoO&r_h5ig>jEw3=FSnjMdQVhB|4NBQDPJ5@ZU+-B z>mZEHN^3Ctx1m2waD#!}yo#KEP|}c7&I3E0CjLNUfkf5X(cnd6B0l~frPj2JyVf`w1eoP{%HmE_We z8Js}~$XAAH*=4d=2R05IZjAgiXzssWw;#*+Vw5hK+xgUj#>TD;(SNshUzH?2h@ckL z=Z#HMsE@f_pesx1a4CL8(ack;)^y(Nod2>p%qXOiV3oFvI7?41HRdrlIYK3=cuO5p zMzH|i4U?yI5ycgIVHF(5xSBT?DXbLwRhQUW{;7229VaoWp>m`S zKIgZ7YGV(CJBa7s7Agv&n=EFKbmvCaaJ~FK2GQ~MfM1apuDrGpx;zxWlEa%7+@vimUj^^GBy z3?~&^T>LPf&HbYL{jiDTWXEm@vw)hh){lM_nb^QrI5F6=g1Aujs(nExScPUj`qzgq zc`!XKA=DjwScFy*7AJh%uy~{%#;(l;aabn$zR|gbr?BHMPW@&Zc+9)5uDyJ|`bP2| zwv!e!;zj?Wq^oM2ucvPC#jlJFi~Ixo{-)*-cn0wlY{x#DdxS4Ix_=tmRI29TOe|3uyNM6zTd=ciB> z<8Ydw%W{eY4fLpo)EORvb5HSA@1R+k}tw6;78ntCd{v&v#-QzutrYX2pMC zt8cJ{nE0zpJ6m5n-QGnve*FYx{-%}C%A{8!!Agn zSx0yYGBdgs@^OwgCmU8mN#SHlp&%$M_5P!LIwhZ9e@nC#4#=h%AWmNSL}mj!punTywrm2t%P z@jtHhUiH>-!)A-i6KQXR#E*q_-hok3(nX3#$cQ;uRVKdvk2C~uUr>3HMr?-9ElbME3$)GM`;*+*abnk(l{eb!ASPiq0Zvg9AEXz z+<%)K{^~R_*Npl5Il`5(`}l@kLlH#XuCuNNxEhH74AY@m+9^zaK(szpsa?9#VBSao zg7RLJ9ADd+5<8zX8{h;R75zsTXMGdYFzG2l4`EphTOLdnyIKC)!Qb`y(kZ`9gJa{c zJ*&r8h`vNF85)&rAS+GC-76TqN+`kt^tXNxRKeSJ_#8 z5C}YudGZnQnh?Wd!u}gcHEzD~c&&Mf`f<9)^J-F^h~x)vzI!o7KD$_*>6hD`2MLaA z#b{M|dl|HUKXIp@%2X4vK0ghFh8Oqnrv$#x3!e$`as8_b+Gw>6Rigt4G({7pyCvLu z^I!!i8YSAk<}`dKB;t2wd%F3LkcH%9VN~?fa=nI4=~7|6x=7nu6Y+$d>vCanc-Q$Q zijcvW^>}gbB1^v8W(|CX6T?jPxFBQ%C;E-o6a4RU;HgziCR|G+^N+&scB>vu4V=8^ z#262WD(S2r5*;7S+{9Y;4irVqIBy3dMeNNvaPiRKQ)0_5r4`5DbaV*62mgR>PFf-(Iflv>cpI!y#aX?kVVSPFdR&X_luT&DY}{F&%&+ zJa`S<9R*RTs^?{fU3jp>%TtelU*wcl5ItCncC-GWAK?~O$U0rvuIV;}D%2I48Ii0Tc zr=5Xf!|&I>_{;X{I!{{p4g#j&Ailp@dzbgews$j%Zy;_bq#vd#y=UZWas$^XjR4WNc%IinKH zBHWs;`>`S6aa6f5g9D7xYY*wn?-#e`Pv_J7k1PP4e0w46~@yc@-IU*psyv1bMg5$1J3-q0LLt@@*7w+08 z_EN{}5s{Jb06=^BNZOIQMxlY;w0pfcF$Q}-nPK6JFE%|v`qB;+?9unfJbz{dOMcee zH41#=4rm+pLv%Hp`j^gYk9`yffTW?!7K{9?yq7erRGRCo-{d@gFYvme8TYN6OhkqN zt@)Lnt8SGoOI+|k0y&`)iUQi$Q*_;(KAjr-juq9?gUdzn=&MYY=)B)e+^I6sqAQ$ z=+cS_(b?a>U)i@`srR>ixmM0$IHoZ~Dq+T*xDww{IYP3we*4Dk){7D?0{*=UwD`z* zY>H^@+p(Bbth}@MsvBkPG49M)ST|5k!8cFKT85Q4xL|RY=DOsF<>#o&7X(xb^Lwop)xWWNnyf@Xy60sxi+a{5gA&GJHc3#y(CcSp6b;9Zy%@852?OKcM0I;B> z<=~#y+Exl7c%oJwefYUE=C0Dr*l=a}(;Mj#Sf5J-lckm&wj8Rm5~OBB3thJx```!r z7jg1?*7Vv^du@0&?{*<(bXu`+w+fR>;0&`P9m^t>9YfD+S^T%OTBs$=xPm+wm#D8^ z?~IXBX?X!%6pDhA0B_!uB#&aTuA+VBlIwU=g07qp-y=B!P7@%8PCMRoI?ydSM! z!lDXq$C7e3bb5Y}CRW`O3ql2$)vWSxtbUb0z3SIyCvU1;c%P&FBWHUobnwTw5IS-I zHt%hj>xX8%$kEZ`&|CNHo2IR!rNmUvPeQ*^xq}=qWVPPxVo1ERdK>t9l)i@Cv)ev>+O_c+Q6c_ zXQlwjo%4ZdhlW}R?(bus2y6J>rODpnsUDn8DLsAr%b8>*jm@DtW}YdpfnvDn-R^>Z zwhYt+O73kjtnKwvHN-cSV8&%>tmoa&2}~*eTmJM&&aL~WDEA4TDk7m1TK+nGAjx@7PDq&o>v&k1zn zWHA{hWBqkzpPsBAo=CM+QBdEY{cjrKFjG^0o&POBkbD0>BM=&m6chg!F8}{a{g4JNQPQ(9CQ5cXiI`hNgAJVw0$ delta 23123 zcmZVlbx<5%_q~DQ?(P=c-QC^Yf;$8Y7NBt(+}+&*!QI_mLvVKw1h~AP-?!@Cs{2=; zI#p9O-7~xQ+G{;0qX&Gn9UMEXI(3+0_V=gmi5IV=z6svfxAk#*;DPEVOu{QUz~9sP z9H(;(8pO)Ww(I)${-EzHEJTP3x?9nC(%^!$CoW!rfe?eiiy?79!=NZa5SZdb@Z{M` zMh_}i%z3i^`ygU$lwquA+;ncR5i7!CHL}D7QSy<1DTk0yYryr_S5O6+M41M($aTHv zXb8NFUOca**2W%axOcY9N!e=^w!biTH7vUX;(l;R2_C-YN>Os41 zMXmijI6v)RN&R-(_!hFc^muY4MNlB}wdZBv{$g)zZ7EON(d*&x&qwFK;BSDxaPGgC zS(6vbW-s93?z@8L-D9axD`M!+4SYXx`(dH~r`cf2o`rZ%&14(N)+9_-HB2Ld z+Dk!caWYBZh)YCEKf3N^Ig#HHKc}<%ng8SJZiRU&lauMEFGyT6pA(BB3LGyA4(R~A zxDgLp&Uf~5nwwTVnig{x@DVr0f7>d2CB>F1|6BCpLHMoA+QrxSojuFg|NU~i+t1zH zUuZMc6xNYXQOHfru*t03H%V||>gwwmyit|`3Wj8KR5~~gWvLzmuTAMZrSifu{(c+z z%`~j!Y^0ThxD(f2U~RsU`%vG#D_icrYh5cI;r0hz5(m^TznhLmfYRzEgry-cx{p`( zzkz2~lRvj=mtuc`JQzi~vQs{P?ka|#2KN`ZGA9Gy*N60QFJAaFl)H@=g8!cW)~})E zw;57%arOUA4my3{XF@aLZR(-!?64Ds_Im<8h%HPtb8*s+5T7d2u?M911Q>B&72ciO zc(q!1CH?Fd?(PbA0fs+H#Ca)^Dp*G&12+TOSJ$kCTBM3&4_y#00d zQM-9@y?=MTIYrj*5D07drJHO!|8H`Hq~~lM>+!}T8~5%MUS-;@>A-`9aqawd?C%P- zQ5{7Q?pGO_CnNpfbnO8fcU*xQ2}!443;ayR$+&rO3yqa4N`Q#))smO3^TEAuBY^AJ z(|$RPnxzI=osyjR_A|EEok%mik zh+7phhFb#JiN~bI+t>1E=CJs$2y_h-Emwe%6&$8*|Gm>hlOLV+(No!c7}^{P*Rye- z99HX}hX?E18{o*tXW-ku(jx>_qNCJGDna5)B!yc`IhQyA8sqtD)RpmP4f#xJSVz06 zte-7aTL2NIAw&j4uTB|1UmW4mA|hh*!cYaFL_R3|iSua7iHR9^p-XNTpJB5P_5NaD zrsdzQxAt_E)+}t^C8UxbdZ%P6iT@xf&t6u-z>vFkHjr}0btuF79y!0v@o=|I*n9_F z>oe`dFPDegLNho+Qh_q@9B9OInLn2@g7Q^3FD`s>g}WH#cx5)ePMXZ z_v7|o=c9A`UY`&V-@n=Cg}dGA%f08V%^P2!UAGB?r(@@iAq@ffr^665Jo3fW3qs)A$^YiD(?pe}}p55y2>@Fej zpM*bKpT5OC`I>+Ev+64(fSh)CrWz2=f7@rp1trSJhZy^#my# z$_*bt3%N}V=G;03rpv@)lwG%%bVF&r(>3EP`2qxork5jy!6N|-R!fvyFr1q_SK>Tg zx+C)_yNxK_=gmmrW1Srz2K*B#A1bBf4Y;*}s@c4-Q9o-N&I4 zlbaDM2!uo!%7vBV^y>ACXpmc9WZp-BO&qus*XKe{BYXpP=}e z^avoq$t+AH-MJDv&%Pb(y#pkqmu~=ZfvMjQfsjaLgo>_)?IIL9d3Q#xa0L@%sG#h` z>9FWXzk$JpPM1{!MZu{b;rN(YW@er?&;8Dp2Snx+NF^~_d8 z=h!GX`e*EWCM+&>v(OW~=`M9QwL2p%-&ErE0$pT&Ts;Y_3mcB?qmUOZ4$2%s(UQ`# z#$r)Q?r%rCB)Z{4PSDuD;E=N?e;MqSjzF@sm@HEu{F#{v+0*iE>yr4bsK|2R1;+El z9XZJa76&L_70@KI(ljK+X0X~MX51}BMHM*^(?~Y0N7NO}%XNUHC4qW+?pTt}UDx~mnUSUAGmb&MAavEV1M0AI;_29H5 zYc1g;Co(i+5%zc+!nD;@yv}EAiqsVewyGqx!hrw6=wvW~94<|D${aP#)769C(a_2V zW6!%Icde#>COngSK>|iugR3vMoKl;I-5=p&QRkh~2dcGtLl{z4yh4n%$jqPv!GQcZ zw&Co_Dr2?xD2>Ig`roWu3v+GLfytg3P#0LkUf)p>_nlICFmrKbpO}n_YXt+w?=zHd<8X zD(kDs0uEvK0Gl64YJU$1F4KPgaTb45-_jY8KTL4TknT3gXO$WD66b5S7hOdj`)QUK z@67+Yo~}%Z$i(b#=;Dx*+rh^tAYKU@Mg>UyRvNpi9puW)*58n2JKE-bJmEH%&6UyZ zWkVCIkb6? zhoF$!tU&T-X+hQw4UxG0{XKaF>)q?Ct1Rtk@M~CI+lcZ98W9F*Il0JVUB}P8zwNDQ zKOjc=Io|3Cy&T%PAC?rRAM*C>3^@VFq5FYcmGB;t*pD&?2^I#ztF!~K%;x=;8{f`o z#Y~CYt1%vbTC2O$3Lz#O&zY4p6^|zR`yG?39KL6nhR5h5QmZ{sA@PddRK4~^Yj*U< z>o12+K5dnU*5L&|!?$M_B-}>4*!vsxttFhm$c~ zRV?I-pcJp}^P&*tTpFiq1a%ZtW6Wk<(hHc%6mrv!I!7P*b$P&vBqF8(QUE75zkKM4 zP)ZCtkFURzu1vlsYoZHOy`$ zzX8YqPv^V3&UwXNWt5|{G?%J59V;af^J~N1aF)RJ$lhEUZqL1ck-Lpmw80eK3PHP3Uj5tL05`jfnlHeY{x1B@=R2PO z!xQxw1M|A>Gq3k#n}~^tpsfpJ((Ocl^{q7qKc|4wzc#%0c7CId_xhcgAqTeF{3#_C zQiZLyC$c#?T&-7O!3iAQ||3Xj?nO;9e7}-y)dh0^C^-dF=}uDX)9Qf8#!6LTdDaYB;%a z;qRc$^vVBQs5{9(>a2Lj3|W$J?Q$bX`lh?=o- ziPSSd7PpE=rr~Byukf3Wl73eeBo5C&)WFbzTv2@<^IN9XAE^OQ90pVNGzqc|$zh5D zWca`d|G+OfBz2Z|vIu8U1Xco*_ix!;_;^06lsFZn;njbJ!`S3veuDMv-o5^~eE-Tx-<@=WgbVzp>z;%gFxD-$lZut8NVcqWaF7}10f)+@z6p@4 zV1j9T$?AbAxr1R8HR?i->oHd>2?#Dy?3pqMZepb93Z2KvJCw}liMdpX&V@^?z|$_X zHTvbJFI}IuRxR6kDMlz&?Ywz+JtF-eTN&`H+oFJhKm=DLgrK*9iG7s~#vR4z46T5a zBGvfS*4OX6Pfq_un^H#$BElGRIs)h&KTFquZRY%*KOMH@QHHXZe!};itPI7XEf+VD zL$&jnaiO^w7&-81MHZ*cat>mHMPh?b^-7inDQ(UygoKzPw(ZD~F31O$geXY&*=4`* z>m$RANU~JB&e>z(O>8c9OBBVS{DzHnRvzDnfwB7NElQ&yL^C$MWZ*A9mg{EmSfn@N5N zA!#8?^+MHaMj2>lkB@Qkfp9>X*)3+0ktF{kF072Ng7avmv3I~}Ufw=yok|fabvS-~ z5KAGwTms6G4R)V6MU7pGn-IrJaTAIm5rN?_J~A-b@`@e2cq^MfE?S_pP>c2zQ>9 z-7Wd9HU5*Uw{y`1Xxc?9N>Mbk3PQt&{ShAQ{k1-iH{!0_Y)i)hDfH^`F%EU>qo(6+ z=u^SKVI<5EP>Camu_8>W!_rea4+iN^HTF}-BSrcJOB!0po>Ng`B$KmAn3>CR0#|q# z_Jo_{vc;MlmaMRQzII6k2+1m~vP3e*?4jik_!zG+k(A#6jz=tvJ8tu^LF)MDKBBjS z>(-Ywc(Phcr`K&(hLzmluNDv^XbaSXiGq?yYFp|6*9iMz5+m}qgH01aDSCm+lOZf(o-u+9IdmpA{zWr4 z9#EfVR+@_^akqnkB&cFL~b^rcxCs8+Q#{=S_ujo}&JwgF7y)+U@dpK75H>H&6k)VUsB>8qPLF9k8 z^1%TErXce5h~5i4&SQB6tqceZQcV(%tE|~q zlvkHe)MOg>OH|^B_zz}n&+KUoFI-6hjf_!13#*ovX1qXCMZAJq!V9w=o5#>p8vF;5 zIuN<|m7P+NbDYpV2Qk;t)PYJC)trH<$c&p`RI|9dR5uVMEEt2*nn zxXJAq&hw^m`r}Eu{%`qQ8|ZSYNfmPv5D!Q^ugMdzJW0E%?UkC&CZ%wtMt+#ke8E#e zuj{47lP4-hXOKD_#$)}R^J~};r;FC!A?JyxO;WM`m)3hY_ysyRMd;unaEwPeWf}8I z%~+;|lUilzyb*HXo)LO*I!LXjdx?YSWB-2hzNbc0=-Q;jwJBL80hO!&m1~zaf(=jH zxk4!!rmnuwFM`7%+a)2S`sX?i9;0qppK-s5ngyP32CLV71<^n4nh|B1;FA-R8(Trr zY}~J9X`q$gWcyzS`gXQnfwyUsU!`5d{qnIz8ev{=;|M0acC(yfG^l+TedTvymeuEk z+PV4BDP4o(m_T^Qoe+~C+lP)mUqSp*&;qHU^6*!fgQJsc4IcwW zCyG=`me{vnLh`adSQr6?4W#oh6pQZZu@D?%iCj5 zC3uO{_w~Tt_5DF5_`}b`?;uC#dq@jue_uz}fb!nj3$-VlOG@wbk09^@vOqosQ3d*d z5xnwVSzBz5-A7~>Al#q}QU53XBnb9TC;2+hbr-G7-ULYMZ%G#1lN~Q|X@eSHM;Eo= zs)2NU+MO$>uHwQU>K+%i+-mVZOMw|xh$Bg2YN12cM^uxgJp*eFK7YNP)elT31;qEB zV1{R0#JRLV@wcHXg&s-u!h~H+y&qu!eZKM$*N&qQkCo$uVhcK-bHuG2C zTEPk0K-ZTIGiP>T19BIgiP@_^eyp_vQpY}z9&ix}jl*cu?9D0uhg|kn4IY%=!asXL8tceqM9Te1~eh9+C-U+sySTsK628u zJYt#N89OOEIYLCsJYMX$yv)h@oDmFr(Df~$D^=WX+uGqts0El#gM0KDNLDw9ThN~4 zV<991n1(;#Lm>_>7Lh`g@~>y#s5&`T^vc3V!hs7YEW(W6!d?!vrCb@+5FZ0eBU749 zwf}-&$J)L!^)#`sTZmm5i+C7o-WCH{F}RizE)1dkXf06-dVIoFjZ{C0f489WqqVi$ z`gsp#$<4$UlI4u_me<;$nxJU@AgYs@v-P6^z!S?=%Lbqt9kXXp?QB{qzf=qRN$?(V z%EIeHLon(irs~7Qx}w)%vMa^vZyi}T_Ax#AtyKHpxdELO5+bB+ZDFz?DVoCb--s_| z-nz`=!4s!z0&d6AsKoi3N_4xHV*xtfXg!1ELA{dH0eWYBxBK2-+$P=ZvRQ4Abvwv` zZCynSer1M(sj%l<#>UoG3ANWnHy#>m8QF)~IwB7qs&BaZdY-O67p)Sb0bHA9V`B@d zF5}{|d!f({)&BR~ghIVn7yc{Gly9sB%px>|XG-NU3r!1O<8&CVBh}wz>-j4XR|Y+F zst{YiaS#cqKv)s3@I4_%V>@^-sjPMY>r?V(L;tlVm^~HN()@`|Lls^ zu!fBIBbZfw1L7=j-9lsjUinxyHr=v>WLZK`0;Q|QjY4+*irYvDHNHt6o{3zW5Li`H z43-;8>>gAddFj|m2s9BSdU9s3B+TkEm;uwdXH;bzfkP99i;N~yd!-KW@>}G8{z2b; zjDDRaMGK=Pi*8gd{t#W_o-&Pfct&HcwC&pCQV@3Iu{o1&98wDcxV#v9vDaFV(i>8g zs#r}?3o@lyk%^Iy}7e}c@?+NKX`m)rh1*F8-W{VV!jqOh50 z-e@;+x!8i0+;#twnll725Y~RvvmW$lLU9i!T!s&6i6>;AlW>w11kuv#R6xL(gvhA| zxz0kc4a1{OABZ3S*Y%hG>-zgCZKj7|OszW}i=sVEy6*q=S;mBdNsQM%j1}T8LX+t4 z)6@>+=KICr(dcSX5h%^;FO-pN{OU`NZ~=Hb%M8Wwz(iH##yH*vX}-_p(%^DgJiGa1 zv`ZS^-UY`>h9LDV@8o{H=f2DDz|9BnchatJUodfp(t=N}?n-siK3^5z5(N(nV?yIT z`~L?d%y1G84=Y^{<0QT5)e_;>=-`pr^V#YC`wJa>9%#+Xupl?fSU8o@3pm+i=KULf zmil}PxC`oaxBni#>S?2)J1Y=w|Al^fGcwZ7@X+#?WHLW&=3;ymfr@ZFeFn&zCMi{U zP;uE3?7#Jf?p&fECHc4Lt<*T3k0mK=Hhan$!F_D2Qg4Og=P<0B&jp-D%3cL+DRSW#+=dOIK(YRjko7E?ItlcxxTLcFV{6n z=vKCIhqywgqr)ibA!}Wdq0KNw$=i~+4n=I4R!ooQyg{kvqzJr*<$-SY?Qw4`2t@bh zOddQ3%-DJ9U~R6$ExmUD*ZUC>Pyb0!AY=c9ufZmNHOt9Q+N48|>?KCeX=&1098d$`!h9h&&&OBF!&8zm`!Smg z9h@WEt)}V3hR&4X)rn0;q$Vak7I}!Tv{7(kS5PRyT4P$bv5P8>B}DtaT`ZnT4<6!@ z5+6nxua+DF&CjAC5LcI?%eUxwa+h zba(BGtK)aT6|6~7 zFGCqFiWVmgdsMz9?WFq{PpDokIk*Itj8#@ZyTBmt-V}3C$Bwa+&%v{)?xah@=g|QY z`80Gg2(6r2*_@tTfU6$iFzc1uE@f#!r-fYms0iRy1zjP9&P;`Nm*9&hq-One?ih~x z&_ZFF7Ae}CCTxBXpNXXOZ0R~Rl6)wh#6GvikYh>Ig{Mbe27U7%;weowjUgOO7yC-# z%}t%j5|Yp7n!)rK%v3Aj!_$j*maX?&c$ij=lwpDZM5h*1f_^ez$H~biUBU`Q+$iTh zXaJ=0=@03RgtX@9h!tyoBZW>1uQ_#b|NLj8iO{ZIC>f=X_70$N_l%K8s@wS zOA#~k(Cb58Ph+c{TayvlV>I57XmzlrUoV%s(lHez&U;)+qAMh!#KB`9A5f;9ft^cd zsS6hB{XI5@mla2iRbD}v3wQ3Ontek6j_Ah+5oiAwv&q!yjVz*o)0FkW1YtW@VXu|Q6UxHQ`xF$xcPGw#fD?o*P4C~wE{^{)EQp!* zLk~Dn^qoEdaWOpmfKW8`ELk9>$w^&Rz7tE{0_=h?z%>=T@d|7lMC9FV@h>~YdMPqr zkB`7NvozifJ8~LkZzkA`v(oVEk@HL1Q@9MVm@TnJM`J)0TT_IzYPL?+y<~8Wwo8s0 zr_BVMDffWQ9}~@ep-F$Uve6&ue!ib7;ItAkYUB3hQ4Dr?GcHyEJ z42SUwrqq$Nffk|3VQ>a1j_SO4R%qi*oUgEn-f0a_UUWYiC*J)u&;2m`>|%&+b_GA)!5t1S92Q~+|8Udm7R+fU<@+pNXw}jr*mFLs%S=6 zPv?zocZvr(78pmL7vTY1WoouVLXzZ=tU>gq+h}BXaq<1@YPP?O6;}j1nNdUvaW!c8 zOsgKuPABiHag!d*TniGz*Xt!H*lvfaR9mRt1dPii)$9@qkc!3d@P0-uN$-P687+O4 z4RBRq@4s`ESdw`-!LzuRH;GT5<;2q44GWRX*X2Knk3edeL~a3+375y#*X*2htxK() z(%W{Xap~Kpvqw|pYwMCNV^=X_Tc>BID1r=_C{`7a#pI%@$y+xFf-X|g8p2GL(IyFk z^dzoL(M;Bes52 zcWsa(i!u2WkmCW!m~8ye339NC9rv2i980LA@s0AliCj>x9*xf-2mi8+b7_R>T;G*u zrZK-Igl+$NakUl?;;P}=6t9wi&E=n=rl9=d1`!=s3LycFfQXpOj!RL?X z!p87HH$BbrQt(-v<8vA(7u_5T75 zW$EIR0_4#ghmXi^GW0*E9x-hukQ%)mo8cLm5w$&AcspW0zon>&(ak7h~K}eYdwL)N&ljpYpu=JPw#u0>}1ixb_8NP zg+GEc0}Q8$T|}QSo;%G>!hZu-wPW@aD(tfp)otznb1F-l`RvhHcc5t#^yP%=r>VnA zGpY_8{rwHgn0Nrkl}X8uw$H3o&(C}RkbvT>;+QhB?iFg06Lmt$+gVTOhc!yhO#!54 z2i01NanD>JoszqvX5P-?Q((~0ve^E(*4R#nR2NM2Z(cV;MGOMvl~k#}ec~XHfMUO* zkf)ogM~LPVib8>ExW;WWb#HqTrs?1U(!J;x?cUBolN^9rG)0VYN|}c3#LRWeKj6>C zTkVeQi>bJgzcTl7mMUSHby*JyVBr3hiNn1vx~F!ob{W-4f?k^&4Ij;G$TyQjGRPFZ zS5lmvj_6kH+B$S`dcN!`y=S+LylEO?nwpI3_v>|ugqEgZ3&oL5GPRlW3%s!(3g%{UvREm_Jb7xsT@s$nrO z9TK!2D5V=z#|Gu#Z4q{^_!odJz&nxp3yXu~5hR7C>1{$H$GV7s7F$kMv@dyQ_+Ice zlU^J?nM-Rqo_-ZaDdm1!a#^ziv&2Fv#lZLYe`1n9L8HRHVkcIgnBaP`-z<@V3h5V_ zUrL*qx*JkIdD9q-RE&vyB_rdq02+}M+Bp^6IHS#P0klsb!kxm>4`9nh(gwV#^m5+! z<(pqn??k#Up{V{`TzEn!aVcJ)DbrgqCb2<0kT=0AAm|ss5y{DY3dB)E$U;wEx(1T( zDlY72YCVS8jNJ|-G*f>B2F6B|K0oDpkv^9!7OzZ|eNPl^j)Fu!PB%uy#g}_dtX#>c z`5TPpNu=9vgIIeP7)HIK8>&ZEJVBGEqC~8uky1`b&6Q8!>@0<{_SFmV#t=nBCw*<8HkNtsx*FL=uMz^aWi+0Wn zExW?VbpmOkA}cfr^)Dowfr_H?(UyAM?@es4^CL{OO(YET9E=V-< z>aryeDkeyh-Bk7QKBjDKA%~k zHa|Z^L?+Z_$-JYLLHn zF6vvOPdKHzl{ZXwhHg4b`ivq6mu|UH1;F71m}2%K?13Xz%c}OfTK``Wv;0@Y$Uw>e zRdI`S@>4BSI^(24{g9~@=*}V11$A~lG9HGeu)0eq85QaCaX5uW1 z8HrRzaRe;+vNt*8GQi3W=R#)W|$OgP(av)ADpx;WQnQe?JIC#dubX*7$SL zVKiIt+t{7?IR3x=HXwgsfUMtDEZlH*rxw9&09So>XAbjcTl&C2+k%SrXn}I!F$%UT zEyx(lwLD2O4Uz9&(NL3yyO(;h2dl=e;(K5~%;lG`l)Xi#|;Jf(*GyoSL{zq5{=yswIGv)Ng z=Dk7q;h#!H->1_^5KR-j^RK1v#1;JS_aEE;X0GGRpWZJsXY6s<*Wk}NW&iR8G!2fw^NUh}>kQ|1~DtfGjS za!O~}5I?I%as%UJz5I?~E*PEglL7=BSdSMXdBQ$q69%a>VunCuWQw!h^bE(no~&L zbyhBgxG4uG2WXM?7B*D`a!+B5OZ6(t(wTAKizqF*DFF7%1$l%C%ZrV^CzVT!8s|Y8 z+X;$(-xe6O4kR__t>q1{BHi`$?wBp7rr^iyhnr>e7VJle#bK%|?r+d!OtOEWQD8Ef zm~)lt={0B>ueVrS8%S1lEEY)rsiPydFN056su;wBgHAORw3$IurD;T4tjb?%2&z$I z#BhKNF##|MS<$HkUBP`*V0Jzss@77D4lk#=nud#d{JMkVlHi)7Fx`GduI%~I5#GP* z(Ly#PB@GxLSx1-+8W;oSW>|-&<^;28@~JtfX7Vsj%@F8p!;*YX7t@5k<}Yd%9b3 z0yh|;IYZ1HbT&pbw4Y<_stljFH73Pf0J&>!7)3uihOvQYw5yrWK*a2)K29cV{@V5f zaB^Dp`Fo~zyW-?yGaZt&9sr*U){qdmqaCoqPETKq!soueelq8e>-|(FgObzC>i&|} zlM?L8-jiMO@lSiY8|A$&dF}6lNv`3fl*>N@s%Oree=gS*>8PcVKTAmuOtk0z^xd{~ zzNghhcK<(4==wY=uL^UsD%ftsaUF0cj@WsJH4{G&ttJgzZ7=eK6wBC2F9lDMjsICE zTL3o)yyd%T&!?Y&kcO&p57HuZTpB0Y9mRoI1kEouH9wM&Q_+@%=F0fup+N zbKtM$U|H=!i+Lt}ylGJN4Xt3j8}`2*wlepEJg=WFxMi}4tBiY?$Visgr~t?*@?Hqw zGp-F_;~{9k(xBX^*?&yhr)HVPT`b6aF3(+_R`)KF*Ayu+I|~&lf5=m=5Kp&3sMU%t zeut)rk*yVN+`x`lCnlN3j&Ox8Gz_HGA@kuivP)+`J3x^1ToN&5seqW{YM2R)$Q+b; zXs_Gzk^D{>;88}Buw0D$07RUQ7VY~;`Xx;Zj?vFQAUS2ZAIfPb)M8gzoJ?-aXAsL- zI;j#g)WB?0Cl}Rg38x>6a`Xqt)%HOVVy@1oOgF(QTOs-a#NwpC?lE~e`i@xNXP}Mg(wmWJ!2~|A=B6-gfP%vcrYj?N<u`i?P>)t(*1;w7YwUpv)7PEdy)yas6CQcwBAo5MGxgcYsEN?iJ76BnQvB ztb5H*D}~QP)DWHH2MkQ3PpqyfoD&_Ww|OTt7gS5+$-g`Fmhg-szW>HmZnoK!NsFtT!6*W)hSLn3{UVyvILTK zh^V-4NX&;mIcZ#<+|@g@W1PXEu2Yw9@&Mad+B60PT5}IsF6LeLKR2Vf=Jhf*OwJi~qA1RSaq?>e` zeoH;vY+O#Y|aX84}k2Ky1^szs<+gzCE+o-KJ47#HEATjB_DA|5`iCELg& zg1}uotttZoAesV7v3DFzlCz24ZGh<@>_X=%H{-K3Atp9i-3D+ro$c?_FZDUc*%h%3 z7iVGSJkqZ(f3wX0Ze{Kr|arBc7HJN_;4Km_Ue(@n5g{w-nbxo3^2W9&&jNV~)c zC+u3A^nF3DbgH$U5GOPl7;cHf)6gW zakn?fIClKEYiu3DMOQRpy%y2YN@$@Q=L57x19%eHNwLHNEoLtWk6jbRTGH|-v|xTC zwW9_)1uZYZpD>JGfQgIz$3@N}2Q4==Nq`XuvcYMs;uk_5YC(mfL)eu_WhO2H>N((z zo?!Zi*gM_k6~}Xzv{d*Cn5pmDla+j%Q-#+UY-tdE2YE0DXALU=!DOi|M65E*!rG~2 z+w+1a-3~+RTUfJ^=o!0=eTx1(dT*_{OSlOR-?waXsA4oMyjYTC53C@wLAJ{DLcOFO zPgaH~qOwN}1IsPPDVn&<0J9muJ+$Q(DUpIrU7^p0AdnBg7bC8hY`11NR^f;p>k%2j za*O7I4(rfw9w{_tcfcSB(r8J_(ahs@n{DE9<;J|m@(GdAho*y+ZFk-aoc|eYQXVwBIAq8j_q4B#ltOumzNPNVO}P zq=|Gp=kl-fccWjgjtQp)rg~0s=l(y24j4N=0%q5?-dVm4+=M^eJHn4MFf+ z>>?|KM_dggif+`_hY)0FmO)D!jlH}s)fD=2I8+z@awsn!EQBg5xwOJK$7~-7y%RHt zV=q}T^>S!*CAcKjE^|M1CVSw})a{*aB^gqS9knj=Ofa!`^kmym3Uf4XwmU`O&bwO*b3c-BD=|V;W6;h4aj%jM_Pby z9THNT&wRoqM6~1Ot>)$bovA4J?@UFMKB06?Xw&6|9yzJTJ4d3GBT*U_boveU)!&z3+b6be; zEfSI+8)Iv^>!r=8Fm6qy{O^^1O{Hodm0aW&jg4EPtV@w^vP3q;=&ka%(Ru_$ku&=2 z6tcmc@KBt?w)(TZ8jv-xp~Moa^cnAtr7D2PjlJKppt%;e{uJ0<{UYvoaR#|`^Meq} z?thc5ssWDO1T^{QWP3SUm&5unZ@J13((q^?3sH`@YUCoL+XkIQZyN4eR4HU%`=|2KxkV-sv$)J(>Pj=a zBr0U@W6-n>jt!9gzoloYfIl6TGCfY7LMEpnJMasGT2LM%I5D7JMH5Z~KCC!ai8z`Q z69tyWvn*sRlJbq#v=B~w-E})*gtNOs1vLapRUY3oV&pa#xZQ0F8N*>9wnY0Ob(g!F`NXVpY*oJJI8AjSZ<&6=#l;e z(0C_z8~tPkg7zt>{elOW670YB>+)zR+GVYiEAE3cg!tNF_R%p829-sq!p-7rV6m_( z`7xg_2`B=Ln!|%z=4KgeZ02TeivLVdP(L2u^Q&a*YukwXrOzlAr1k5@DNLxri|0;K z;sR}Ak$-JjIh+%6=hj8X_>BxFQJ9+#4A{z+=G;Z|ox0{kColzjGAKS4=w#g?2 zN~_B1!F7+Pw8ZjfP$@KXwbf2n#9pK}RIx3%xf&_gPPlroP}Vp-?=GCKC%zM>HDD@x z5f#GN@q$Z3fnTsf(^DMb*%AeF0dVRtYeeozNrDDL5R`CF7q2 zc_@HuN94Kw{|!lM&%XN!q$QTL*YfVNx()g}4>pB4EgP@eU4$CTvt*Z3D6pKj-iudJ z4KjAcm*D=o7f&_z!jsfZH6EQu^o;C9{u*&FF16qAD87exJT{2SY8nx8+$&2Lpmrjr zMqD}dhF2b?Xwx9Ei&*pvcNb9>fGI*(%de|!KB?mPJkf~8eHxKH8@TP8*lJA^8BPmg zxxanBMzg#LW$d1Hr+U9`{rq>W9}+kfE|Ej2Il4kgHV}K1 z?y{{Q)dKIF5XmgXv@VZT*9z|(AL)Oj;zr?E+U%JGuUCs_ z-H^#sv2utu&);Xi{r@o3)6cWuoxkOilz;w#nt((fr)gKgJQ_g>d_;hJSjFGb#LBG! zx8`~D2FV2tovc^bi0In5xP-%g)tLF6olQ)>1UZlh6n;|F7lP5>X{^cAy_JM?@wr%o z=8-q{y<5}5bd^GBYC&V(ui*aQo;t$5oP@)F|I4?g`%T<2yENzBr&HhMLvu1t2N5Ma zM>EY}#!g28UL>$m?t}6giXjMkm`?flfP%7Y7pi}Hhc*{mQ$V26e*_6<2D$Qk^D_@* zrTgObA{FsPB*MAyL;QkAdVdue7N^dB83u^QkwD~XAq7AH}n&}DpR+N1NrS;e?8vgq4) zcxIXk`!+L^%SYHPH0F&pw5}9~<*QpsslsFi(7i{51X!wp^<~RUR^#x4q#VxfZF;C% zUl+cBq%7(q7%7)A6C%hpSd1tO8@By{Ckep|5=)wNT2lfN#_44w5@Jb0PrBL@9?0H8!2L8$)$HqtuPZQHv@{uJHao@ zp$o*Mq0}kW2SUj59Voa)N6S@b;o0|aTxI4)AfV@Y4T*ihm2`Jw1xroks=+CthbJ-5 z+NL(ySUSFi2`p!vWC&PYKF_!Aof=ui!E35n#R*x)J;1mumrS)c=A6yvMpY|xY_vCC zcoa7wnGrYlp4HL{(}7WFH_Q>Ms+R*r?!j!ztGVo)=*`qy12Za!80Hc`Ro9DSs!qk0US zHCG{WIr>SpngNH7%fw=Y`*L{=cA14*w?Xvr&mTMAa%69pI2Z2$MQvn^>%}17bbF>0 z)qqB~3MJ~GLrJmTKs;$QomC65?k%xx?X&siq&S}w8{K=^LLAJL9oD{`Z$s6Yg9d@& zTJYnsX!V~cProE-Jo~s*JZ1bMby=@|wlQ>6@(TDDdUonZbFi=llp?C2>^~Ga?|qcRz&;dZM#8KHJUYMn$%SKqBRyuE&VkXSnL68s(D;6$y|A z3OVC3Db}@E*&xPKh-9y6KNmIeh!PwBSe0l6| zr4+r{Wbz$+3JQ6um?4*KCEASJfZNuv1rBvA9*p14JEMj z@Kl_FIng_F5$}WPlrDQsyVC5S4mkc0-!BEAi=tnCtnsX?`=yi}CxG*)pH+bVrl^4w z_MHjc{F;pBaj@j-^HnBuGeV4)f0N3Ty&8ov2+4{^<cIQbWZj5Q_;(Wc&y9Y%I~)i1;PP{RO_`aSMf*&+J8Rg# z-dSeC%g(Q!D;{O%Dd0!M1Qm5VUMBoq^>?9z&5-pvY37$F_`;820$k_`Gy{JG2(On+{;46>m zKU&jHpTU5m7~JFA-FAu47PWO@HtT2dMnj2p<@@6XTnC79@7>~fFMi_|{^U)<=Vh9- zL0r3^LEnE4yFVg8$G-tBZA_lCw&N|6rYx+_wrb1nla`mKx5=_wYG8BQYhCB5=n~az zA-l=S$?9~lIjpN06EQ}z*Ee&2rQvS=T!#I%;qb}u(N*|(_b;IKqC_|?3GQJI;>vn8 zF>b=a5ek8iG?ODzmQn#i3gU|D+I{kYsWD%M2hL_Igh?1%YxVEzrB*sx(#J`szaxI9 z0kPrqO5K8c*lxkQHfM>(ssPZe3>QUmW?H&3M}&e0voe69q^7$hOsMW1pl*6xBrA%k z_Yfq9DEvM`nm3Y~QJDXc^KNG+(#r$Kx=DOI9>T>2_z2d9l;bCk5)&2`HGX0|^%4e61M`34Q4@SxD0zIy17r#>!_kF`++g4DtYB#iW%yMy1We*~P_bI^+INN_G@=BVD(=NLG0KODFOu9~ zlJHbT+GeD7e5+9`-^N|^lcwD`9KFD5RFp~XzzL!N7j|x|poo{3Sl2O8e&p$^R1z6| zZQUn)jd{!B#+m)irDOb(TR0LcE|5L&e)Krcd_UQB8Cmf+{yCcz$=50@y2b}r3qKOL~;`eyDNRkE?2bDD{j4AUxB$p|n@JwAz}##G#?848XZ5EzLj`2P%V; zA?;MfpnVLN`SXy>GA2_dxpDJj$OlhpO3p}Lo%olPPQhH}Z=mw6*uS@SwtdQEZgBeetg7i%4N86v!}2ysL`Guv?R=cPdt);wkV~74x-xxN!|#mkUPYM;rGLi!EMH}$9*j&4po48ra_>Cp4`auImqn zJ_C=QEz*&rhKmP>R-*%ZGEKWfcm7%Ka#j<>TM_0SGmYn!)|taokb|sJJL3x2Ab9JD zeUx;AUC70@>U_D?&ML*wn;Q|OrCPLOIj+?*(7z?4?X28nlhiJy&|?2P_T0stm7ult zRNb&1RinA98V1J93@H29@a|UYw-12y4+cVK;?mur zDKV$aoK7J;oKnop-S4FdTMHQNwjl#d_j?BiiNK|7_J}0N>a`Q!gP%VjlO0ie?y9n0 zY)Khh8`%rGDt|9le}w+z;z+Ne$J}u*`aj4cBgo?gSd@xnr}CjuAmh1&wTM&dAy!c< zg4m_1X&K^o8`6>F#26Iegy~#*dZptI^sb^#Um)m6Q^Cl-RFDvk1YFFjxlw+ z<6Ds1%WmAu&guE)C9iYxqScUM#DDsu^5%ekJlcPzse0xTq`0wk zwX|iismJ%!5>Dcw=C3=1r7a8;SX<`%(+Fw3iT=ZRc3nP&r|AC!>%5e|2h_>HOVb#I zNtKJ*ONS!Y*45|9(({l3zWUSI>&XM)z#Xi$xJy1Ss<+2lEngsK z(}Vs^k1nc$jC_6^rob13dI8bll1lbYZa8mtbO$+}@UbgDI)T1wmv=S*=MzHo*TL?n z8D=VKBO4~D(w4AXVJ&5HDiP^lNnKIS&Z=h?prLRd$L=*OFYa}V9xvn`iR3rboIxFQ z3ilECV-d78GpMwb_*mBeeDXprfdf=sZmPJYw| zfP>wUHJwO`W#nPZP1UL|g!U{$GO>kJk%V;S#Vy2FiIq@9bMVNvVPE8vePg6Vv9e69 z&#TPMG{=8o$a6m9Pn@f|ysPGv=)NU>io+Ia!4i@;8!FlUs^RfsL*Q#N<~KY7Wc*;y zMCXrHt|osWn4#KF{i#r{9jLkrJr*V{R_BCkM+M9T>aSuB3AG3?S7A2QP#fZ--ljM{ zx)VvsVUik!isNo9kn6r@anYFekdIO#r0n>7#G_0v)`Kmg`a)#JtTSj$9J!{$H}ET} zO#jX9zIeGsmatO-r<{=A%3M4u{T`jpmPQ>tZ$+>>7Xz@kaTO>H<3kbA-YJOHwp5RK zdW(Z|wR~(1x#-_LC)&+R{e&O;Ph(X-k_|e`zZg?D7toM^8 za_tI|tmI=0U|`LG(4x3-ld<9{kgfY=!tfxYMFFIj+S$a>EXluXmRfaBXk@9y={BB?i}&g^3!!tJ#v1oGr{|` zd$Yc5%3s;7z2|>G4)O%u`Douhm7!i>Fej{r(JiA^uQ!PEJ%MpdUEPJ7mSHo zvws1+KqHMj?V}?D^S2)UkwV{9p>o%dx|;C}a7%^3)!vGDnCjjdM6n`sY-Z%a{*wtfOU<55eTvF>OMspfbWwH9rdD*A%_l42bx*(q4 z8RFvRWfPpQDR^L1*zVyuy6-0898jSw@sHAGfv~2<3YMlevIf7W^kLaH_e|vF zd5~DpEF-3Dd=yb!NkT&SxCtyFGz*p5P&)n1gGSp=L>gqau-Mv2f0e{215Q&(`Am*@ zZfXLn)LX(Iu>JM9m8+omQANdibP5i^h3NEnu!x?*>WDohL%Ovzd&h&fn4KKqUyQft zga^45qSx)_oyCGSROos0neaMvgJ2d&@n*T3pNg4gbV-`55CsH?EJ24t<1vQN<2<5&uaeG_Nqd6`RA0?Oc z@b}}jJquax0ymyKlQ?*eF~wehnONqNKPo7D&mAJFScBgX&E<$_!1OWozq_87_sF#C znD-J3I=2$l20Nfog>QBZJVfzrb;Gg_Hhwe!}J(y?&*I3y~ns+9|^7ZMKof;W`1h?o+nDU9Z?v9PF9VLy%s0#8qe*4Xq z&b6we8_CTG37zcd2&@H|P=BIZMM5D|4>ILO`T%n&`o~=03u~bc{X_-eYKX@9?v!-n z@d4dm&NA6rkVQv|kJsna`ea?UTE6NvULace1tqHtbQSm(9dMqpwal9jMbc(PXY;7_ zN#>P?e$-~idWO<}efd%e8QS#K z54`G3aG%*9cEN4hOHG0+vKQDo9Q`h2J1V+&5H&S!}9O~6jc zAXB`0yDOWUp>RED{DdD|w>eWi>%7&@56E7!-h^#z+5{0zgn_S-hm2Y{P2^n_$z`O6Huv^JAbT%Bz@4j-CyjgMbDzyV%&J)x630p^Ek9)?Ei%_cGQI(PFPP zfAi(qD|4_Mf&+9}rFN8gipBLc0c=77_A#P#D zVPWcK%5GtHl&q|}$Vv8#uqy;uVjbLgjX`J?bJWAfbl4ML$c0q!#Ys^rBSO*7RKeu1 z*;eS#{O}HP*z8!dh~p|XuwZZVUCzH|BW1^Nr#h3Ve|&BEhr9$NAvysW4>Vt64`fjgHS9V$)Un6hM-7yP-gix9dV^8f;4V~<8bD7dnlcfESAn07Lh3#zKMs? z7yBvFXfb>>V8t}dW|lznDCj%)p4DcU*k(-pj9oH@Pc(2zgN4#XCoA~(I#85=$SWWO zKS8##4mT;1j6w2!gfCPueb!v{oEeYIOj)=VBbxTg7defgeXRXJR(j|F6F*UD<14mg zt&IXiU^7P-6-Suptp}d^T5Fb1ghCE7S@Xm$N~{DsS4c33!nR9L|H7D?f}0p)BVwNT zEjLoqDpeF+FSBdOR6C?}#rJ8!NY~J~K}?hd_jZ#;)aD3XtGdp~K#R$Sj|TPVTmol+ z_D4p6Lj2#3d7-N8q7Ch!R{0qAB3y319ZeV{NO9uhEalhTpOFnkva1u6s)vnb^K&+Q z;{DS!e&n%Gb%1@{rGgL0s!6`af(Uo`9gYZJZfARnkzSB zvGnCH3OMww{RAs1x#5_r(w-^n(3Dj$%Gb@pnRGzP3iRiD6R_XwWNw#u`n1GdQdqc6 zJLWWVm~qljwDR-~exIJ6j^3@Rri6qnhVq|?&d_orPL2PR;m^tYukkg&`x*WGPrm>E aL;h2)|9eM%z_TTc!3$0U diff --git a/pixels-daemon/src/test/java/io/pixelsdb/pixels/daemon/metadata/dao/impl/TestRdbFileDaoCleanupAt.java b/pixels-daemon/src/test/java/io/pixelsdb/pixels/daemon/metadata/dao/impl/TestRdbFileDaoCleanupAt.java new file mode 100644 index 0000000000..47879c9afb --- /dev/null +++ b/pixels-daemon/src/test/java/io/pixelsdb/pixels/daemon/metadata/dao/impl/TestRdbFileDaoCleanupAt.java @@ -0,0 +1,406 @@ +/* + * Copyright 2026 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the Affero GNU General Public + * License along with Pixels. If not, see + * . + */ +package io.pixelsdb.pixels.daemon.metadata.dao.impl; + +import io.pixelsdb.pixels.common.utils.MetaDBUtil; +import io.pixelsdb.pixels.daemon.MetadataProto; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Types; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Mockito-based unit tests for the c01.1 changes in {@link RdbFileDao} that govern how the + * optional {@code FILE_CLEANUP_AT} column is persisted and restored. + * + *

Contract under test: + *

    + *
  • On INSERT / UPDATE: {@code FILE_CLEANUP_AT} is bound to a real {@code long} only when + * {@code type == RETIRED && hasCleanupAt()}. Every other combination must bind {@code NULL}.
    • + *
  • On SELECT: a {@code wasNull()} column on the result set must materialise as + * {@code !proto.hasCleanupAt()} on the wire.
    • + *
  • {@code atomicSwapFiles} must clear {@code FILE_CLEANUP_AT} (set to NULL) when it + * promotes the new file to {@code REGULAR}, otherwise stale deadlines would leak across + * the swap boundary.
    • + *
+ * + *

The DAO calls {@code MetaDBUtil.Instance().getConnection()} on every method, so we + * inject a mock {@link Connection} into the singleton via reflection. This keeps the test + * a true unit test (no JDBC driver, no schema, no network). + * + * @author tdd-guide + * @create 2026-05-13 + */ +public class TestRdbFileDaoCleanupAt +{ + private static final int RETIRED_VALUE = MetadataProto.File.Type.RETIRED.getNumber(); + private static final int REGULAR_VALUE = MetadataProto.File.Type.REGULAR.getNumber(); + private static final int TEMPORARY_INGEST_VALUE = + MetadataProto.File.Type.TEMPORARY_INGEST.getNumber(); + + private Connection mockConn; + private Connection originalConn; + + private RdbFileDao dao; + + @Before + public void setUp() throws Exception + { + mockConn = mock(Connection.class); + // The DAO does conn.isValid(30) on lazy reconnect; force it to report healthy so the + // production code path stays on our mock rather than re-acquiring a real connection. + when(mockConn.isValid(anyInt())).thenReturn(true); + + originalConn = swapConnection(mockConn); + dao = new RdbFileDao(); + } + + @After + public void tearDown() throws Exception + { + // Always restore the real connection so subsequent tests in the same JVM are unaffected. + swapConnection(originalConn); + } + + // ------------------------------------------------------------------------- + // INSERT — single row + // ------------------------------------------------------------------------- + + /** + * For non-RETIRED file types (REGULAR, TEMPORARY_INGEST, TEMPORARY_GC), the DAO must NOT persist a cleanup deadline, + * even if a stray {@code cleanupAt} happens to be present on the proto. + */ + @Test + public void insert_nonRetiredFileTypes_bindCleanupAtAsNull() throws Exception + { + // Test REGULAR file with stray cleanupAt value + PreparedStatement pst1 = stubPreparedStatementForInsert(); + MetadataProto.File regularFile = baseFile("a.pxl", REGULAR_VALUE) + .setCleanupAt(123_456_789L) // deliberately stray; type != RETIRED so MUST be ignored + .build(); + dao.insert(regularFile); + verify(pst1).setNull(7, Types.BIGINT); + verify(pst1, never()).setLong(eq(7), anyLong()); + + // Test TEMPORARY_INGEST file (no cleanupAt) + PreparedStatement pst2 = stubPreparedStatementForInsert(); + MetadataProto.File ingestFile = baseFile("ingest.pxl", TEMPORARY_INGEST_VALUE).build(); + dao.insert(ingestFile); + verify(pst2).setNull(7, Types.BIGINT); + verify(pst2, never()).setLong(eq(7), anyLong()); + } + + /** + * RETIRED file binding tests covering various cleanupAt scenarios + */ + @Test + public void insert_retiredFile_bindingScenarios() throws Exception + { + // Test RETIRED file with cleanup deadline + PreparedStatement pst1 = stubPreparedStatementForInsert(); + long deadline = 1_700_000_000_000L; + MetadataProto.File retiredWithDeadline = baseFile("retired.pxl", RETIRED_VALUE) + .setCleanupAt(deadline) + .build(); + dao.insert(retiredWithDeadline); + verify(pst1).setLong(7, deadline); + verify(pst1, never()).setNull(eq(7), anyInt()); + + // Test RETIRED file without cleanupAt (should bind NULL) + PreparedStatement pst2 = stubPreparedStatementForInsert(); + MetadataProto.File retiredNoDeadline = baseFile("retired_unset.pxl", RETIRED_VALUE).build(); + dao.insert(retiredNoDeadline); + verify(pst2).setNull(7, Types.BIGINT); + verify(pst2, never()).setLong(eq(7), anyLong()); + + // Test RETIRED file with cleanupAt = 0L (should bind as long zero, not NULL) + PreparedStatement pst3 = stubPreparedStatementForInsert(); + MetadataProto.File retiredZero = baseFile("retired_zero.pxl", RETIRED_VALUE) + .setCleanupAt(0L) + .build(); + dao.insert(retiredZero); + verify(pst3).setLong(7, 0L); + verify(pst3, never()).setNull(eq(7), anyInt()); + } + + // ------------------------------------------------------------------------- + // INSERT BATCH — verifies per-row binding semantics + // ------------------------------------------------------------------------- + + @Test + public void insertBatch_mixedTypes_bindsCleanupAtPerRow() throws Exception + { + PreparedStatement pst = stubPreparedStatementForInsert(); + + MetadataProto.File regular = baseFile("r.pxl", REGULAR_VALUE).build(); + MetadataProto.File retiredWithDeadline = baseFile("d.pxl", RETIRED_VALUE) + .setCleanupAt(42L).build(); + MetadataProto.File retiredNoDeadline = baseFile("nd.pxl", RETIRED_VALUE).build(); + + dao.insertBatch(Arrays.asList(regular, retiredWithDeadline, retiredNoDeadline)); + + // Two rows must bind NULL (regular + retired-without-deadline), one row binds a long. + verify(pst, times(2)).setNull(7, Types.BIGINT); + verify(pst, times(1)).setLong(7, 42L); + verify(pst).executeBatch(); + } + + // ------------------------------------------------------------------------- + // UPDATE — index 6 carries cleanupAt (id is bound at index 7) + // ------------------------------------------------------------------------- + + /** + * UPDATE operation binding tests for different file types and cleanupAt scenarios + */ + @Test + public void update_bindingScenarios() throws Exception + { + // Test REGULAR file - should bind cleanupAt as NULL + PreparedStatement pst1 = mock(PreparedStatement.class); + when(mockConn.prepareStatement(anyString())).thenReturn(pst1); + when(pst1.executeUpdate()).thenReturn(1); + + MetadataProto.File regularFile = baseFile("u.pxl", REGULAR_VALUE).setId(7L).build(); + boolean ok1 = dao.update(regularFile); + + assertTrue(ok1); + verify(pst1).setNull(6, Types.BIGINT); + verify(pst1).setLong(7, 7L); // WHERE FILE_ID = ? + + // Test RETIRED file with cleanup deadline - should bind as long + PreparedStatement pst2 = mock(PreparedStatement.class); + when(mockConn.prepareStatement(anyString())).thenReturn(pst2); + when(pst2.executeUpdate()).thenReturn(1); + + long deadline = 1_700_000_000_999L; + MetadataProto.File retiredFile = baseFile("u.pxl", RETIRED_VALUE) + .setId(8L) + .setCleanupAt(deadline) + .build(); + boolean ok2 = dao.update(retiredFile); + + assertTrue(ok2); + verify(pst2).setLong(6, deadline); + verify(pst2).setLong(7, 8L); + } + + // ------------------------------------------------------------------------- + // atomicSwapFiles — cleanupAt must be reset to NULL on promote + // ------------------------------------------------------------------------- + + /** + * The promote step must use the SQL fragment {@code FILE_CLEANUP_AT=NULL}. Without it, + * a file that was previously RETIRED and is being recycled into a fresh REGULAR slot + * would silently retain its deadline, eventually getting GC'd while live. + */ + @Test + public void atomicSwapFiles_promoteSqlClearsCleanupAt() throws Exception + { + PreparedStatement updatePst = mock(PreparedStatement.class); + PreparedStatement deletePst = mock(PreparedStatement.class); + + when(mockConn.prepareStatement(anyString())).thenAnswer(inv -> { + String sql = inv.getArgument(0); + if (sql.startsWith("UPDATE")) + { + return updatePst; + } + if (sql.startsWith("DELETE")) + { + return deletePst; + } + return mock(PreparedStatement.class); + }); + + boolean ok = dao.atomicSwapFiles(101L, Arrays.asList(11L, 12L)); + assertTrue(ok); + + // Capture the actual SQL string the production code sent to the JDBC driver. + org.mockito.ArgumentCaptor sqlCaptor = org.mockito.ArgumentCaptor.forClass(String.class); + verify(mockConn, atLeastOnce()).prepareStatement(sqlCaptor.capture()); + boolean clearsCleanupAt = false; + for (String sql : sqlCaptor.getAllValues()) + { + if (sql.contains("FILE_TYPE=?") && sql.contains("FILE_CLEANUP_AT=NULL")) + { + clearsCleanupAt = true; + break; + } + } + assertTrue("promote SQL must clear FILE_CLEANUP_AT to NULL together with the type update", + clearsCleanupAt); + + // The promote binds REGULAR + the new id, then commits. These behaviours are tied + // to the same transaction as the DELETE, so we check both ran on the same connection. + verify(updatePst).setInt(1, REGULAR_VALUE); + verify(updatePst).setLong(2, 101L); + verify(updatePst).executeUpdate(); + verify(deletePst).setLong(1, 11L); + verify(deletePst).setLong(2, 12L); + verify(deletePst).executeUpdate(); + verify(mockConn).setAutoCommit(false); + verify(mockConn).commit(); + } + + @Test + public void atomicSwapFiles_rollsBackOnSqlException() throws Exception + { + when(mockConn.prepareStatement(anyString())) + .thenThrow(new SQLException("boom")); + + boolean ok = dao.atomicSwapFiles(1L, Collections.singletonList(2L)); + + assertFalse("atomicSwapFiles must report failure when the JDBC layer throws", ok); + verify(mockConn).setAutoCommit(false); + verify(mockConn).rollback(); + verify(mockConn).setAutoCommit(true); // finally block must restore auto-commit + verify(mockConn, never()).commit(); + } + + // ------------------------------------------------------------------------- + // SELECT (buildFile) — cleanupAt round-trip from ResultSet to proto + // ------------------------------------------------------------------------- + + /** + * SELECT operation tests covering different cleanupAt scenarios from ResultSet to proto + */ + @Test + public void getById_cleanupAtRoundTripScenarios() throws Exception + { + // Test scenario 1: ResultSet with cleanupAt value (non-NULL) + Statement st1 = mock(Statement.class); + ResultSet rs1 = mock(ResultSet.class); + when(mockConn.createStatement()).thenReturn(st1); + when(st1.executeQuery(anyString())).thenReturn(rs1); + when(rs1.next()).thenReturn(true).thenReturn(false); + + when(rs1.getLong("FILE_ID")).thenReturn(99L); + when(rs1.getString("FILE_NAME")).thenReturn("x.pxl"); + when(rs1.getInt("FILE_TYPE")).thenReturn(RETIRED_VALUE); + when(rs1.getInt("FILE_NUM_RG")).thenReturn(2); + when(rs1.getLong("FILE_MIN_ROW_ID")).thenReturn(0L); + when(rs1.getLong("FILE_MAX_ROW_ID")).thenReturn(127L); + when(rs1.getLong("PATHS_PATH_ID")).thenReturn(5L); + when(rs1.getLong("FILE_CLEANUP_AT")).thenReturn(1_700_000_000_000L); + when(rs1.wasNull()).thenReturn(false); + + MetadataProto.File proto1 = dao.getById(99L); + + assertNotNull(proto1); + assertEquals(99L, proto1.getId()); + assertEquals(MetadataProto.File.Type.RETIRED, proto1.getType()); + assertTrue("non-NULL FILE_CLEANUP_AT column must surface as hasCleanupAt()", + proto1.hasCleanupAt()); + assertEquals(1_700_000_000_000L, proto1.getCleanupAt()); + + // Test scenario 2: ResultSet with NULL cleanupAt + Statement st2 = mock(Statement.class); + ResultSet rs2 = mock(ResultSet.class); + when(mockConn.createStatement()).thenReturn(st2); + when(st2.executeQuery(anyString())).thenReturn(rs2); + when(rs2.next()).thenReturn(true).thenReturn(false); + + when(rs2.getLong("FILE_ID")).thenReturn(1L); + when(rs2.getString("FILE_NAME")).thenReturn("r.pxl"); + when(rs2.getInt("FILE_TYPE")).thenReturn(REGULAR_VALUE); + when(rs2.getInt("FILE_NUM_RG")).thenReturn(1); + when(rs2.getLong("FILE_MIN_ROW_ID")).thenReturn(0L); + when(rs2.getLong("FILE_MAX_ROW_ID")).thenReturn(0L); + when(rs2.getLong("PATHS_PATH_ID")).thenReturn(1L); + when(rs2.getLong("FILE_CLEANUP_AT")).thenReturn(0L); + when(rs2.wasNull()).thenReturn(true); // critical: NULL column + + MetadataProto.File proto2 = dao.getById(1L); + + assertNotNull(proto2); + assertFalse("NULL FILE_CLEANUP_AT column must surface as !hasCleanupAt()", + proto2.hasCleanupAt()); + } + + // ------------------------------------------------------------------------- + // helpers + // ------------------------------------------------------------------------- + + private PreparedStatement stubPreparedStatementForInsert() throws SQLException + { + PreparedStatement pst = mock(PreparedStatement.class); + when(mockConn.prepareStatement(anyString())).thenReturn(pst); + when(pst.executeUpdate()).thenReturn(1); + + // After a successful insert, the DAO calls executeQuery("SELECT LAST_INSERT_ID()") + // on the same PreparedStatement. Stub a single-row ResultSet so the call returns cleanly. + ResultSet idRs = mock(ResultSet.class); + when(pst.executeQuery(anyString())).thenReturn(idRs); + when(idRs.next()).thenReturn(true); + when(idRs.getLong(1)).thenReturn(1L); + return pst; + } + + private static MetadataProto.File.Builder baseFile(String name, int typeValue) + { + return MetadataProto.File.newBuilder() + .setName(name) + .setTypeValue(typeValue) + .setNumRowGroup(1) + .setMinRowId(0L) + .setMaxRowId(0L) + .setPathId(1L); + } + + /** + * Replace the private {@code connection} field in the {@link MetaDBUtil} singleton with + * the supplied connection, returning the previous value. Using reflection here keeps + * the production class untouched while still letting us inject a Mockito-managed + * {@link Connection} for the duration of a single test. + */ + private static Connection swapConnection(Connection replacement) throws Exception + { + Field f = MetaDBUtil.class.getDeclaredField("connection"); + f.setAccessible(true); + Connection previous = (Connection) f.get(MetaDBUtil.Instance()); + f.set(MetaDBUtil.Instance(), replacement); + return previous; + } +} diff --git a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/FileWriterManager.java b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/FileWriterManager.java index c66c063177..7c463e0814 100644 --- a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/FileWriterManager.java +++ b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/FileWriterManager.java @@ -87,7 +87,7 @@ public FileWriterManager(long tableId, TypeDescription schema, MetadataService metadataService = MetadataService.Instance(); file = new File(); this.file.setName(targetFileName); - this.file.setType(File.Type.TEMPORARY); + this.file.setType(File.Type.TEMPORARY_INGEST); this.file.setNumRowGroup(1); this.file.setPathId(targetOrderedDirPath.getId()); if (!metadataService.addFiles(Collections.singletonList(file))) diff --git a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/StorageGarbageCollector.java b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/StorageGarbageCollector.java index c0b21ec7d8..9aa31eaea1 100644 --- a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/StorageGarbageCollector.java +++ b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/StorageGarbageCollector.java @@ -607,7 +607,7 @@ void processFileGroups(List fileGroups, long safeGcTs, * Rewrites all files in one {@link FileGroup} into a single new file, filtering out * rows marked as deleted in {@code gcSnapshotBitmaps}. * - *

The new file is registered as {@code TEMPORARY} in the catalog and its + *

The new file is registered as {@code TEMPORARY_GC} in the catalog and its * {@link RGVisibility} objects are initialised with {@code baseTimestamp = safeGcTs}. * *

After rewriting completes the {@code gcSnapshotBitmaps} entries for this group @@ -877,7 +877,7 @@ RewriteResult rewriteFileGroup(FileGroup group, long safeGcTs, backwardInfos.add(new BackwardInfo(fc.fileId, bwdMappings, oldFileRgRowStart)); } - // Register the new file as TEMPORARY in the catalog and initialise Visibility. + // Register the new file as TEMPORARY_GC in the catalog and initialise Visibility. // Track registration progress so that partial state can be cleaned up on failure. long newFileId = -1; int registeredRgCount = 0; @@ -891,7 +891,7 @@ RewriteResult rewriteFileGroup(FileGroup group, long safeGcTs, } File newFile = new File(); newFile.setName(newFileName); - newFile.setType(File.Type.TEMPORARY); + newFile.setType(File.Type.TEMPORARY_GC); newFile.setNumRowGroup(newFileRgCount); newFile.setMinRowId(minRowId); newFile.setMaxRowId(maxRowId); @@ -920,7 +920,7 @@ RewriteResult rewriteFileGroup(FileGroup group, long safeGcTs, } /** - * Best-effort cleanup of a partially-created TEMPORARY file. Removes the + * Best-effort cleanup of a partially-created TEMPORARY_GC file. Removes the * catalog record, the physical file, and any RGVisibility keys that were * registered before the failure. */ @@ -1173,7 +1173,7 @@ private List updateSinglePointIndex(RewriteResult result, long tableId, // ------------------------------------------------------------------------- /** - * Atomically promotes the new TEMPORARY file to REGULAR, deletes old files from + * Atomically promotes the new TEMPORARY_GC file to REGULAR, deletes old files from * the catalog, unregisters dual-write, and enqueues the old files for delayed cleanup. */ void commitFileGroup(RewriteResult result) throws Exception diff --git a/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestIngestFilePublisher.java b/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestIngestFilePublisher.java index c6f9069015..817b27a1c2 100644 --- a/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestIngestFilePublisher.java +++ b/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestIngestFilePublisher.java @@ -62,8 +62,8 @@ public void finishClosesPhysicalFileOnlyOnceAndLeavesMetadataTemporary() throws assertSame(firstFinish, secondFinish); assertEquals(1, writer.closeCount.get()); - assertEquals(File.Type.TEMPORARY, file.getType()); - assertEquals(File.Type.TEMPORARY, fileWriterManager.getFileSnapshot().getType()); + assertEquals(File.Type.TEMPORARY_INGEST, file.getType()); + assertEquals(File.Type.TEMPORARY_INGEST, fileWriterManager.getFileSnapshot().getType()); assertTrue(firstFinish.isDone()); assertFalse(firstFinish.isCompletedExceptionally()); } @@ -90,8 +90,8 @@ public void finishFailureIsPropagatedAndDoesNotPublishMetadata() throws Exceptio assertSame(firstFinish, secondFinish); assertTrue(secondFinish.isCompletedExceptionally()); assertEquals(1, writer.closeCount.get()); - assertEquals(File.Type.TEMPORARY, file.getType()); - assertEquals(File.Type.TEMPORARY, fileWriterManager.getFileSnapshot().getType()); + assertEquals(File.Type.TEMPORARY_INGEST, file.getType()); + assertEquals(File.Type.TEMPORARY_INGEST, fileWriterManager.getFileSnapshot().getType()); } @Test @@ -129,7 +129,7 @@ public void fileSnapshotDoesNotExposeInternalFileState() File freshSnapshot = fileWriterManager.getFileSnapshot(); assertEquals("ingest_203.pxl", file.getName()); - assertEquals(File.Type.TEMPORARY, file.getType()); + assertEquals(File.Type.TEMPORARY_INGEST, file.getType()); assertEquals(1, file.getNumRowGroup()); assertEquals(0, file.getMinRowId()); assertEquals(63, file.getMaxRowId()); @@ -150,7 +150,7 @@ public void fileSnapshotReflectsMutationsOnUnderlyingFile() FileWriterManager fileWriterManager = testFileWriterManager(writer, file); File before = fileWriterManager.getFileSnapshot(); - assertEquals(File.Type.TEMPORARY, before.getType()); + assertEquals(File.Type.TEMPORARY_INGEST, before.getType()); assertEquals(63L, before.getMaxRowId()); // Mutations on the underlying file (e.g. visibility/row id updates) must be observed @@ -206,7 +206,7 @@ public void addRowBatchSucceedsAndForwardsToWriter() throws Exception assertEquals(3, writer.addRowBatchCount.get()); assertEquals(0, writer.closeCount.get()); - assertEquals(File.Type.TEMPORARY, file.getType()); + assertEquals(File.Type.TEMPORARY_INGEST, file.getType()); } @Test @@ -227,10 +227,10 @@ public void addRowBatchFailureLeavesManagerUsableForFinish() throws Exception } // After a failed addRowBatch, finish() must still close the underlying writer exactly once - // and keep the file in TEMPORARY state (publication is the buffer's responsibility). + // and keep the file in TEMPORARY_INGEST state (publication is the buffer's responsibility). fileWriterManager.finish().get(5, TimeUnit.SECONDS); assertEquals(1, writer.closeCount.get()); - assertEquals(File.Type.TEMPORARY, file.getType()); + assertEquals(File.Type.TEMPORARY_INGEST, file.getType()); } @SuppressWarnings("unchecked") @@ -280,7 +280,7 @@ public void finishIsIdempotentUnderConcurrentCallers() throws Exception firstFinish.get(5, TimeUnit.SECONDS); assertEquals("writer.close() must run at most once even under concurrent finish() calls", 1, writer.closeCount.get()); - assertEquals(File.Type.TEMPORARY, file.getType()); + assertEquals(File.Type.TEMPORARY_INGEST, file.getType()); } @Test @@ -328,7 +328,7 @@ public void finishPropagatesRuntimeExceptionFromClose() throws Exception assertSame(firstFinish, secondFinish); assertTrue(secondFinish.isCompletedExceptionally()); assertEquals(1, writer.closeCount.get()); - assertEquals(File.Type.TEMPORARY, file.getType()); + assertEquals(File.Type.TEMPORARY_INGEST, file.getType()); } @Test(timeout = 10_000L) @@ -400,7 +400,7 @@ public void concurrentAddRowBatchesAreAllForwardedToWriter() throws Exception assertEquals(callerCount * callsPerCaller, writer.addRowBatchCount.get()); assertEquals(0, writer.closeCount.get()); - assertEquals(File.Type.TEMPORARY, file.getType()); + assertEquals(File.Type.TEMPORARY_INGEST, file.getType()); } @Test @@ -429,7 +429,7 @@ public void finishIsRobustAgainstFileMetadataMutationsBeforeReturn() throws Exce File snapshot = fileWriterManager.getFileSnapshot(); assertEquals(255L, snapshot.getMaxRowId()); assertEquals(3, snapshot.getNumRowGroup()); - assertEquals(File.Type.TEMPORARY, snapshot.getType()); + assertEquals(File.Type.TEMPORARY_INGEST, snapshot.getType()); assertEquals(1, writer.closeCount.get()); } @@ -469,7 +469,7 @@ public boolean addRowBatch(VectorizedRowBatch rowBatch) throws IOException // After a runtime failure inside the writer, finish() must still be able to close it. fileWriterManager.finish().get(5, TimeUnit.SECONDS); assertEquals(1, writer.closeCount.get()); - assertEquals(File.Type.TEMPORARY, file.getType()); + assertEquals(File.Type.TEMPORARY_INGEST, file.getType()); } private static File temporaryFile(long id) @@ -477,7 +477,7 @@ private static File temporaryFile(long id) File file = new File(); file.setId(id); file.setName("ingest_" + id + ".pxl"); - file.setType(File.Type.TEMPORARY); + file.setType(File.Type.TEMPORARY_INGEST); file.setNumRowGroup(1); file.setMinRowId(0); file.setMaxRowId(63); diff --git a/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestStorageGarbageCollector.java b/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestStorageGarbageCollector.java index bc2e14a21a..81be9522cd 100644 --- a/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestStorageGarbageCollector.java +++ b/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestStorageGarbageCollector.java @@ -1882,7 +1882,7 @@ public void testDualWrite_concurrentPressure() throws Exception // ======================================================================= /** - * Atomicity with multiple old files: one TEMPORARY new file and three REGULAR + * Atomicity with multiple old files: one TEMPORARY_GC new file and three REGULAR * old files are swapped in a single call. Verifies that after the call the new * file is promoted to REGULAR and all old files are removed from the * catalog—i.e., the UPDATE and DELETE execute as one indivisible transaction. @@ -1901,12 +1901,12 @@ public void testAtomicSwap_multipleOldFilesAtomicity() throws Exception new String[]{"atom_old1.pxl", "atom_old2.pxl", "atom_old3.pxl"}, new File.Type[]{File.Type.REGULAR, File.Type.REGULAR, File.Type.REGULAR}, new int[]{1, 1, 1}, new long[]{0, 0, 0}, new long[]{1, 1, 1}); - long newFileId = registerTestFile("atom_new.pxl", File.Type.TEMPORARY, 1, 0, 1); + long newFileId = registerTestFile("atom_new.pxl", File.Type.TEMPORARY_GC, 1, 0, 1); File preSwapNew = metadataService.getFileById(newFileId); assertNotNull("New file must exist before swap", preSwapNew); - assertEquals("New file should be TEMPORARY before swap", - File.Type.TEMPORARY, preSwapNew.getType()); + assertEquals("New file should be TEMPORARY_GC before swap", + File.Type.TEMPORARY_GC, preSwapNew.getType()); metadataService.atomicSwapFiles(newFileId, Arrays.asList(oldIds[0], oldIds[1], oldIds[2])); @@ -1927,7 +1927,7 @@ public void testAtomicSwap_idempotent() throws Exception { writeTestFile("idem_old.pxl", LONG_ID_SCHEMA, new long[]{0, 1, 2}, true, new long[]{100, 100, 100}); long oldFileId = registerTestFile("idem_old.pxl", File.Type.REGULAR, 1, 0, 2); - long newFileId = registerTestFile("idem_new.pxl", File.Type.TEMPORARY, 1, 0, 2); + long newFileId = registerTestFile("idem_new.pxl", File.Type.TEMPORARY_GC, 1, 0, 2); metadataService.atomicSwapFiles(newFileId, Collections.singletonList(oldFileId)); assertFileRegular(newFileId, "File should be REGULAR after first swap"); @@ -1939,8 +1939,8 @@ public void testAtomicSwap_idempotent() throws Exception } /** - * TEMPORARY visibility semantics: before the swap, {@code getFiles(pathId)} must - * not return the TEMPORARY new file (the DAO filters {@code FILE_TYPE = REGULAR}). + * TEMPORARY_GC visibility semantics: before the swap, {@code getFiles(pathId)} must + * not return the TEMPORARY_GC new file (the DAO filters {@code FILE_TYPE = REGULAR}). * After the swap the promoted file is visible and the old file disappears. */ @Test @@ -1949,7 +1949,7 @@ public void testAtomicSwap_temporaryInvisibleViaGetFiles() throws Exception writeTestFile("vis_old.pxl", LONG_ID_SCHEMA, new long[]{0, 1}, true, new long[]{100, 100}); long[] fileIds = registerTestFiles( new String[]{"vis_old.pxl", "vis_new_temp.pxl"}, - new File.Type[]{File.Type.REGULAR, File.Type.TEMPORARY}, + new File.Type[]{File.Type.REGULAR, File.Type.TEMPORARY_GC}, new int[]{1, 1}, new long[]{0, 0}, new long[]{1, 1}); long oldFileId = fileIds[0]; long tempFileId = fileIds[1]; @@ -1962,7 +1962,7 @@ public void testAtomicSwap_temporaryInvisibleViaGetFiles() throws Exception } assertTrue("REGULAR old file should be visible via getFiles before swap", beforeIds.contains(oldFileId)); - assertFalse("TEMPORARY new file must NOT be visible via getFiles before swap", + assertFalse("TEMPORARY_GC new file must NOT be visible via getFiles before swap", beforeIds.contains(tempFileId)); metadataService.atomicSwapFiles(tempFileId, Collections.singletonList(oldFileId)); @@ -2000,9 +2000,9 @@ public void testGetFiles_mixedAllFileTypes_onlyRegular() throws Exception regularId = registerTestFile("mix_regular_" + suffix + ".pxl", File.Type.REGULAR, 1, 0L, 1L); tempId = registerTestFile("mix_temp_" + suffix + ".pxl", - File.Type.TEMPORARY, 1, 0L, 1L); + File.Type.TEMPORARY_INGEST, 1, 0L, 1L); nonRegularPositiveId = insertRawFileWithType("mix_non_regular_" + suffix + ".pxl", - File.Type.REGULAR.ordinal() + 1, 1, 0L, 1L); + File.Type.TEMPORARY_GC.getNumber(), 1, 0L, 1L); negativeId = insertRawFileWithType("mix_negative_" + suffix + ".pxl", -2, 1, 0L, 1L); extremeId = insertRawFileWithType("mix_extreme_max_" + suffix + ".pxl", @@ -2018,7 +2018,7 @@ public void testGetFiles_mixedAllFileTypes_onlyRegular() throws Exception } assertTrue("REGULAR member of the mix must be visible", visible.contains(regularId)); - assertFalse("TEMPORARY (FILE_TYPE=0) must be hidden", + assertFalse("TEMPORARY_INGEST (FILE_TYPE=0) must be hidden", visible.contains(tempId)); assertFalse("non-REGULAR positive FILE_TYPE must be hidden", visible.contains(nonRegularPositiveId)); @@ -2039,6 +2039,135 @@ public void testGetFiles_mixedAllFileTypes_onlyRegular() throws Exception } } + // ------------------------------------------------------------------------- + // c01.1 regression — RETIRED is a new File.Type and must be invisible to + // query-time enumeration just like the two TEMPORARY_* states. These tests + // pin down the contract that the DAO filters FILE_TYPE = REGULAR and nothing + // else, so future refactors cannot accidentally widen the visible set. + // ------------------------------------------------------------------------- + + + + /** + * Exhaustive coverage: for every defined non-REGULAR {@link File.Type}, getFiles must + * exclude that file. Using {@link File.Type#values()} guards against future enum + * additions silently leaking into query results. + */ + @Test + public void testGetFiles_allNonRegularTypes_allHidden() throws Exception + { + List registeredIds = new ArrayList<>(); + long regularId = -1L; + try + { + String suffix = Long.toString(System.nanoTime()); + regularId = registerTestFile("all_types_regular_" + suffix + ".pxl", + File.Type.REGULAR, 1, 0L, 1L); + + // Register one file per non-REGULAR type, including RETIRED. + Set nonRegularIds = new HashSet<>(); + for (File.Type t : File.Type.values()) + { + if (t == File.Type.REGULAR) continue; + long id = insertRawFileWithType( + "all_types_" + t + "_" + suffix + ".pxl", + t.getNumber(), 1, 0L, 1L); + registeredIds.add(id); + nonRegularIds.add(id); + } + registeredIds.add(regularId); + + List visible = metadataService.getFiles(testPathId); + Set visibleIds = new HashSet<>(); + for (File f : visible) + { + assertEquals("every visible file must carry FILE_TYPE = REGULAR", + File.Type.REGULAR, f.getType()); + visibleIds.add(f.getId()); + } + assertTrue("the seed REGULAR file must be visible", + visibleIds.contains(regularId)); + for (long id : nonRegularIds) + { + assertFalse("non-REGULAR file (id=" + id + ") leaked into getFiles", + visibleIds.contains(id)); + } + } + finally + { + if (!registeredIds.isEmpty()) metadataService.deleteFiles(registeredIds); + } + } + + /** + * After the swap of a TEMPORARY_GC -> REGULAR, a RETIRED tombstone for the *old* file + * (i.e. the same file ids that were just deleted) cannot pollute the new visible set + * even if the catalog still carries unrelated RETIRED entries on the same path. + */ + @Test + public void testGetFiles_retiredCoexistsWithFreshlyPromoted() throws Exception + { + long oldRegularId = -1L; + long tempGcId = -1L; + long retiredCoexistingId = -1L; + try + { + String suffix = Long.toString(System.nanoTime()); + + // Pre-existing RETIRED file on the same path. This must remain hidden + // throughout the entire scenario. + retiredCoexistingId = insertRawFileWithType( + "coexist_retired_" + suffix + ".pxl", + File.Type.RETIRED.getNumber(), 1, 0L, 1L); + + // The classic swap pair. + oldRegularId = registerTestFile("coexist_old_regular_" + suffix + ".pxl", + File.Type.REGULAR, 1, 0L, 1L); + tempGcId = registerTestFile("coexist_new_temp_gc_" + suffix + ".pxl", + File.Type.TEMPORARY_GC, 1, 0L, 1L); + + // Before swap: only oldRegular visible; RETIRED + TEMPORARY_GC hidden. + Set beforeIds = new HashSet<>(); + for (File f : metadataService.getFiles(testPathId)) beforeIds.add(f.getId()); + assertTrue("old REGULAR must be visible before swap", + beforeIds.contains(oldRegularId)); + assertFalse("RETIRED tombstone must be hidden before swap", + beforeIds.contains(retiredCoexistingId)); + assertFalse("TEMPORARY_GC must be hidden before swap", + beforeIds.contains(tempGcId)); + + metadataService.atomicSwapFiles(tempGcId, Collections.singletonList(oldRegularId)); + + // After swap: tempGcId is now REGULAR (visible); old REGULAR is gone; the + // coexisting RETIRED file must STILL be hidden (the swap did not promote it). + Set afterIds = new HashSet<>(); + for (File f : metadataService.getFiles(testPathId)) + { + assertEquals("getFiles must only emit REGULAR after swap", + File.Type.REGULAR, f.getType()); + afterIds.add(f.getId()); + } + assertTrue("freshly-promoted file must be visible after swap", + afterIds.contains(tempGcId)); + assertFalse("the deleted old REGULAR must be gone after swap", + afterIds.contains(oldRegularId)); + assertFalse("the unrelated RETIRED tombstone must remain hidden after swap", + afterIds.contains(retiredCoexistingId)); + + // After the promote, the old file ids are deleted — clear the local handle so + // the cleanup block below does not double-delete a non-existent row. + oldRegularId = -1L; + } + finally + { + List cleanup = new ArrayList<>(); + if (oldRegularId > 0) cleanup.add(oldRegularId); + if (tempGcId > 0) cleanup.add(tempGcId); + if (retiredCoexistingId > 0) cleanup.add(retiredCoexistingId); + if (!cleanup.isEmpty()) metadataService.deleteFiles(cleanup); + } + } + /** * A minimum-size REGULAR file is returned with its catalog fields intact. */ @@ -2117,9 +2246,9 @@ public void testGetFiles_concurrentReaders_consistentRegularOnly() throws Except regularId = registerTestFile("conc_regular_" + suffix + ".pxl", File.Type.REGULAR, 1, 0L, 1L); tempId = registerTestFile("conc_temp_" + suffix + ".pxl", - File.Type.TEMPORARY, 1, 0L, 1L); + File.Type.TEMPORARY_INGEST, 1, 0L, 1L); nonRegularPositiveId = insertRawFileWithType("conc_non_regular_" + suffix + ".pxl", - File.Type.REGULAR.ordinal() + 1, 1, 0L, 1L); + File.Type.TEMPORARY_GC.getNumber(), 1, 0L, 1L); final int threads = 8; final int iterations = 16; @@ -2166,7 +2295,7 @@ public void testGetFiles_concurrentReaders_consistentRegularOnly() throws Except CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])) .get(30, java.util.concurrent.TimeUnit.SECONDS); - assertEquals("no concurrent reader may observe a TEMPORARY file", + assertEquals("no concurrent reader may observe a TEMPORARY_INGEST file", 0, leakedTemporary.get()); assertEquals("no concurrent reader may observe a non-REGULAR file", 0, leakedNonRegular.get()); @@ -2222,7 +2351,7 @@ public void testAtomicSwap_multipleSerialSwaps() throws Exception long[] pair = registerTestFiles( new String[]{oldName, newName}, - new File.Type[]{File.Type.REGULAR, File.Type.TEMPORARY}, + new File.Type[]{File.Type.REGULAR, File.Type.TEMPORARY_GC}, new int[]{1, 1}, new long[]{0, 0}, new long[]{0, 0}); oldFileIds[i] = pair[0]; newFileIds[i] = pair[1]; @@ -2261,7 +2390,7 @@ public void testAtomicSwap_partialOldFilesAlreadyGone() throws Exception metadataService.deleteFiles(Collections.singletonList(oldIds[0])); assertFileGone(oldIds[0], "old1 should be gone before swap"); - long newFileId = registerTestFile("partial_new.pxl", File.Type.TEMPORARY, 1, 0, 1); + long newFileId = registerTestFile("partial_new.pxl", File.Type.TEMPORARY_GC, 1, 0, 1); metadataService.atomicSwapFiles(newFileId, Arrays.asList(oldIds[0], oldIds[1])); assertFileRegular(newFileId, "New file must be REGULAR"); @@ -2270,7 +2399,7 @@ public void testAtomicSwap_partialOldFilesAlreadyGone() throws Exception /** * Rollback after rewrite + dual-write: verifies that Visibility entries for the new - * file are removed, dual-write is unregistered, the TEMPORARY catalog entry is deleted, + * file are removed, dual-write is unregistered, the TEMPORARY_GC catalog entry is deleted, * and the physical file is cleaned up. */ @Test @@ -2354,7 +2483,7 @@ public void testAtomicSwap_delayedCleanup() throws Exception * Phase 3 (ts=200, dual-write active): delete row 3 → propagated to both files * Sync visibility → export + coord-transform + import * Phase 4 (ts=300, post-sync, dual-write still active): delete row 5 - * Commit → atomic swap (TEMPORARY→REGULAR), old file removed from catalog + * Commit -> atomic swap (TEMPORARY_GC -> REGULAR), old file removed from catalog * Verify: multi-snap_ts consistency on new file at ts=100..500 * Verify: old file gone from catalog, new file REGULAR * diff --git a/proto/metadata.proto b/proto/metadata.proto index b10c30194c..b21556a102 100644 --- a/proto/metadata.proto +++ b/proto/metadata.proto @@ -174,8 +174,10 @@ message Path { message File { enum Type { - TEMPORARY = 0; + TEMPORARY_INGEST = 0; REGULAR = 1; + TEMPORARY_GC = 2; + RETIRED = 3; } uint64 id = 1; string name = 2; @@ -184,6 +186,7 @@ message File { uint64 minRowId = 5; uint64 maxRowId = 6; uint64 pathId = 7; + optional uint64 cleanupAt = 8; } message SchemaVersion { @@ -707,7 +710,7 @@ message GetFileTypeRequest { message GetFileTypeResponse { ResponseHeader header = 1; - File.Type fileType = 2; // the type of the file, e.g., REGULAR or EMPTY + File.Type fileType = 2; // the type of the file, e.g., REGULAR or RETIRED } message UpdateFileRequest { diff --git a/scripts/sql/metadata_schema.sql b/scripts/sql/metadata_schema.sql index 3f077e4417..c3b0c5a7e9 100644 --- a/scripts/sql/metadata_schema.sql +++ b/scripts/sql/metadata_schema.sql @@ -318,15 +318,17 @@ CREATE TABLE IF NOT EXISTS `pixels_metadata`.`PEER_PATHS` ( CREATE TABLE IF NOT EXISTS `pixels_metadata`.`FILES` ( `FILE_ID` BIGINT NOT NULL AUTO_INCREMENT, `FILE_NAME` VARCHAR(128) NOT NULL, - `FILE_TYPE` TINYINT NOT NULL COMMENT "Valid value can be 0 (temporary), or 1 (regular).", + `FILE_TYPE` TINYINT NOT NULL COMMENT "Valid value can be 0 (temporary ingest), 1 (regular), 2 (temporary gc), or 3 (retired).", `FILE_NUM_RG` INT NOT NULL, `FILE_MIN_ROW_ID` BIGINT NOT NULL, `FILE_MAX_ROW_ID` BIGINT NOT NULL, + `FILE_CLEANUP_AT` BIGINT NULL COMMENT "Earliest cleanup deadline in epoch milliseconds; meaningful only when FILE_TYPE = 3 (retired).", `PATHS_PATH_ID` BIGINT NOT NULL, PRIMARY KEY (`FILE_ID`), INDEX `fk_FILES_PATHS_idx` (`PATHS_PATH_ID` ASC), UNIQUE INDEX `PATH_ID_FILE_NAME_UNIQUE` (`PATHS_PATH_ID` ASC, `FILE_NAME` ASC), INDEX `FILE_ROW_ID_INDEX` USING BTREE (`FILE_MIN_ROW_ID`, `FILE_MAX_ROW_ID`), + INDEX `FILE_CLEANUP_AT_INDEX` USING BTREE (`FILE_TYPE`, `FILE_CLEANUP_AT`), CONSTRAINT `fk_FILES_PATHS` FOREIGN KEY (`PATHS_PATH_ID`) REFERENCES `pixels_metadata`.`PATHS` (`PATH_ID`) From 3ebfc6714001eefa7a4d9d64347307998b644f57 Mon Sep 17 00:00:00 2001 From: Dongyang Geng Date: Thu, 14 May 2026 17:34:07 +0800 Subject: [PATCH 09/17] feat(metadata)!: track cleanup deadlines for retired files --- .../common/metadata/MetadataService.java | 147 ++++- .../pixels/common/utils/DateUtil.java | 2 + .../common/utils/PixelsFileNameUtils.java | 33 + .../src/main/resources/pixels.properties | 2 + .../metadata/domain/TestFileDomain.java | 6 +- .../common/utils/TestPixelsFileNameUtils.java | 174 ++++++ .../pixels/daemon/cache/CacheCoordinator.java | 2 +- .../daemon/metadata/MetadataServiceImpl.java | 14 +- .../pixels/daemon/metadata/dao/FileDao.java | 7 +- .../daemon/metadata/dao/impl/RdbFileDao.java | 116 +++- .../daemon/retina/RetinaServerImpl.java | 4 +- .../daemon/metadata/dao/TestRdbFileDao.java | 581 ++++++++++++++++++ .../dao/impl/TestRdbFileDaoCleanupAt.java | 406 ------------ .../daemon/retina/TestRetinaServer.java | 4 +- .../pixels/planner/PixelsPlanner.java | 2 +- .../retina/StorageGarbageCollector.java | 2 +- .../retina/TestStorageGarbageCollector.java | 40 +- proto/metadata.proto | 11 +- scripts/sql/metadata_schema.sql | 1 - 19 files changed, 1071 insertions(+), 483 deletions(-) create mode 100644 pixels-common/src/test/java/io/pixelsdb/pixels/common/utils/TestPixelsFileNameUtils.java create mode 100644 pixels-daemon/src/test/java/io/pixelsdb/pixels/daemon/metadata/dao/TestRdbFileDao.java delete mode 100644 pixels-daemon/src/test/java/io/pixelsdb/pixels/daemon/metadata/dao/impl/TestRdbFileDaoCleanupAt.java diff --git a/pixels-common/src/main/java/io/pixelsdb/pixels/common/metadata/MetadataService.java b/pixels-common/src/main/java/io/pixelsdb/pixels/common/metadata/MetadataService.java index f840c21ce7..f83d952e51 100644 --- a/pixels-common/src/main/java/io/pixelsdb/pixels/common/metadata/MetadataService.java +++ b/pixels-common/src/main/java/io/pixelsdb/pixels/common/metadata/MetadataService.java @@ -28,6 +28,7 @@ import io.pixelsdb.pixels.common.physical.Storage; import io.pixelsdb.pixels.common.server.HostAddress; import io.pixelsdb.pixels.common.utils.ConfigFactory; +import io.pixelsdb.pixels.common.utils.PixelsFileNameUtils; import io.pixelsdb.pixels.common.utils.ShutdownHookManager; import io.pixelsdb.pixels.daemon.MetadataProto; import io.pixelsdb.pixels.daemon.MetadataServiceGrpc; @@ -1429,16 +1430,56 @@ public File.Type getFileType(String filePathUri) throws MetadataException } /** - * Return query-visible REGULAR files under the path. + * Return query-visible {@link File.Type#REGULAR} files under the path. */ - public List getFiles(long pathId) throws MetadataException + public List getRegularFiles(long pathId) throws MetadataException { + return getFilesByType(pathId, EnumSet.of(File.Type.REGULAR)); + } + + /** + * Return files of the requested types, scoped to a single path. + */ + public List getFilesByType(long pathId, Set types) throws MetadataException + { + return invokeGetFilesByType(pathId, types, "get files by type"); + } + + /** + * Catalog-wide cross-path enumeration of the requested types. + */ + public List getFilesByType(Set types) throws MetadataException + { + return invokeGetFilesByType(null, types, "get files by type (cross-path)"); + } + + private List invokeGetFilesByType(Long pathId, Set types, String errorContext) + throws MetadataException + { + if (types == null || types.isEmpty()) + { + throw new IllegalArgumentException( + errorContext + ": 'types' must be non-null and non-empty"); + } String token = UUID.randomUUID().toString(); - MetadataProto.GetFilesRequest request = MetadataProto.GetFilesRequest.newBuilder() - .setHeader(MetadataProto.RequestHeader.newBuilder().setToken(token)).setPathId(pathId).build(); + MetadataProto.GetFilesByTypeRequest.Builder requestBuilder = + MetadataProto.GetFilesByTypeRequest.newBuilder() + .setHeader(MetadataProto.RequestHeader.newBuilder().setToken(token)); + if (pathId != null) + { + requestBuilder.setPathId(pathId); + } + for (File.Type type : types) + { + if (type != null) + { + requestBuilder.addFileTypesValue(type.getNumber()); + } + } + try { - MetadataProto.GetFilesResponse response = this.stub.getFiles(request); + MetadataProto.GetFilesByTypeResponse response = this.stub.getFilesByType(requestBuilder.build()); if (response.getHeader().getErrorCode() != 0) { throw new MetadataException("error code=" + response.getHeader().getErrorCode() @@ -1450,10 +1491,104 @@ public List getFiles(long pathId) throws MetadataException } return File.convertFiles(response.getFilesList()); } + catch (MetadataException e) + { + throw e; + } catch (Exception e) { - throw new MetadataException("failed to get files", e); + throw new MetadataException("failed to " + errorContext, e); + } + } + + /** + * Return temporary files (TEMPORARY_INGEST + TEMPORARY_GC) whose filename + * create time plus {@code ttlMs} is not later than now. + * + *

The create time is decoded from the {@code yyyyMMddHHmmss} timestamp in + * the file name. Files with unparsable names are logged and skipped. + * + *

For background sweepers only; not for query-visible callers. + * + * @param ttlMs temporary-file TTL in milliseconds. Must be {@code >= 0}. + */ + public List listTemporaryFilesDue(long ttlMs) throws MetadataException + { + if (ttlMs < 0) + { + throw new IllegalArgumentException("listTemporaryFilesDue: ttlMs must be >= 0, got " + ttlMs); + } + long now = System.currentTimeMillis(); + List all = getFilesByType( + EnumSet.of(File.Type.TEMPORARY_INGEST, File.Type.TEMPORARY_GC)); + List due = new ArrayList<>(all.size()); + int skippedParseFailure = 0; + for (File f : all) + { + OptionalLong createTime = PixelsFileNameUtils.extractCreateTimeMillis(f.getName()); + if (!createTime.isPresent()) + { + skippedParseFailure++; + logger.warn("listTemporaryFilesDue: cannot decode createTime from file name '{}' " + + "(id={}, pathId={}, type={}); skipping. event=sweep.parse_failure", + f.getName(), f.getId(), f.getPathId(), f.getType()); + continue; + } + if (createTime.getAsLong() + ttlMs <= now) + { + due.add(f); + } + } + if (skippedParseFailure > 0) + { + logger.warn("listTemporaryFilesDue: skipped {} temporary file(s) due to filename parse failure; " + + "investigate writer-side filename generation. event=sweep.parse_failure.summary", + skippedParseFailure); + } + // Oldest-first ordering for reproducible sweep batches. The createTime is already + // parsed once above, but the file list is small (sweep batch), so re-parsing here + // is acceptable and keeps the sort key self-contained. + due.sort(Comparator + .comparingLong((File f) -> PixelsFileNameUtils.extractCreateTimeMillis(f.getName()) + .orElse(Long.MAX_VALUE)) + .thenComparingLong(File::getId)); + return due; + } + + /** + * Return RETIRED files whose {@code cleanupAt} deadline has arrived. + */ + public List listRetiredFilesDue() throws MetadataException + { + long now = System.currentTimeMillis(); + List all = getFilesByType(EnumSet.of(File.Type.RETIRED)); + List due = new ArrayList<>(all.size()); + int skippedInvariantViolation = 0; + for (File f : all) + { + Long cleanupAt = f.getCleanupAt(); + if (cleanupAt == null) + { + skippedInvariantViolation++; + logger.warn("listRetiredFilesDue: RETIRED file '{}' (id={}, pathId={}) carries no cleanupAt; " + + "skipping. event=sweep.invariant_violation", + f.getName(), f.getId(), f.getPathId()); + continue; + } + if (cleanupAt <= now) + { + due.add(f); + } + } + if (skippedInvariantViolation > 0) + { + logger.warn("listRetiredFilesDue: skipped {} RETIRED file(s) missing cleanupAt; " + + "investigate DAO write path. event=sweep.invariant_violation.summary", + skippedInvariantViolation); } + due.sort(Comparator.comparingLong((File f) -> f.getCleanupAt()) + .thenComparingLong(File::getId)); + return due; } public boolean updateFile(File file) throws MetadataException diff --git a/pixels-common/src/main/java/io/pixelsdb/pixels/common/utils/DateUtil.java b/pixels-common/src/main/java/io/pixelsdb/pixels/common/utils/DateUtil.java index 39e2ae88c0..b9091c5ba0 100644 --- a/pixels-common/src/main/java/io/pixelsdb/pixels/common/utils/DateUtil.java +++ b/pixels-common/src/main/java/io/pixelsdb/pixels/common/utils/DateUtil.java @@ -22,6 +22,7 @@ import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.Date; +import java.util.TimeZone; import java.util.concurrent.atomic.AtomicInteger; /** @@ -49,6 +50,7 @@ public static String formatTime(Date time) public static String getCurTime() { SimpleDateFormat df = new SimpleDateFormat("yyyyMMddHHmmss");//set the style + df.setTimeZone(TimeZone.getTimeZone(ConfigFactory.Instance().getProperty("pxl.file.timestamp.zone"))); return df.format(new Date()) + "_" + count.getAndIncrement(); } } diff --git a/pixels-common/src/main/java/io/pixelsdb/pixels/common/utils/PixelsFileNameUtils.java b/pixels-common/src/main/java/io/pixelsdb/pixels/common/utils/PixelsFileNameUtils.java index 67586f7dd1..19209bbdc1 100644 --- a/pixels-common/src/main/java/io/pixelsdb/pixels/common/utils/PixelsFileNameUtils.java +++ b/pixels-common/src/main/java/io/pixelsdb/pixels/common/utils/PixelsFileNameUtils.java @@ -19,6 +19,11 @@ */ package io.pixelsdb.pixels.common.utils; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.OptionalLong; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -269,6 +274,34 @@ public static PxlFileType extractFileType(String path) return (m != null) ? PxlFileType.fromLabel(m.group(5)) : null; } + /** + * Extracts the embedded {@code yyyyMMddHHmmss} create time from a {@code .pxl} path. + * + * @param path absolute or relative file path + * @return {@code epoch-millis} of the embedded timestamp, or + * {@link OptionalLong#empty()} if {@code path} does not match the + * unified format or the timestamp segment fails to parse. + */ + public static OptionalLong extractCreateTimeMillis(String path) + { + Matcher m = match(path); + if (m == null) + { + return OptionalLong.empty(); + } + try + { + return OptionalLong.of(LocalDateTime.parse(m.group(2), DateTimeFormatter.ofPattern("yyyyMMddHHmmss")) + .atZone(ZoneId.of(ConfigFactory.Instance().getProperty("pxl.file.timestamp.zone"))) + .toInstant() + .toEpochMilli()); + } + catch (DateTimeParseException e) + { + return OptionalLong.empty(); + } + } + /** * Returns {@code true} if the file at {@code path} is eligible for Storage GC, * i.e. its type is one of {@link PxlFileType#ORDERED} or {@link PxlFileType#COMPACT}. diff --git a/pixels-common/src/main/resources/pixels.properties b/pixels-common/src/main/resources/pixels.properties index 700eb3f3d0..95587fb7c2 100644 --- a/pixels-common/src/main/resources/pixels.properties +++ b/pixels-common/src/main/resources/pixels.properties @@ -104,6 +104,8 @@ compression.block.size=1048576 compact.factor=32 # row batch size for pixels record reader, default value is 10000 row.batch.size=10000 +# time zone used to format and parse the yyyyMMddHHmmss segment in .pxl file names +pxl.file.timestamp.zone=Asia/Shanghai ### file storage and I/O ### # the scheme of the storage systems that are enabled, e.g., hdfs,file,s3,gcs,minio,redis,s3qs,httpstream diff --git a/pixels-common/src/test/java/io/pixelsdb/pixels/common/metadata/domain/TestFileDomain.java b/pixels-common/src/test/java/io/pixelsdb/pixels/common/metadata/domain/TestFileDomain.java index 5e66d66e29..3907948f18 100644 --- a/pixels-common/src/test/java/io/pixelsdb/pixels/common/metadata/domain/TestFileDomain.java +++ b/pixels-common/src/test/java/io/pixelsdb/pixels/common/metadata/domain/TestFileDomain.java @@ -239,9 +239,9 @@ public void roundTrip_domainProtoDomain_isLossless_forEveryType() { for (File.Type t : File.Type.values()) { - // RETIRED carries cleanupAt; the others should not. We deliberately set cleanupAt - // independently of type to verify the domain object preserves whatever it is given. - Long cleanup = (t == File.Type.RETIRED) ? 1_700_000_000_999L : null; + // The domain object preserves cleanupAt exactly as provided; lifecycle-specific + // invariants are enforced by callers that create or update catalog rows. + Long cleanup = (t == File.Type.REGULAR) ? null : 1_700_000_000_999L; File original = makeFile(7L, "x_" + t + ".pxl", t, 1, 0L, 63L, 3L, cleanup); File restored = new File(original.toProto()); diff --git a/pixels-common/src/test/java/io/pixelsdb/pixels/common/utils/TestPixelsFileNameUtils.java b/pixels-common/src/test/java/io/pixelsdb/pixels/common/utils/TestPixelsFileNameUtils.java new file mode 100644 index 0000000000..965b48f5d6 --- /dev/null +++ b/pixels-common/src/test/java/io/pixelsdb/pixels/common/utils/TestPixelsFileNameUtils.java @@ -0,0 +1,174 @@ +/* + * Copyright 2026 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the Affero GNU General Public + * License along with Pixels. If not, see + * . + */ +package io.pixelsdb.pixels.common.utils; + +import org.junit.Test; + +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.util.OptionalLong; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Tests timestamp extraction from {@code .pxl} file names. + */ +public class TestPixelsFileNameUtils +{ + private static final String PXL_FILE_TIMESTAMP_ZONE_KEY = "pxl.file.timestamp.zone"; + private static final String DEFAULT_PXL_FILE_TIMESTAMP_ZONE = "UTC"; + + @Test + public void extractCreateTimeMillis_decodesEmbeddedTimestampUsingConfiguredDefaultZone() + { + ConfigFactory.Instance().addProperty(PXL_FILE_TIMESTAMP_ZONE_KEY, + DEFAULT_PXL_FILE_TIMESTAMP_ZONE); + String name = "host_20260514071200_0_3_ordered.pxl"; + long expected = LocalDateTime.of(2026, 5, 14, 7, 12, 0) + .toInstant(ZoneOffset.UTC).toEpochMilli(); + + OptionalLong actual = PixelsFileNameUtils.extractCreateTimeMillis(name); + assertTrue("well-formed file name must decode", actual.isPresent()); + assertEquals(expected, actual.getAsLong()); + } + + @Test + public void extractCreateTimeMillis_honorsConfiguredTimestampZone() + { + ConfigFactory.Instance().addProperty(PXL_FILE_TIMESTAMP_ZONE_KEY, "Asia/Shanghai"); + try + { + String name = "host_20260514071200_0_3_ordered.pxl"; + long expected = LocalDateTime.of(2026, 5, 14, 7, 12, 0) + .atZone(ZoneId.of("Asia/Shanghai")).toInstant().toEpochMilli(); + + OptionalLong actual = PixelsFileNameUtils.extractCreateTimeMillis(name); + assertTrue("well-formed file name must decode", actual.isPresent()); + assertEquals(expected, actual.getAsLong()); + } + finally + { + ConfigFactory.Instance().addProperty(PXL_FILE_TIMESTAMP_ZONE_KEY, + DEFAULT_PXL_FILE_TIMESTAMP_ZONE); + } + } + + @Test + public void extractCreateTimeMillis_roundTripsThroughDateUtilGetCurTimeWithConfiguredZone() + { + ConfigFactory.Instance().addProperty(PXL_FILE_TIMESTAMP_ZONE_KEY, "Asia/Shanghai"); + try + { + long before = System.currentTimeMillis(); + String name = "host_" + DateUtil.getCurTime() + "_3_ordered.pxl"; + long after = System.currentTimeMillis(); + + OptionalLong decoded = PixelsFileNameUtils.extractCreateTimeMillis(name); + assertTrue("DateUtil-generated filename must decode", decoded.isPresent()); + + long beforeSec = (before / 1000L) * 1000L; + long afterSec = ((after / 1000L) + 1L) * 1000L; + assertTrue("decoded createTime " + decoded.getAsLong() + + " out of [" + beforeSec + ", " + afterSec + "]", + decoded.getAsLong() >= beforeSec && decoded.getAsLong() <= afterSec); + } + finally + { + ConfigFactory.Instance().addProperty(PXL_FILE_TIMESTAMP_ZONE_KEY, + DEFAULT_PXL_FILE_TIMESTAMP_ZONE); + } + } + + @Test + public void extractCreateTimeMillis_handlesAbsolutePathPrefix() + { + ConfigFactory.Instance().addProperty(PXL_FILE_TIMESTAMP_ZONE_KEY, + DEFAULT_PXL_FILE_TIMESTAMP_ZONE); + String path = "/data/p/host_20200101000000_42_-1_single.pxl"; + long expected = LocalDateTime.of(2020, 1, 1, 0, 0, 0) + .toInstant(ZoneOffset.UTC).toEpochMilli(); + + OptionalLong actual = PixelsFileNameUtils.extractCreateTimeMillis(path); + assertTrue(actual.isPresent()); + assertEquals(expected, actual.getAsLong()); + } + + @Test + public void extractCreateTimeMillis_handlesHostnameWithUnderscores() + { + ConfigFactory.Instance().addProperty(PXL_FILE_TIMESTAMP_ZONE_KEY, + DEFAULT_PXL_FILE_TIMESTAMP_ZONE); + // Host names may contain underscores before the timestamp. + String name = "retina_node_3_20260514071200_7_2_compact.pxl"; + long expected = LocalDateTime.of(2026, 5, 14, 7, 12, 0) + .toInstant(ZoneOffset.UTC).toEpochMilli(); + + OptionalLong actual = PixelsFileNameUtils.extractCreateTimeMillis(name); + assertTrue(actual.isPresent()); + assertEquals(expected, actual.getAsLong()); + } + + @Test + public void extractCreateTimeMillis_returnsEmptyOnUnrecognisedFormat() + { + assertFalse(PixelsFileNameUtils.extractCreateTimeMillis(null).isPresent()); + assertFalse(PixelsFileNameUtils.extractCreateTimeMillis("").isPresent()); + assertFalse(PixelsFileNameUtils.extractCreateTimeMillis("random.txt").isPresent()); + // Unknown file type label. + assertFalse(PixelsFileNameUtils.extractCreateTimeMillis( + "host_20260514071200_0_3_unknown.pxl").isPresent()); + // Timestamp must be exactly 14 digits. + assertFalse(PixelsFileNameUtils.extractCreateTimeMillis( + "host_2026051407120_0_3_ordered.pxl").isPresent()); + } + + @Test + public void extractCreateTimeMillis_returnsEmptyOnStructurallyInvalidTimestamp() + { + // Structurally valid name with an invalid timestamp. + OptionalLong actual = PixelsFileNameUtils.extractCreateTimeMillis( + "host_20261314071200_0_3_ordered.pxl"); + assertFalse(actual.isPresent()); + } + + @Test + public void extractCreateTimeMillis_roundTripsThroughDateUtilGetCurTime() + { + ConfigFactory.Instance().addProperty(PXL_FILE_TIMESTAMP_ZONE_KEY, + DEFAULT_PXL_FILE_TIMESTAMP_ZONE); + // DateUtil.getCurTime() should produce a decodable filename timestamp. + long before = System.currentTimeMillis(); + String name = "host_" + DateUtil.getCurTime() + "_3_ordered.pxl"; + long after = System.currentTimeMillis(); + + OptionalLong decoded = PixelsFileNameUtils.extractCreateTimeMillis(name); + assertTrue("DateUtil-generated filename must decode", decoded.isPresent()); + + // Decoded timestamp has second-level precision. + long beforeSec = (before / 1000L) * 1000L; + long afterSec = ((after / 1000L) + 1L) * 1000L; + assertTrue("decoded createTime " + decoded.getAsLong() + + " out of [" + beforeSec + ", " + afterSec + "]", + decoded.getAsLong() >= beforeSec && decoded.getAsLong() <= afterSec); + } +} diff --git a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/cache/CacheCoordinator.java b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/cache/CacheCoordinator.java index e355e9021e..4b311dd741 100644 --- a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/cache/CacheCoordinator.java +++ b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/cache/CacheCoordinator.java @@ -297,7 +297,7 @@ private List select(Layout layout) throws MetadataException // Issue #723: files are managed in metadata, do not get file paths from storage. for (Path compactPath : compactPaths) { - this.metadataService.getFiles(compactPath.getId()).forEach( + this.metadataService.getRegularFiles(compactPath.getId()).forEach( file -> filePaths.add(File.getFilePath(compactPath, file))); } return filePaths; diff --git a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/MetadataServiceImpl.java b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/MetadataServiceImpl.java index 5b65dd637e..09bad50998 100644 --- a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/MetadataServiceImpl.java +++ b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/MetadataServiceImpl.java @@ -1351,14 +1351,18 @@ public void addFiles(MetadataProto.AddFilesRequest request, } @Override - public void getFiles(MetadataProto.GetFilesRequest request, - StreamObserver responseObserver) + public void getFilesByType(MetadataProto.GetFilesByTypeRequest request, + StreamObserver responseObserver) { + // pathId is optional; absent means scanning across paths. MetadataProto.ResponseHeader.Builder headerBuilder = MetadataProto.ResponseHeader.newBuilder() .setToken(request.getHeader().getToken()); - MetadataProto.GetFilesResponse.Builder responseBuilder = MetadataProto.GetFilesResponse.newBuilder(); - List files = this.fileDao.getAllByPathId(request.getPathId()); + MetadataProto.GetFilesByTypeResponse.Builder responseBuilder = + MetadataProto.GetFilesByTypeResponse.newBuilder(); + Long pathId = request.hasPathId() ? request.getPathId() : null; + List files = + this.fileDao.getFilesByType(pathId, request.getFileTypesList()); if (files != null) { headerBuilder.setErrorCode(SUCCESS).setErrorMsg(""); @@ -1366,7 +1370,7 @@ public void getFiles(MetadataProto.GetFilesRequest request, } else { - headerBuilder.setErrorCode(METADATA_GET_FILES_FAILED).setErrorMsg("get files by path id failed"); + headerBuilder.setErrorCode(METADATA_GET_FILES_FAILED).setErrorMsg("get files by type failed"); responseBuilder.setHeader(headerBuilder); } diff --git a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/dao/FileDao.java b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/dao/FileDao.java index d400256e7a..844fcf34e2 100644 --- a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/dao/FileDao.java +++ b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/dao/FileDao.java @@ -39,9 +39,12 @@ public List getAll() } /** - * Return query-visible REGULAR files under a path. + * Return files of the requested types. + * + * @param pathId path scope, or {@code null} for all paths + * @param types file types to include; null or empty returns no files */ - public abstract List getAllByPathId(long pathId); + public abstract List getFilesByType(Long pathId, List types); public abstract MetadataProto.File getByPathIdAndFileName(long pathId, String fileName); diff --git a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/dao/impl/RdbFileDao.java b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/dao/impl/RdbFileDao.java index 8db4d04783..6c1af4c24a 100644 --- a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/dao/impl/RdbFileDao.java +++ b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/dao/impl/RdbFileDao.java @@ -27,6 +27,8 @@ import java.sql.*; import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashSet; import java.util.List; import java.util.stream.Collectors; @@ -60,14 +62,32 @@ private static MetadataProto.File buildFile(ResultSet rs) throws SQLException return builder.build(); } + /** + * Bind {@code FILE_CLEANUP_AT} for a file row. + * + *

{@code RETIRED} files must carry a cleanup deadline; other types must not. + */ private static void setCleanupAt(PreparedStatement pst, int index, MetadataProto.File file) throws SQLException { - if (file.getTypeValue() == MetadataProto.File.Type.RETIRED.getNumber() && file.hasCleanupAt()) + if (file.getTypeValue() == MetadataProto.File.Type.RETIRED.getNumber()) { + if (!file.hasCleanupAt()) + { + throw new SQLException("FILES row invariant violated: RETIRED file '" + + file.getName() + "' (id=" + file.getId() + + ") must carry a non-null FILE_CLEANUP_AT"); + } pst.setLong(index, file.getCleanupAt()); } else { + if (file.hasCleanupAt()) + { + throw new SQLException("FILES row invariant violated: non-RETIRED file '" + + file.getName() + "' (id=" + file.getId() + + ", type=" + file.getType() + + ") must NOT carry FILE_CLEANUP_AT (got " + file.getCleanupAt() + ")"); + } pst.setNull(index, Types.BIGINT); } } @@ -76,9 +96,9 @@ private static void setCleanupAt(PreparedStatement pst, int index, MetadataProto public MetadataProto.File getById(long id) { Connection conn = db.getConnection(); - try (Statement st = conn.createStatement()) + try (Statement st = conn.createStatement(); + ResultSet rs = st.executeQuery("SELECT * FROM FILES WHERE FILE_ID=" + id)) { - ResultSet rs = st.executeQuery("SELECT * FROM FILES WHERE FILE_ID=" + id); if (rs.next()) { return buildFile(rs); @@ -92,25 +112,59 @@ public MetadataProto.File getById(long id) } @Override - public List getAllByPathId(long pathId) + public List getFilesByType(Long pathId, List types) { + if (types == null || types.isEmpty()) + { + return Collections.emptyList(); + } + // De-duplicate while preserving insertion order so the SQL bind order is stable. + LinkedHashSet typeNumbers = new LinkedHashSet<>(); + for (MetadataProto.File.Type type : types) + { + if (type != null) + { + typeNumbers.add(type.getNumber()); + } + } + if (typeNumbers.isEmpty()) + { + return Collections.emptyList(); + } + + StringBuilder sql = new StringBuilder("SELECT * FROM FILES WHERE "); + if (pathId != null) + { + sql.append("PATHS_PATH_ID = ? AND "); + } + sql.append("FILE_TYPE IN (") + .append(String.join(",", Collections.nCopies(typeNumbers.size(), "?"))) + .append(") ORDER BY FILE_ID"); + Connection conn = db.getConnection(); - String sql = "SELECT * FROM FILES WHERE FILE_TYPE = ? AND PATHS_PATH_ID = ?"; - try (PreparedStatement st = conn.prepareStatement(sql)) + try (PreparedStatement pst = conn.prepareStatement(sql.toString())) { - // Query-visible file enumeration is REGULAR-only. - st.setInt(1, MetadataProto.File.Type.REGULAR.getNumber()); - st.setLong(2, pathId); - ResultSet rs = st.executeQuery(); - List files = new ArrayList<>(); - while (rs.next()) + int index = 1; + if (pathId != null) { - files.add(buildFile(rs)); + pst.setLong(index++, pathId); + } + for (Integer number : typeNumbers) + { + pst.setInt(index++, number); + } + try (ResultSet rs = pst.executeQuery()) + { + List files = new ArrayList<>(); + while (rs.next()) + { + files.add(buildFile(rs)); + } + return files; } - return files; } catch (SQLException e) { - log.error("getAllByPathId in RdbFileDao", e); + log.error("getFilesByType in RdbFileDao", e); } return null; @@ -125,10 +179,12 @@ public MetadataProto.File getByPathIdAndFileName(long pathId, String fileName) { st.setLong(1, pathId); st.setString(2, fileName); - ResultSet rs = st.executeQuery(); - if (rs.next()) + try (ResultSet rs = st.executeQuery()) { - return buildFile(rs); + if (rs.next()) + { + return buildFile(rs); + } } } catch (SQLException e) { @@ -145,10 +201,12 @@ public boolean exists(MetadataProto.File file) try (Statement st = conn.createStatement()) { String sql = "SELECT 1 FROM FILES WHERE FILE_ID=" + file.getId(); - ResultSet rs = st.executeQuery(sql); - if (rs.next()) + try (ResultSet rs = st.executeQuery(sql)) { - return true; + if (rs.next()) + { + return true; + } } } catch (SQLException e) { @@ -181,14 +239,16 @@ public long insert(MetadataProto.File file) setCleanupAt(pst, 7, file); if (pst.executeUpdate() == 1) { - ResultSet rs = pst.executeQuery("SELECT LAST_INSERT_ID()"); - if (rs.next()) - { - return rs.getLong(1); - } - else + try (ResultSet rs = pst.executeQuery("SELECT LAST_INSERT_ID()")) { - return -1; + if (rs.next()) + { + return rs.getLong(1); + } + else + { + return -1; + } } } else diff --git a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/retina/RetinaServerImpl.java b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/retina/RetinaServerImpl.java index d2763698b9..c9be09752c 100644 --- a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/retina/RetinaServerImpl.java +++ b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/retina/RetinaServerImpl.java @@ -132,7 +132,7 @@ private void initializeRetinaResources() throws Exception // add visibility for ordered files List orderedPaths = layout.getOrderedPaths(); validateOrderedOrCompactPaths(orderedPaths); - List orderedFiles = this.metadataService.getFiles(orderedPaths.get(0).getId()); + List orderedFiles = this.metadataService.getRegularFiles(orderedPaths.get(0).getId()); files.addAll(orderedFiles.stream() .map(file -> orderedPaths.get(0).getUri() + "/" + file.getName()) .collect(Collectors.toList())); @@ -140,7 +140,7 @@ private void initializeRetinaResources() throws Exception // add visibility for compact files List compactPaths = layout.getCompactPaths(); validateOrderedOrCompactPaths(compactPaths); - List compactFiles = this.metadataService.getFiles(compactPaths.get(0).getId()); + List compactFiles = this.metadataService.getRegularFiles(compactPaths.get(0).getId()); files.addAll(compactFiles.stream() .map(file -> compactPaths.get(0).getUri() + "/" + file.getName()) .collect(Collectors.toList())); diff --git a/pixels-daemon/src/test/java/io/pixelsdb/pixels/daemon/metadata/dao/TestRdbFileDao.java b/pixels-daemon/src/test/java/io/pixelsdb/pixels/daemon/metadata/dao/TestRdbFileDao.java new file mode 100644 index 0000000000..984516c958 --- /dev/null +++ b/pixels-daemon/src/test/java/io/pixelsdb/pixels/daemon/metadata/dao/TestRdbFileDao.java @@ -0,0 +1,581 @@ +/* + * Copyright 2026 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the Affero GNU General Public + * License along with Pixels. If not, see + * . + */ +package io.pixelsdb.pixels.daemon.metadata.dao; + +import io.pixelsdb.pixels.common.utils.MetaDBUtil; +import io.pixelsdb.pixels.daemon.MetadataProto; +import io.pixelsdb.pixels.daemon.metadata.dao.impl.RdbFileDao; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; + +import java.lang.reflect.Field; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Types; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Unit tests for {@link RdbFileDao} cleanup-at handling and typed file enumeration. + */ +public class TestRdbFileDao +{ + private static final MetadataProto.File.Type REGULAR = MetadataProto.File.Type.REGULAR; + private static final MetadataProto.File.Type RETIRED = MetadataProto.File.Type.RETIRED; + private static final MetadataProto.File.Type TEMPORARY_INGEST = + MetadataProto.File.Type.TEMPORARY_INGEST; + private static final MetadataProto.File.Type TEMPORARY_GC = + MetadataProto.File.Type.TEMPORARY_GC; + + private static final int REGULAR_VALUE = REGULAR.getNumber(); + private static final int RETIRED_VALUE = RETIRED.getNumber(); + private static final int TEMPORARY_INGEST_VALUE = TEMPORARY_INGEST.getNumber(); + private static final int TEMPORARY_GC_VALUE = TEMPORARY_GC.getNumber(); + + private Connection mockConn; + private Connection originalConn; + private RdbFileDao dao; + + @Before + public void setUp() throws Exception + { + mockConn = mock(Connection.class); + // Keep lazy reconnect on the mock connection. + when(mockConn.isValid(anyInt())).thenReturn(true); + originalConn = swapConnection(mockConn); + dao = new RdbFileDao(); + } + + @After + public void tearDown() throws Exception + { + swapConnection(originalConn); + } + + // ========================================================================= + // INSERT / UPDATE cleanup-at binding + // ========================================================================= + + /** + * Non-RETIRED rows bind {@code FILE_CLEANUP_AT} as {@code NULL}. + */ + @Test + public void insert_nonRetired_withoutCleanupAt_bindsNull() throws Exception + { + PreparedStatement pstRegular = stubPreparedStatementForInsert(); + dao.insert(baseFile("a.pxl", REGULAR_VALUE).build()); + verify(pstRegular).setNull(7, Types.BIGINT); + verify(pstRegular, never()).setLong(eq(7), anyLong()); + + PreparedStatement pstIngest = stubPreparedStatementForInsert(); + dao.insert(baseFile("ingest_unset.pxl", TEMPORARY_INGEST_VALUE).build()); + verify(pstIngest).setNull(7, Types.BIGINT); + verify(pstIngest, never()).setLong(eq(7), anyLong()); + + PreparedStatement pstGc = stubPreparedStatementForInsert(); + dao.insert(baseFile("gc_unset.pxl", TEMPORARY_GC_VALUE).build()); + verify(pstGc).setNull(7, Types.BIGINT); + verify(pstGc, never()).setLong(eq(7), anyLong()); + } + + /** + * Non-RETIRED rows with {@code cleanupAt} are rejected before writing. + */ + @Test + public void insert_nonRetired_withCleanupAt_failsFast() throws Exception + { + PreparedStatement pst = stubPreparedStatementForInsert(); + long unwanted = 123_456_789L; + long id = dao.insert(baseFile("a.pxl", REGULAR_VALUE).setCleanupAt(unwanted).build()); + assertEquals("DAO must surface the invariant violation as the -1 failure sentinel", -1L, id); + verify(pst, never()).setLong(eq(7), anyLong()); + verify(pst, never()).setNull(eq(7), anyInt()); + verify(pst, never()).executeUpdate(); + + PreparedStatement pst2 = stubPreparedStatementForInsert(); + long id2 = dao.insert(baseFile("t.pxl", TEMPORARY_GC_VALUE).setCleanupAt(24L).build()); + assertEquals(-1L, id2); + verify(pst2, never()).executeUpdate(); + } + + /** + * RETIRED rows bind the provided cleanup deadline. + */ + @Test + public void insert_retiredFile_bindingScenarios() throws Exception + { + PreparedStatement pst1 = stubPreparedStatementForInsert(); + long deadline = 1_700_000_000_000L; + dao.insert(baseFile("retired.pxl", RETIRED_VALUE).setCleanupAt(deadline).build()); + verify(pst1).setLong(7, deadline); + verify(pst1, never()).setNull(eq(7), anyInt()); + + PreparedStatement pst2 = stubPreparedStatementForInsert(); + dao.insert(baseFile("retired_zero.pxl", RETIRED_VALUE).setCleanupAt(0L).build()); + verify(pst2).setLong(7, 0L); + verify(pst2, never()).setNull(eq(7), anyInt()); + } + + /** + * RETIRED rows without {@code cleanupAt} are rejected. + */ + @Test + public void insert_retired_withoutCleanupAt_failsFast() throws Exception + { + PreparedStatement pst = stubPreparedStatementForInsert(); + long id = dao.insert(baseFile("nd.pxl", RETIRED_VALUE).build()); + assertEquals(-1L, id); + verify(pst, never()).executeUpdate(); + } + + @Test + public void insertBatch_mixedTypes_bindsCleanupAtPerRow() throws Exception + { + PreparedStatement pst = stubPreparedStatementForInsert(); + + MetadataProto.File regular = baseFile("r.pxl", REGULAR_VALUE).build(); + MetadataProto.File temporaryNoDeadline = baseFile("t.pxl", TEMPORARY_GC_VALUE).build(); + MetadataProto.File ingestNoDeadline = baseFile("i.pxl", TEMPORARY_INGEST_VALUE).build(); + MetadataProto.File retiredWithDeadline = baseFile("d.pxl", RETIRED_VALUE) + .setCleanupAt(42L).build(); + + assertTrue(dao.insertBatch( + Arrays.asList(regular, temporaryNoDeadline, ingestNoDeadline, retiredWithDeadline))); + + // Three non-RETIRED rows bind NULL; the single RETIRED row binds its deadline. + verify(pst, times(3)).setNull(7, Types.BIGINT); + verify(pst, times(1)).setLong(7, 42L); + verify(pst).executeBatch(); + } + + /** + * Any invalid cleanup-at row rejects the whole batch. + */ + @Test + public void insertBatch_invariantViolation_rejectsWholeBatch() throws Exception + { + PreparedStatement pst = stubPreparedStatementForInsert(); + + // Mix one legal RETIRED with one illegal TEMPORARY_GC+cleanupAt. + MetadataProto.File legal = baseFile("d.pxl", RETIRED_VALUE).setCleanupAt(42L).build(); + MetadataProto.File illegal = baseFile("t.pxl", TEMPORARY_GC_VALUE).setCleanupAt(24L).build(); + + assertFalse(dao.insertBatch(Arrays.asList(legal, illegal))); + verify(pst, never()).executeBatch(); + } + + /** + * UPDATE binds cleanup-at at index 6 and the WHERE id at index 7. + */ + @Test + public void update_bindingScenarios() throws Exception + { + PreparedStatement pst1 = mock(PreparedStatement.class); + when(mockConn.prepareStatement(anyString())).thenReturn(pst1); + when(pst1.executeUpdate()).thenReturn(1); + assertTrue(dao.update(baseFile("u.pxl", REGULAR_VALUE).setId(7L).build())); + verify(pst1).setNull(6, Types.BIGINT); + verify(pst1).setLong(7, 7L); + + PreparedStatement pst2 = mock(PreparedStatement.class); + when(mockConn.prepareStatement(anyString())).thenReturn(pst2); + when(pst2.executeUpdate()).thenReturn(1); + long deadline = 1_700_000_000_999L; + assertTrue(dao.update(baseFile("u.pxl", RETIRED_VALUE).setId(8L) + .setCleanupAt(deadline).build())); + verify(pst2).setLong(6, deadline); + verify(pst2).setLong(7, 8L); + } + + /** + * Invalid cleanup-at combinations are rejected on UPDATE. + */ + @Test + public void update_invariantViolations_failFast() throws Exception + { + PreparedStatement pst1 = mock(PreparedStatement.class); + when(mockConn.prepareStatement(anyString())).thenReturn(pst1); + assertFalse(dao.update(baseFile("u.pxl", TEMPORARY_GC_VALUE).setId(8L) + .setCleanupAt(99L).build())); + verify(pst1, never()).executeUpdate(); + + PreparedStatement pst2 = mock(PreparedStatement.class); + when(mockConn.prepareStatement(anyString())).thenReturn(pst2); + assertFalse(dao.update(baseFile("u.pxl", RETIRED_VALUE).setId(9L).build())); + verify(pst2, never()).executeUpdate(); + } + + // ========================================================================= + // atomicSwapFiles transactional behaviour + // ========================================================================= + + /** + * Promoting a file clears {@code FILE_CLEANUP_AT} with the type update. + */ + @Test + public void atomicSwapFiles_promoteSqlClearsCleanupAt() throws Exception + { + PreparedStatement updatePst = mock(PreparedStatement.class); + PreparedStatement deletePst = mock(PreparedStatement.class); + when(mockConn.prepareStatement(anyString())).thenAnswer(inv -> { + String sql = inv.getArgument(0); + if (sql.startsWith("UPDATE")) return updatePst; + if (sql.startsWith("DELETE")) return deletePst; + return mock(PreparedStatement.class); + }); + + assertTrue(dao.atomicSwapFiles(101L, Arrays.asList(11L, 12L))); + + ArgumentCaptor sqlCaptor = ArgumentCaptor.forClass(String.class); + verify(mockConn, atLeastOnce()).prepareStatement(sqlCaptor.capture()); + boolean clearsCleanupAt = false; + for (String sql : sqlCaptor.getAllValues()) + { + if (sql.contains("FILE_TYPE=?") && sql.contains("FILE_CLEANUP_AT=NULL")) + { + clearsCleanupAt = true; + break; + } + } + assertTrue("promote SQL must clear FILE_CLEANUP_AT to NULL together with the type update", + clearsCleanupAt); + + verify(updatePst).setInt(1, REGULAR_VALUE); + verify(updatePst).setLong(2, 101L); + verify(updatePst).executeUpdate(); + verify(deletePst).setLong(1, 11L); + verify(deletePst).setLong(2, 12L); + verify(deletePst).executeUpdate(); + verify(mockConn).setAutoCommit(false); + verify(mockConn).commit(); + } + + @Test + public void atomicSwapFiles_rollsBackOnSqlException() throws Exception + { + when(mockConn.prepareStatement(anyString())).thenThrow(new SQLException("boom")); + + assertFalse("atomicSwapFiles must report failure when the JDBC layer throws", + dao.atomicSwapFiles(1L, Collections.singletonList(2L))); + verify(mockConn).setAutoCommit(false); + verify(mockConn).rollback(); + verify(mockConn).setAutoCommit(true); + verify(mockConn, never()).commit(); + } + + // ========================================================================= + // SELECT cleanup-at round-trip + // ========================================================================= + + /** + * SQL {@code NULL} cleanup-at values surface as unset proto fields. + */ + @Test + public void getById_cleanupAtRoundTripScenarios() throws Exception + { + // Scenario 1: non-NULL deadline must surface as hasCleanupAt() == true + Statement st1 = mock(Statement.class); + ResultSet rs1 = mock(ResultSet.class); + when(mockConn.createStatement()).thenReturn(st1); + when(st1.executeQuery(anyString())).thenReturn(rs1); + when(rs1.next()).thenReturn(true).thenReturn(false); + stubFileRow(rs1, 99L, "x.pxl", RETIRED_VALUE, 5L, 1_700_000_000_000L, /*wasNull*/ false); + + MetadataProto.File proto1 = dao.getById(99L); + assertNotNull(proto1); + assertEquals(99L, proto1.getId()); + assertEquals(RETIRED, proto1.getType()); + assertTrue("non-NULL FILE_CLEANUP_AT column must surface as hasCleanupAt()", + proto1.hasCleanupAt()); + assertEquals(1_700_000_000_000L, proto1.getCleanupAt()); + + // Scenario 2: NULL column must surface as !hasCleanupAt() + Statement st2 = mock(Statement.class); + ResultSet rs2 = mock(ResultSet.class); + when(mockConn.createStatement()).thenReturn(st2); + when(st2.executeQuery(anyString())).thenReturn(rs2); + when(rs2.next()).thenReturn(true).thenReturn(false); + stubFileRow(rs2, 1L, "r.pxl", REGULAR_VALUE, 1L, 0L, /*wasNull*/ true); + + MetadataProto.File proto2 = dao.getById(1L); + assertNotNull(proto2); + assertFalse("NULL FILE_CLEANUP_AT column must surface as !hasCleanupAt()", + proto2.hasCleanupAt()); + } + + // ========================================================================= + // getFilesByType + // ========================================================================= + + /** + * Single-path queries bind path id first, then requested file types. + */ + @Test + public void getFilesByType_singlePath_bindsPathIdAndRequestedTypes() throws Exception + { + PreparedStatement pst = stubEmptyQuery(); + + dao.getFilesByType(9L, Arrays.asList(TEMPORARY_INGEST, RETIRED)); + + ArgumentCaptor sqlCaptor = ArgumentCaptor.forClass(String.class); + verify(mockConn).prepareStatement(sqlCaptor.capture()); + String sql = sqlCaptor.getValue(); + assertTrue("single-path enumeration must filter by PATHS_PATH_ID", + sql.contains("PATHS_PATH_ID = ?")); + assertTrue("enumeration must filter by FILE_TYPE IN (...)", + sql.contains("FILE_TYPE IN (")); + assertTrue("enumeration must order by FILE_ID for stable iteration", + sql.contains("ORDER BY FILE_ID")); + + verify(pst).setLong(1, 9L); + verify(pst).setInt(2, TEMPORARY_INGEST_VALUE); + verify(pst).setInt(3, RETIRED_VALUE); + } + + /** + * Cross-path queries omit the path predicate and bind types from index 1. + */ + @Test + public void getFilesByType_crossPath_omitsPathPredicateAndBindsTypesAtIndexOne() + throws Exception + { + PreparedStatement pst = stubEmptyQuery(); + + dao.getFilesByType(/*pathId*/ null, Arrays.asList(TEMPORARY_INGEST, TEMPORARY_GC)); + + ArgumentCaptor sqlCaptor = ArgumentCaptor.forClass(String.class); + verify(mockConn).prepareStatement(sqlCaptor.capture()); + String sql = sqlCaptor.getValue(); + assertFalse("cross-path enumeration must NOT include the PATHS_PATH_ID predicate", + sql.contains("PATHS_PATH_ID")); + assertTrue("cross-path enumeration must still filter by FILE_TYPE IN (...)", + sql.contains("FILE_TYPE IN (")); + assertTrue("cross-path enumeration must order by FILE_ID", + sql.contains("ORDER BY FILE_ID")); + + // No path bind — type numbers start at index 1. + verify(pst, never()).setLong(eq(1), anyLong()); + verify(pst).setInt(1, TEMPORARY_INGEST_VALUE); + verify(pst).setInt(2, TEMPORARY_GC_VALUE); + } + + /** + * Repeated file types share one SQL placeholder. + */ + @Test + public void getFilesByType_dedupesRepeatedTypes() throws Exception + { + PreparedStatement pst = stubEmptyQuery(); + + dao.getFilesByType(2L, Arrays.asList(REGULAR, REGULAR, REGULAR)); + + ArgumentCaptor sqlCaptor = ArgumentCaptor.forClass(String.class); + verify(mockConn).prepareStatement(sqlCaptor.capture()); + String sql = sqlCaptor.getValue(); + int inStart = sql.indexOf("FILE_TYPE IN ("); + int inEnd = sql.indexOf(")", inStart); + String inClause = sql.substring(inStart, inEnd); + assertEquals("duplicate types must be deduped to a single placeholder", + 1, countOccurrences(inClause, '?')); + + verify(pst).setLong(1, 2L); + verify(pst).setInt(2, REGULAR_VALUE); + verify(pst, never()).setInt(eq(3), anyInt()); + } + + /** + * Empty or null type lists return an empty result without querying JDBC. + */ + @Test + public void getFilesByType_emptyTypes_returnsEmptyWithoutQuerying() throws Exception + { + // Single-path empty / null + List emptyResult = dao.getFilesByType(5L, Collections.emptyList()); + assertNotNull(emptyResult); + assertTrue(emptyResult.isEmpty()); + + List nullResult = dao.getFilesByType(5L, null); + assertNotNull(nullResult); + assertTrue(nullResult.isEmpty()); + + // Cross-path empty / null + List crossEmpty = dao.getFilesByType(null, Collections.emptyList()); + assertNotNull(crossEmpty); + assertTrue(crossEmpty.isEmpty()); + + List crossNull = dao.getFilesByType(null, null); + assertNotNull(crossNull); + assertTrue(crossNull.isEmpty()); + + verify(mockConn, never()).prepareStatement(anyString()); + verify(mockConn, never()).createStatement(); + } + + /** + * SQL exceptions return {@code null} on single-path queries. + */ + @Test + public void getFilesByType_singlePath_sqlException_returnsNull() throws Exception + { + when(mockConn.prepareStatement(anyString())).thenThrow(new SQLException("boom")); + + List failure = + dao.getFilesByType(7L, Collections.singletonList(REGULAR)); + assertNull("SQL exception on single-path enumeration must surface as null", failure); + } + + /** + * SQL exceptions return {@code null} on cross-path queries. + */ + @Test + public void getFilesByType_crossPath_sqlException_returnsNull() throws Exception + { + when(mockConn.prepareStatement(anyString())).thenThrow(new SQLException("boom")); + + List failure = + dao.getFilesByType(null, Collections.singletonList(RETIRED)); + assertNull("SQL exception on cross-path enumeration must surface as null", failure); + } + + // ========================================================================= + // deleteByIds + // ========================================================================= + + /** + * deleteByIds batches {@code FILE_ID} deletes with one SQL template. + */ + @Test + public void deleteByIds_batchesBindsAndIssuesSingleSqlTemplate() throws Exception + { + PreparedStatement pst = mock(PreparedStatement.class); + when(mockConn.prepareStatement(anyString())).thenReturn(pst); + + assertTrue(dao.deleteByIds(Arrays.asList(11L, 22L, 33L))); + + ArgumentCaptor sqlCaptor = ArgumentCaptor.forClass(String.class); + verify(mockConn).prepareStatement(sqlCaptor.capture()); + String sql = sqlCaptor.getValue(); + assertEquals("deleteByIds must use a positional FILE_ID=? template (batched)", + "DELETE FROM FILES WHERE FILE_ID=?", sql); + + verify(pst).setLong(1, 11L); + verify(pst).setLong(1, 22L); + verify(pst).setLong(1, 33L); + verify(pst, times(3)).addBatch(); + verify(pst).executeBatch(); + } + + // ========================================================================= + // helpers + // ========================================================================= + + private PreparedStatement stubPreparedStatementForInsert() throws SQLException + { + PreparedStatement pst = mock(PreparedStatement.class); + when(mockConn.prepareStatement(anyString())).thenReturn(pst); + when(pst.executeUpdate()).thenReturn(1); + // Stub LAST_INSERT_ID() on the insert statement. + ResultSet idRs = mock(ResultSet.class); + when(pst.executeQuery(anyString())).thenReturn(idRs); + when(idRs.next()).thenReturn(true); + when(idRs.getLong(1)).thenReturn(1L); + return pst; + } + + private PreparedStatement stubEmptyQuery() throws SQLException + { + PreparedStatement pst = mock(PreparedStatement.class); + ResultSet rs = mock(ResultSet.class); + when(mockConn.prepareStatement(anyString())).thenReturn(pst); + when(pst.executeQuery()).thenReturn(rs); + when(rs.next()).thenReturn(false); + return pst; + } + + private static MetadataProto.File.Builder baseFile(String name, int typeValue) + { + return MetadataProto.File.newBuilder() + .setName(name) + .setTypeValue(typeValue) + .setNumRowGroup(1) + .setMinRowId(0L) + .setMaxRowId(0L) + .setPathId(1L); + } + + private static void stubFileRow(ResultSet rs, long id, String name, int typeValue, + long pathId, long cleanupAt, boolean cleanupAtWasNull) + throws SQLException + { + when(rs.getLong("FILE_ID")).thenReturn(id); + when(rs.getString("FILE_NAME")).thenReturn(name); + when(rs.getInt("FILE_TYPE")).thenReturn(typeValue); + when(rs.getInt("FILE_NUM_RG")).thenReturn(1); + when(rs.getLong("FILE_MIN_ROW_ID")).thenReturn(0L); + when(rs.getLong("FILE_MAX_ROW_ID")).thenReturn(0L); + when(rs.getLong("PATHS_PATH_ID")).thenReturn(pathId); + when(rs.getLong("FILE_CLEANUP_AT")).thenReturn(cleanupAt); + when(rs.wasNull()).thenReturn(cleanupAtWasNull); + } + + private static int countOccurrences(String haystack, char needle) + { + int n = 0; + for (int i = 0; i < haystack.length(); i++) + { + if (haystack.charAt(i) == needle) n++; + } + return n; + } + + /** + * Swap the {@link MetaDBUtil} singleton connection for this test. + */ + private static Connection swapConnection(Connection replacement) throws Exception + { + Field f = MetaDBUtil.class.getDeclaredField("connection"); + f.setAccessible(true); + Connection previous = (Connection) f.get(MetaDBUtil.Instance()); + f.set(MetaDBUtil.Instance(), replacement); + return previous; + } +} diff --git a/pixels-daemon/src/test/java/io/pixelsdb/pixels/daemon/metadata/dao/impl/TestRdbFileDaoCleanupAt.java b/pixels-daemon/src/test/java/io/pixelsdb/pixels/daemon/metadata/dao/impl/TestRdbFileDaoCleanupAt.java deleted file mode 100644 index 47879c9afb..0000000000 --- a/pixels-daemon/src/test/java/io/pixelsdb/pixels/daemon/metadata/dao/impl/TestRdbFileDaoCleanupAt.java +++ /dev/null @@ -1,406 +0,0 @@ -/* - * Copyright 2026 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ -package io.pixelsdb.pixels.daemon.metadata.dao.impl; - -import io.pixelsdb.pixels.common.utils.MetaDBUtil; -import io.pixelsdb.pixels.daemon.MetadataProto; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.lang.reflect.Field; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.sql.Types; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.atLeastOnce; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -/** - * Mockito-based unit tests for the c01.1 changes in {@link RdbFileDao} that govern how the - * optional {@code FILE_CLEANUP_AT} column is persisted and restored. - * - *

Contract under test: - *

    - *
  • On INSERT / UPDATE: {@code FILE_CLEANUP_AT} is bound to a real {@code long} only when - * {@code type == RETIRED && hasCleanupAt()}. Every other combination must bind {@code NULL}.
    • - *
  • On SELECT: a {@code wasNull()} column on the result set must materialise as - * {@code !proto.hasCleanupAt()} on the wire.
    • - *
  • {@code atomicSwapFiles} must clear {@code FILE_CLEANUP_AT} (set to NULL) when it - * promotes the new file to {@code REGULAR}, otherwise stale deadlines would leak across - * the swap boundary.
    • - *
- * - *

The DAO calls {@code MetaDBUtil.Instance().getConnection()} on every method, so we - * inject a mock {@link Connection} into the singleton via reflection. This keeps the test - * a true unit test (no JDBC driver, no schema, no network). - * - * @author tdd-guide - * @create 2026-05-13 - */ -public class TestRdbFileDaoCleanupAt -{ - private static final int RETIRED_VALUE = MetadataProto.File.Type.RETIRED.getNumber(); - private static final int REGULAR_VALUE = MetadataProto.File.Type.REGULAR.getNumber(); - private static final int TEMPORARY_INGEST_VALUE = - MetadataProto.File.Type.TEMPORARY_INGEST.getNumber(); - - private Connection mockConn; - private Connection originalConn; - - private RdbFileDao dao; - - @Before - public void setUp() throws Exception - { - mockConn = mock(Connection.class); - // The DAO does conn.isValid(30) on lazy reconnect; force it to report healthy so the - // production code path stays on our mock rather than re-acquiring a real connection. - when(mockConn.isValid(anyInt())).thenReturn(true); - - originalConn = swapConnection(mockConn); - dao = new RdbFileDao(); - } - - @After - public void tearDown() throws Exception - { - // Always restore the real connection so subsequent tests in the same JVM are unaffected. - swapConnection(originalConn); - } - - // ------------------------------------------------------------------------- - // INSERT — single row - // ------------------------------------------------------------------------- - - /** - * For non-RETIRED file types (REGULAR, TEMPORARY_INGEST, TEMPORARY_GC), the DAO must NOT persist a cleanup deadline, - * even if a stray {@code cleanupAt} happens to be present on the proto. - */ - @Test - public void insert_nonRetiredFileTypes_bindCleanupAtAsNull() throws Exception - { - // Test REGULAR file with stray cleanupAt value - PreparedStatement pst1 = stubPreparedStatementForInsert(); - MetadataProto.File regularFile = baseFile("a.pxl", REGULAR_VALUE) - .setCleanupAt(123_456_789L) // deliberately stray; type != RETIRED so MUST be ignored - .build(); - dao.insert(regularFile); - verify(pst1).setNull(7, Types.BIGINT); - verify(pst1, never()).setLong(eq(7), anyLong()); - - // Test TEMPORARY_INGEST file (no cleanupAt) - PreparedStatement pst2 = stubPreparedStatementForInsert(); - MetadataProto.File ingestFile = baseFile("ingest.pxl", TEMPORARY_INGEST_VALUE).build(); - dao.insert(ingestFile); - verify(pst2).setNull(7, Types.BIGINT); - verify(pst2, never()).setLong(eq(7), anyLong()); - } - - /** - * RETIRED file binding tests covering various cleanupAt scenarios - */ - @Test - public void insert_retiredFile_bindingScenarios() throws Exception - { - // Test RETIRED file with cleanup deadline - PreparedStatement pst1 = stubPreparedStatementForInsert(); - long deadline = 1_700_000_000_000L; - MetadataProto.File retiredWithDeadline = baseFile("retired.pxl", RETIRED_VALUE) - .setCleanupAt(deadline) - .build(); - dao.insert(retiredWithDeadline); - verify(pst1).setLong(7, deadline); - verify(pst1, never()).setNull(eq(7), anyInt()); - - // Test RETIRED file without cleanupAt (should bind NULL) - PreparedStatement pst2 = stubPreparedStatementForInsert(); - MetadataProto.File retiredNoDeadline = baseFile("retired_unset.pxl", RETIRED_VALUE).build(); - dao.insert(retiredNoDeadline); - verify(pst2).setNull(7, Types.BIGINT); - verify(pst2, never()).setLong(eq(7), anyLong()); - - // Test RETIRED file with cleanupAt = 0L (should bind as long zero, not NULL) - PreparedStatement pst3 = stubPreparedStatementForInsert(); - MetadataProto.File retiredZero = baseFile("retired_zero.pxl", RETIRED_VALUE) - .setCleanupAt(0L) - .build(); - dao.insert(retiredZero); - verify(pst3).setLong(7, 0L); - verify(pst3, never()).setNull(eq(7), anyInt()); - } - - // ------------------------------------------------------------------------- - // INSERT BATCH — verifies per-row binding semantics - // ------------------------------------------------------------------------- - - @Test - public void insertBatch_mixedTypes_bindsCleanupAtPerRow() throws Exception - { - PreparedStatement pst = stubPreparedStatementForInsert(); - - MetadataProto.File regular = baseFile("r.pxl", REGULAR_VALUE).build(); - MetadataProto.File retiredWithDeadline = baseFile("d.pxl", RETIRED_VALUE) - .setCleanupAt(42L).build(); - MetadataProto.File retiredNoDeadline = baseFile("nd.pxl", RETIRED_VALUE).build(); - - dao.insertBatch(Arrays.asList(regular, retiredWithDeadline, retiredNoDeadline)); - - // Two rows must bind NULL (regular + retired-without-deadline), one row binds a long. - verify(pst, times(2)).setNull(7, Types.BIGINT); - verify(pst, times(1)).setLong(7, 42L); - verify(pst).executeBatch(); - } - - // ------------------------------------------------------------------------- - // UPDATE — index 6 carries cleanupAt (id is bound at index 7) - // ------------------------------------------------------------------------- - - /** - * UPDATE operation binding tests for different file types and cleanupAt scenarios - */ - @Test - public void update_bindingScenarios() throws Exception - { - // Test REGULAR file - should bind cleanupAt as NULL - PreparedStatement pst1 = mock(PreparedStatement.class); - when(mockConn.prepareStatement(anyString())).thenReturn(pst1); - when(pst1.executeUpdate()).thenReturn(1); - - MetadataProto.File regularFile = baseFile("u.pxl", REGULAR_VALUE).setId(7L).build(); - boolean ok1 = dao.update(regularFile); - - assertTrue(ok1); - verify(pst1).setNull(6, Types.BIGINT); - verify(pst1).setLong(7, 7L); // WHERE FILE_ID = ? - - // Test RETIRED file with cleanup deadline - should bind as long - PreparedStatement pst2 = mock(PreparedStatement.class); - when(mockConn.prepareStatement(anyString())).thenReturn(pst2); - when(pst2.executeUpdate()).thenReturn(1); - - long deadline = 1_700_000_000_999L; - MetadataProto.File retiredFile = baseFile("u.pxl", RETIRED_VALUE) - .setId(8L) - .setCleanupAt(deadline) - .build(); - boolean ok2 = dao.update(retiredFile); - - assertTrue(ok2); - verify(pst2).setLong(6, deadline); - verify(pst2).setLong(7, 8L); - } - - // ------------------------------------------------------------------------- - // atomicSwapFiles — cleanupAt must be reset to NULL on promote - // ------------------------------------------------------------------------- - - /** - * The promote step must use the SQL fragment {@code FILE_CLEANUP_AT=NULL}. Without it, - * a file that was previously RETIRED and is being recycled into a fresh REGULAR slot - * would silently retain its deadline, eventually getting GC'd while live. - */ - @Test - public void atomicSwapFiles_promoteSqlClearsCleanupAt() throws Exception - { - PreparedStatement updatePst = mock(PreparedStatement.class); - PreparedStatement deletePst = mock(PreparedStatement.class); - - when(mockConn.prepareStatement(anyString())).thenAnswer(inv -> { - String sql = inv.getArgument(0); - if (sql.startsWith("UPDATE")) - { - return updatePst; - } - if (sql.startsWith("DELETE")) - { - return deletePst; - } - return mock(PreparedStatement.class); - }); - - boolean ok = dao.atomicSwapFiles(101L, Arrays.asList(11L, 12L)); - assertTrue(ok); - - // Capture the actual SQL string the production code sent to the JDBC driver. - org.mockito.ArgumentCaptor sqlCaptor = org.mockito.ArgumentCaptor.forClass(String.class); - verify(mockConn, atLeastOnce()).prepareStatement(sqlCaptor.capture()); - boolean clearsCleanupAt = false; - for (String sql : sqlCaptor.getAllValues()) - { - if (sql.contains("FILE_TYPE=?") && sql.contains("FILE_CLEANUP_AT=NULL")) - { - clearsCleanupAt = true; - break; - } - } - assertTrue("promote SQL must clear FILE_CLEANUP_AT to NULL together with the type update", - clearsCleanupAt); - - // The promote binds REGULAR + the new id, then commits. These behaviours are tied - // to the same transaction as the DELETE, so we check both ran on the same connection. - verify(updatePst).setInt(1, REGULAR_VALUE); - verify(updatePst).setLong(2, 101L); - verify(updatePst).executeUpdate(); - verify(deletePst).setLong(1, 11L); - verify(deletePst).setLong(2, 12L); - verify(deletePst).executeUpdate(); - verify(mockConn).setAutoCommit(false); - verify(mockConn).commit(); - } - - @Test - public void atomicSwapFiles_rollsBackOnSqlException() throws Exception - { - when(mockConn.prepareStatement(anyString())) - .thenThrow(new SQLException("boom")); - - boolean ok = dao.atomicSwapFiles(1L, Collections.singletonList(2L)); - - assertFalse("atomicSwapFiles must report failure when the JDBC layer throws", ok); - verify(mockConn).setAutoCommit(false); - verify(mockConn).rollback(); - verify(mockConn).setAutoCommit(true); // finally block must restore auto-commit - verify(mockConn, never()).commit(); - } - - // ------------------------------------------------------------------------- - // SELECT (buildFile) — cleanupAt round-trip from ResultSet to proto - // ------------------------------------------------------------------------- - - /** - * SELECT operation tests covering different cleanupAt scenarios from ResultSet to proto - */ - @Test - public void getById_cleanupAtRoundTripScenarios() throws Exception - { - // Test scenario 1: ResultSet with cleanupAt value (non-NULL) - Statement st1 = mock(Statement.class); - ResultSet rs1 = mock(ResultSet.class); - when(mockConn.createStatement()).thenReturn(st1); - when(st1.executeQuery(anyString())).thenReturn(rs1); - when(rs1.next()).thenReturn(true).thenReturn(false); - - when(rs1.getLong("FILE_ID")).thenReturn(99L); - when(rs1.getString("FILE_NAME")).thenReturn("x.pxl"); - when(rs1.getInt("FILE_TYPE")).thenReturn(RETIRED_VALUE); - when(rs1.getInt("FILE_NUM_RG")).thenReturn(2); - when(rs1.getLong("FILE_MIN_ROW_ID")).thenReturn(0L); - when(rs1.getLong("FILE_MAX_ROW_ID")).thenReturn(127L); - when(rs1.getLong("PATHS_PATH_ID")).thenReturn(5L); - when(rs1.getLong("FILE_CLEANUP_AT")).thenReturn(1_700_000_000_000L); - when(rs1.wasNull()).thenReturn(false); - - MetadataProto.File proto1 = dao.getById(99L); - - assertNotNull(proto1); - assertEquals(99L, proto1.getId()); - assertEquals(MetadataProto.File.Type.RETIRED, proto1.getType()); - assertTrue("non-NULL FILE_CLEANUP_AT column must surface as hasCleanupAt()", - proto1.hasCleanupAt()); - assertEquals(1_700_000_000_000L, proto1.getCleanupAt()); - - // Test scenario 2: ResultSet with NULL cleanupAt - Statement st2 = mock(Statement.class); - ResultSet rs2 = mock(ResultSet.class); - when(mockConn.createStatement()).thenReturn(st2); - when(st2.executeQuery(anyString())).thenReturn(rs2); - when(rs2.next()).thenReturn(true).thenReturn(false); - - when(rs2.getLong("FILE_ID")).thenReturn(1L); - when(rs2.getString("FILE_NAME")).thenReturn("r.pxl"); - when(rs2.getInt("FILE_TYPE")).thenReturn(REGULAR_VALUE); - when(rs2.getInt("FILE_NUM_RG")).thenReturn(1); - when(rs2.getLong("FILE_MIN_ROW_ID")).thenReturn(0L); - when(rs2.getLong("FILE_MAX_ROW_ID")).thenReturn(0L); - when(rs2.getLong("PATHS_PATH_ID")).thenReturn(1L); - when(rs2.getLong("FILE_CLEANUP_AT")).thenReturn(0L); - when(rs2.wasNull()).thenReturn(true); // critical: NULL column - - MetadataProto.File proto2 = dao.getById(1L); - - assertNotNull(proto2); - assertFalse("NULL FILE_CLEANUP_AT column must surface as !hasCleanupAt()", - proto2.hasCleanupAt()); - } - - // ------------------------------------------------------------------------- - // helpers - // ------------------------------------------------------------------------- - - private PreparedStatement stubPreparedStatementForInsert() throws SQLException - { - PreparedStatement pst = mock(PreparedStatement.class); - when(mockConn.prepareStatement(anyString())).thenReturn(pst); - when(pst.executeUpdate()).thenReturn(1); - - // After a successful insert, the DAO calls executeQuery("SELECT LAST_INSERT_ID()") - // on the same PreparedStatement. Stub a single-row ResultSet so the call returns cleanly. - ResultSet idRs = mock(ResultSet.class); - when(pst.executeQuery(anyString())).thenReturn(idRs); - when(idRs.next()).thenReturn(true); - when(idRs.getLong(1)).thenReturn(1L); - return pst; - } - - private static MetadataProto.File.Builder baseFile(String name, int typeValue) - { - return MetadataProto.File.newBuilder() - .setName(name) - .setTypeValue(typeValue) - .setNumRowGroup(1) - .setMinRowId(0L) - .setMaxRowId(0L) - .setPathId(1L); - } - - /** - * Replace the private {@code connection} field in the {@link MetaDBUtil} singleton with - * the supplied connection, returning the previous value. Using reflection here keeps - * the production class untouched while still letting us inject a Mockito-managed - * {@link Connection} for the duration of a single test. - */ - private static Connection swapConnection(Connection replacement) throws Exception - { - Field f = MetaDBUtil.class.getDeclaredField("connection"); - f.setAccessible(true); - Connection previous = (Connection) f.get(MetaDBUtil.Instance()); - f.set(MetaDBUtil.Instance(), replacement); - return previous; - } -} diff --git a/pixels-daemon/src/test/java/io/pixelsdb/pixels/daemon/retina/TestRetinaServer.java b/pixels-daemon/src/test/java/io/pixelsdb/pixels/daemon/retina/TestRetinaServer.java index cf4d0e526a..43e6de6bac 100644 --- a/pixels-daemon/src/test/java/io/pixelsdb/pixels/daemon/retina/TestRetinaServer.java +++ b/pixels-daemon/src/test/java/io/pixelsdb/pixels/daemon/retina/TestRetinaServer.java @@ -116,8 +116,8 @@ public void testRetinaServerImplStartsBackgroundGcAfterSuccessfulInitialization( when(metadataService.getSchemas()).thenReturn(Collections.singletonList(schema)); when(metadataService.getTables(schema.getName())).thenReturn(Collections.singletonList(table)); when(metadataService.getLayouts(schema.getName(), table.getName())).thenReturn(Collections.singletonList(layout)); - when(metadataService.getFiles(orderedPath.getId())).thenReturn(Collections.singletonList(orderedFile)); - when(metadataService.getFiles(compactPath.getId())).thenReturn(Collections.singletonList(compactFile)); + when(metadataService.getRegularFiles(orderedPath.getId())).thenReturn(Collections.singletonList(orderedFile)); + when(metadataService.getRegularFiles(compactPath.getId())).thenReturn(Collections.singletonList(compactFile)); doAnswer(invocation -> { lifecycleEvents.add("recover"); return null; diff --git a/pixels-planner/src/main/java/io/pixelsdb/pixels/planner/PixelsPlanner.java b/pixels-planner/src/main/java/io/pixelsdb/pixels/planner/PixelsPlanner.java index aef36f4cfb..f4d9b68481 100644 --- a/pixels-planner/src/main/java/io/pixelsdb/pixels/planner/PixelsPlanner.java +++ b/pixels-planner/src/main/java/io/pixelsdb/pixels/planner/PixelsPlanner.java @@ -1895,7 +1895,7 @@ public static List getFilePaths(List dirPaths, MetadataService met { base += "/"; } - for (File file : metadataService.getFiles(dirPath.getId())) + for (File file : metadataService.getRegularFiles(dirPath.getId())) { filePaths.add(base + file.getName()); } diff --git a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/StorageGarbageCollector.java b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/StorageGarbageCollector.java index 9aa31eaea1..ee2aeb7986 100644 --- a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/StorageGarbageCollector.java +++ b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/StorageGarbageCollector.java @@ -381,7 +381,7 @@ List scanAndGroupFiles(Set candidateFileIds, List files; try { - files = metadataService.getFiles(path.getId()); + files = metadataService.getRegularFiles(path.getId()); } catch (MetadataException e) { diff --git a/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestStorageGarbageCollector.java b/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestStorageGarbageCollector.java index 81be9522cd..f1d8d43f53 100644 --- a/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestStorageGarbageCollector.java +++ b/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestStorageGarbageCollector.java @@ -1939,12 +1939,12 @@ public void testAtomicSwap_idempotent() throws Exception } /** - * TEMPORARY_GC visibility semantics: before the swap, {@code getFiles(pathId)} must + * TEMPORARY_GC visibility semantics: before the swap, {@code getRegularFiles(pathId)} must * not return the TEMPORARY_GC new file (the DAO filters {@code FILE_TYPE = REGULAR}). * After the swap the promoted file is visible and the old file disappears. */ @Test - public void testAtomicSwap_temporaryInvisibleViaGetFiles() throws Exception + public void testAtomicSwap_temporaryInvisibleViaGetRegularFiles() throws Exception { writeTestFile("vis_old.pxl", LONG_ID_SCHEMA, new long[]{0, 1}, true, new long[]{100, 100}); long[] fileIds = registerTestFiles( @@ -1954,33 +1954,33 @@ public void testAtomicSwap_temporaryInvisibleViaGetFiles() throws Exception long oldFileId = fileIds[0]; long tempFileId = fileIds[1]; - List beforeSwap = metadataService.getFiles(testPathId); + List beforeSwap = metadataService.getRegularFiles(testPathId); Set beforeIds = new HashSet<>(); for (File f : beforeSwap) { beforeIds.add(f.getId()); } - assertTrue("REGULAR old file should be visible via getFiles before swap", + assertTrue("REGULAR old file should be visible via getRegularFiles before swap", beforeIds.contains(oldFileId)); - assertFalse("TEMPORARY_GC new file must NOT be visible via getFiles before swap", + assertFalse("TEMPORARY_GC new file must NOT be visible via getRegularFiles before swap", beforeIds.contains(tempFileId)); metadataService.atomicSwapFiles(tempFileId, Collections.singletonList(oldFileId)); - List afterSwap = metadataService.getFiles(testPathId); + List afterSwap = metadataService.getRegularFiles(testPathId); Set afterIds = new HashSet<>(); for (File f : afterSwap) { afterIds.add(f.getId()); } - assertTrue("Promoted file should be visible via getFiles after swap", + assertTrue("Promoted file should be visible via getRegularFiles after swap", afterIds.contains(tempFileId)); - assertFalse("Old file should NOT be visible via getFiles after swap", + assertFalse("Old file should NOT be visible via getRegularFiles after swap", afterIds.contains(oldFileId)); } // ----------------------------------------------------------------------- - // Coverage for getFiles(pathId) REGULAR-only enumeration. + // Coverage for getRegularFiles(pathId) REGULAR-only enumeration. // ----------------------------------------------------------------------- /** @@ -2008,11 +2008,11 @@ public void testGetFiles_mixedAllFileTypes_onlyRegular() throws Exception extremeId = insertRawFileWithType("mix_extreme_max_" + suffix + ".pxl", Integer.MAX_VALUE, 1, 0L, 1L); - List files = metadataService.getFiles(testPathId); + List files = metadataService.getRegularFiles(testPathId); Set visible = new HashSet<>(); for (File f : files) { - assertEquals("getFiles must only emit REGULAR", + assertEquals("getRegularFiles must only emit REGULAR", File.Type.REGULAR, f.getType()); visible.add(f.getId()); } @@ -2077,7 +2077,7 @@ public void testGetFiles_allNonRegularTypes_allHidden() throws Exception } registeredIds.add(regularId); - List visible = metadataService.getFiles(testPathId); + List visible = metadataService.getRegularFiles(testPathId); Set visibleIds = new HashSet<>(); for (File f : visible) { @@ -2128,7 +2128,7 @@ public void testGetFiles_retiredCoexistsWithFreshlyPromoted() throws Exception // Before swap: only oldRegular visible; RETIRED + TEMPORARY_GC hidden. Set beforeIds = new HashSet<>(); - for (File f : metadataService.getFiles(testPathId)) beforeIds.add(f.getId()); + for (File f : metadataService.getRegularFiles(testPathId)) beforeIds.add(f.getId()); assertTrue("old REGULAR must be visible before swap", beforeIds.contains(oldRegularId)); assertFalse("RETIRED tombstone must be hidden before swap", @@ -2141,9 +2141,9 @@ public void testGetFiles_retiredCoexistsWithFreshlyPromoted() throws Exception // After swap: tempGcId is now REGULAR (visible); old REGULAR is gone; the // coexisting RETIRED file must STILL be hidden (the swap did not promote it). Set afterIds = new HashSet<>(); - for (File f : metadataService.getFiles(testPathId)) + for (File f : metadataService.getRegularFiles(testPathId)) { - assertEquals("getFiles must only emit REGULAR after swap", + assertEquals("getRegularFiles must only emit REGULAR after swap", File.Type.REGULAR, f.getType()); afterIds.add(f.getId()); } @@ -2179,7 +2179,7 @@ public void testGetFiles_singleRegularMinimumData() throws Exception { fileId = registerTestFile("min_single_regular_" + System.nanoTime() + ".pxl", File.Type.REGULAR, 1, 0L, 0L); - List files = metadataService.getFiles(testPathId); + List files = metadataService.getRegularFiles(testPathId); File found = null; for (File f : files) { @@ -2214,7 +2214,7 @@ public void testGetFiles_deletedRegular_notVisible() throws Exception long regularId = registerTestFile("delete_visibility_" + System.nanoTime() + ".pxl", File.Type.REGULAR, 1, 0L, 1L); - List beforeDelete = metadataService.getFiles(testPathId); + List beforeDelete = metadataService.getRegularFiles(testPathId); Set beforeIds = new HashSet<>(); for (File f : beforeDelete) beforeIds.add(f.getId()); assertTrue("REGULAR file must be visible before delete", @@ -2222,7 +2222,7 @@ public void testGetFiles_deletedRegular_notVisible() throws Exception metadataService.deleteFiles(Collections.singletonList(regularId)); - List afterDelete = metadataService.getFiles(testPathId); + List afterDelete = metadataService.getRegularFiles(testPathId); for (File f : afterDelete) { assertFalse("deleted REGULAR file must no longer be visible", @@ -2271,7 +2271,7 @@ public void testGetFiles_concurrentReaders_consistentRegularOnly() throws Except startGate.await(); for (int i = 0; i < iterations; i++) { - List snapshot = metadataService.getFiles(testPathId); + List snapshot = metadataService.getRegularFiles(testPathId); boolean sawRegular = false; for (File f : snapshot) { @@ -2303,7 +2303,7 @@ public void testGetFiles_concurrentReaders_consistentRegularOnly() throws Except 0, missingRegular.get()); // A follow-up call should remain REGULAR-only after the concurrent burst. - List followUp = metadataService.getFiles(testPathId); + List followUp = metadataService.getRegularFiles(testPathId); assertNotNull("follow-up getFiles must not return null", followUp); for (File f : followUp) { diff --git a/proto/metadata.proto b/proto/metadata.proto index b21556a102..492b497a30 100644 --- a/proto/metadata.proto +++ b/proto/metadata.proto @@ -65,7 +65,7 @@ service MetadataService { rpc UpdatePath (UpdatePathRequest) returns (UpdatePathResponse); rpc DeletePaths (DeletePathsRequest) returns (DeletePathsResponse); rpc AddFiles (AddFilesRequest) returns (AddFilesResponse); - rpc GetFiles (GetFilesRequest) returns (GetFilesResponse); + rpc GetFilesByType (GetFilesByTypeRequest) returns (GetFilesByTypeResponse); rpc GetFileId (GetFileIdRequest) returns (GetFileIdResponse); rpc GetFileType (GetFileTypeRequest) returns (GetFileTypeResponse); rpc UpdateFile (UpdateFileRequest) returns (UpdateFileResponse); @@ -682,13 +682,14 @@ message AddFilesResponse { ResponseHeader header = 1; } -message GetFilesRequest { - // Query-visible REGULAR file enumeration. +message GetFilesByTypeRequest { + // If set, restricts the scan to one path; otherwise scans across paths. RequestHeader header = 1; - uint64 pathId = 2; + optional uint64 pathId = 2; + repeated File.Type fileTypes = 3; } -message GetFilesResponse { +message GetFilesByTypeResponse { ResponseHeader header = 1; repeated File files = 2; } diff --git a/scripts/sql/metadata_schema.sql b/scripts/sql/metadata_schema.sql index c3b0c5a7e9..2558d2d1af 100644 --- a/scripts/sql/metadata_schema.sql +++ b/scripts/sql/metadata_schema.sql @@ -328,7 +328,6 @@ CREATE TABLE IF NOT EXISTS `pixels_metadata`.`FILES` ( INDEX `fk_FILES_PATHS_idx` (`PATHS_PATH_ID` ASC), UNIQUE INDEX `PATH_ID_FILE_NAME_UNIQUE` (`PATHS_PATH_ID` ASC, `FILE_NAME` ASC), INDEX `FILE_ROW_ID_INDEX` USING BTREE (`FILE_MIN_ROW_ID`, `FILE_MAX_ROW_ID`), - INDEX `FILE_CLEANUP_AT_INDEX` USING BTREE (`FILE_TYPE`, `FILE_CLEANUP_AT`), CONSTRAINT `fk_FILES_PATHS` FOREIGN KEY (`PATHS_PATH_ID`) REFERENCES `pixels_metadata`.`PATHS` (`PATH_ID`) From 36d957c2d3eed51fb462650cf38ebdf4dfc596d9 Mon Sep 17 00:00:00 2001 From: Dongyang Geng Date: Thu, 14 May 2026 21:19:39 +0800 Subject: [PATCH 10/17] fix: retire old files during atomic swap --- .../common/metadata/MetadataService.java | 11 +- .../daemon/metadata/MetadataServiceImpl.java | 4 +- .../pixels/daemon/metadata/dao/FileDao.java | 9 +- .../daemon/metadata/dao/impl/RdbFileDao.java | 20 +-- .../daemon/metadata/dao/TestRdbFileDao.java | 90 +++++++----- .../retina/StorageGarbageCollector.java | 6 +- .../retina/TestStorageGarbageCollector.java | 129 ++++++++---------- proto/metadata.proto | 1 + 8 files changed, 145 insertions(+), 125 deletions(-) diff --git a/pixels-common/src/main/java/io/pixelsdb/pixels/common/metadata/MetadataService.java b/pixels-common/src/main/java/io/pixelsdb/pixels/common/metadata/MetadataService.java index f83d952e51..6486e7c0c4 100644 --- a/pixels-common/src/main/java/io/pixelsdb/pixels/common/metadata/MetadataService.java +++ b/pixels-common/src/main/java/io/pixelsdb/pixels/common/metadata/MetadataService.java @@ -1675,17 +1675,18 @@ public File getFileById(long fileId) throws MetadataException } /** - * Atomically promote a temporary file to REGULAR and delete the old files. - * @param newFileId the id of the new temporary file to promote - * @param oldFileIds the ids of old files to delete + * Atomically promote a temporary GC file to REGULAR and retire the old files. + * @param newFileId the id of the new temporary GC file to promote + * @param oldFileIds the ids of old files to retire + * @param cleanupAt the cleanup deadline to write on retired old files * @throws MetadataException if the request fails */ - public void atomicSwapFiles(long newFileId, List oldFileIds) throws MetadataException + public void atomicSwapFiles(long newFileId, List oldFileIds, long cleanupAt) throws MetadataException { String token = UUID.randomUUID().toString(); MetadataProto.AtomicSwapFilesRequest request = MetadataProto.AtomicSwapFilesRequest.newBuilder() .setHeader(MetadataProto.RequestHeader.newBuilder().setToken(token)) - .setNewFileId(newFileId).addAllOldFileIds(oldFileIds).build(); + .setNewFileId(newFileId).addAllOldFileIds(oldFileIds).setCleanupAt(cleanupAt).build(); try { MetadataProto.AtomicSwapFilesResponse response = this.stub.atomicSwapFiles(request); diff --git a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/MetadataServiceImpl.java b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/MetadataServiceImpl.java index 09bad50998..1e9957e73e 100644 --- a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/MetadataServiceImpl.java +++ b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/MetadataServiceImpl.java @@ -1526,7 +1526,9 @@ public void atomicSwapFiles(MetadataProto.AtomicSwapFilesRequest request, MetadataProto.ResponseHeader.Builder headerBuilder = MetadataProto.ResponseHeader.newBuilder() .setToken(request.getHeader().getToken()); - if (this.fileDao.atomicSwapFiles(request.getNewFileId(), request.getOldFileIdsList())) + if (request.hasCleanupAt() && + this.fileDao.atomicSwapFiles(request.getNewFileId(), request.getOldFileIdsList(), + request.getCleanupAt())) { headerBuilder.setErrorCode(SUCCESS).setErrorMsg(""); } diff --git a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/dao/FileDao.java b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/dao/FileDao.java index 844fcf34e2..a3d9920355 100644 --- a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/dao/FileDao.java +++ b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/dao/FileDao.java @@ -81,10 +81,11 @@ public boolean save (MetadataProto.File file) abstract public boolean deleteByIds (List ids); /** - * Atomically promote a temporary file to REGULAR and delete the old files in a single transaction. - * @param newFileId the id of the new temporary file to promote - * @param oldFileIds the ids of old files to delete + * Atomically promote a temporary GC file to REGULAR and retire old files in a single transaction. + * @param newFileId the id of the new temporary GC file to promote + * @param oldFileIds the ids of old regular files to retire + * @param cleanupAt the cleanup deadline to write on retired old files * @return true if the transaction committed successfully */ - abstract public boolean atomicSwapFiles(long newFileId, List oldFileIds); + abstract public boolean atomicSwapFiles(long newFileId, List oldFileIds, long cleanupAt); } diff --git a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/dao/impl/RdbFileDao.java b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/dao/impl/RdbFileDao.java index 6c1af4c24a..f205de88e9 100644 --- a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/dao/impl/RdbFileDao.java +++ b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/dao/impl/RdbFileDao.java @@ -30,7 +30,6 @@ import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; -import java.util.stream.Collectors; /** * @author hank @@ -350,7 +349,7 @@ public boolean deleteByIds(List ids) } @Override - public boolean atomicSwapFiles(long newFileId, List oldFileIds) + public boolean atomicSwapFiles(long newFileId, List oldFileIds, long cleanupAt) { Connection conn = db.getConnection(); try @@ -363,15 +362,20 @@ public boolean atomicSwapFiles(long newFileId, List oldFileIds) pst.setLong(2, newFileId); pst.executeUpdate(); } - String inClause = oldFileIds.stream().map(id -> "?").collect(Collectors.joining(",")); - try (PreparedStatement pst = conn.prepareStatement( - "DELETE FROM FILES WHERE FILE_ID IN (" + inClause + ")")) + if (oldFileIds != null && !oldFileIds.isEmpty()) { - for (int i = 0; i < oldFileIds.size(); i++) + try (PreparedStatement pst = conn.prepareStatement( + "UPDATE FILES SET FILE_TYPE=?, FILE_CLEANUP_AT=? WHERE FILE_ID=?")) { - pst.setLong(i + 1, oldFileIds.get(i)); + for (Long oldFileId : oldFileIds) + { + pst.setInt(1, MetadataProto.File.Type.RETIRED.getNumber()); + pst.setLong(2, cleanupAt); + pst.setLong(3, oldFileId); + pst.addBatch(); + } + pst.executeBatch(); } - pst.executeUpdate(); } conn.commit(); return true; diff --git a/pixels-daemon/src/test/java/io/pixelsdb/pixels/daemon/metadata/dao/TestRdbFileDao.java b/pixels-daemon/src/test/java/io/pixelsdb/pixels/daemon/metadata/dao/TestRdbFileDao.java index 984516c958..02197516c3 100644 --- a/pixels-daemon/src/test/java/io/pixelsdb/pixels/daemon/metadata/dao/TestRdbFileDao.java +++ b/pixels-daemon/src/test/java/io/pixelsdb/pixels/daemon/metadata/dao/TestRdbFileDao.java @@ -47,7 +47,6 @@ import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; @@ -249,44 +248,71 @@ public void update_invariantViolations_failFast() throws Exception // ========================================================================= /** - * Promoting a file clears {@code FILE_CLEANUP_AT} with the type update. + * Promoting a file clears {@code FILE_CLEANUP_AT}; retiring old files writes the shared deadline. */ @Test - public void atomicSwapFiles_promoteSqlClearsCleanupAt() throws Exception + public void atomicSwapFiles_promotesNewFileAndRetiresOldFilesWithCleanupAt() throws Exception { - PreparedStatement updatePst = mock(PreparedStatement.class); - PreparedStatement deletePst = mock(PreparedStatement.class); - when(mockConn.prepareStatement(anyString())).thenAnswer(inv -> { - String sql = inv.getArgument(0); - if (sql.startsWith("UPDATE")) return updatePst; - if (sql.startsWith("DELETE")) return deletePst; - return mock(PreparedStatement.class); - }); + PreparedStatement promotePst = mock(PreparedStatement.class); + PreparedStatement retirePst = mock(PreparedStatement.class); + when(mockConn.prepareStatement(anyString())).thenReturn(promotePst).thenReturn(retirePst); - assertTrue(dao.atomicSwapFiles(101L, Arrays.asList(11L, 12L))); + long cleanupAt = 1_700_000_001_234L; + assertTrue(dao.atomicSwapFiles(101L, Arrays.asList(11L, 12L), cleanupAt)); ArgumentCaptor sqlCaptor = ArgumentCaptor.forClass(String.class); - verify(mockConn, atLeastOnce()).prepareStatement(sqlCaptor.capture()); - boolean clearsCleanupAt = false; - for (String sql : sqlCaptor.getAllValues()) - { - if (sql.contains("FILE_TYPE=?") && sql.contains("FILE_CLEANUP_AT=NULL")) - { - clearsCleanupAt = true; - break; - } - } - assertTrue("promote SQL must clear FILE_CLEANUP_AT to NULL together with the type update", - clearsCleanupAt); - - verify(updatePst).setInt(1, REGULAR_VALUE); - verify(updatePst).setLong(2, 101L); - verify(updatePst).executeUpdate(); - verify(deletePst).setLong(1, 11L); - verify(deletePst).setLong(2, 12L); - verify(deletePst).executeUpdate(); + verify(mockConn, times(2)).prepareStatement(sqlCaptor.capture()); + String promoteSql = sqlCaptor.getAllValues().get(0); + String retireSql = sqlCaptor.getAllValues().get(1); + assertTrue("promote SQL must update FILE_TYPE", + promoteSql.contains("FILE_TYPE=?")); + assertTrue("promote SQL must clear FILE_CLEANUP_AT to NULL", + promoteSql.contains("FILE_CLEANUP_AT=NULL")); + assertTrue("retire SQL must update FILE_TYPE", + retireSql.contains("FILE_TYPE=?")); + assertTrue("retire SQL must bind FILE_CLEANUP_AT", + retireSql.contains("FILE_CLEANUP_AT=?")); + assertTrue("retire SQL must address old files by FILE_ID", + retireSql.contains("WHERE FILE_ID=?")); + + verify(promotePst).setInt(1, REGULAR_VALUE); + verify(promotePst).setLong(2, 101L); + verify(promotePst).executeUpdate(); + + verify(retirePst, times(2)).setInt(1, RETIRED_VALUE); + verify(retirePst, times(2)).setLong(2, cleanupAt); + verify(retirePst).setLong(3, 11L); + verify(retirePst).setLong(3, 12L); + verify(retirePst, times(2)).addBatch(); + verify(retirePst).executeBatch(); + verify(mockConn).setAutoCommit(false); verify(mockConn).commit(); + verify(mockConn).setAutoCommit(true); + } + + @Test + public void atomicSwapFiles_withNoOldFiles_onlyPromotesNewFile() throws Exception + { + PreparedStatement promotePst = mock(PreparedStatement.class); + when(mockConn.prepareStatement(anyString())).thenReturn(promotePst); + + assertTrue(dao.atomicSwapFiles(202L, Collections.emptyList(), 1_700_000_002_000L)); + + ArgumentCaptor sqlCaptor = ArgumentCaptor.forClass(String.class); + verify(mockConn).prepareStatement(sqlCaptor.capture()); + String promoteSql = sqlCaptor.getValue(); + assertTrue("promote SQL must update FILE_TYPE", + promoteSql.contains("FILE_TYPE=?")); + assertTrue("promote SQL must clear FILE_CLEANUP_AT to NULL", + promoteSql.contains("FILE_CLEANUP_AT=NULL")); + + verify(promotePst).setInt(1, REGULAR_VALUE); + verify(promotePst).setLong(2, 202L); + verify(promotePst).executeUpdate(); + verify(mockConn).setAutoCommit(false); + verify(mockConn).commit(); + verify(mockConn).setAutoCommit(true); } @Test @@ -295,7 +321,7 @@ public void atomicSwapFiles_rollsBackOnSqlException() throws Exception when(mockConn.prepareStatement(anyString())).thenThrow(new SQLException("boom")); assertFalse("atomicSwapFiles must report failure when the JDBC layer throws", - dao.atomicSwapFiles(1L, Collections.singletonList(2L))); + dao.atomicSwapFiles(1L, Collections.singletonList(2L), 42L)); verify(mockConn).setAutoCommit(false); verify(mockConn).rollback(); verify(mockConn).setAutoCommit(true); diff --git a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/StorageGarbageCollector.java b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/StorageGarbageCollector.java index ee2aeb7986..d72ef5aaa9 100644 --- a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/StorageGarbageCollector.java +++ b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/StorageGarbageCollector.java @@ -1173,17 +1173,18 @@ private List updateSinglePointIndex(RewriteResult result, long tableId, // ------------------------------------------------------------------------- /** - * Atomically promotes the new TEMPORARY_GC file to REGULAR, deletes old files from + * Atomically promotes the new TEMPORARY_GC file to REGULAR, retires old files in * the catalog, unregisters dual-write, and enqueues the old files for delayed cleanup. */ void commitFileGroup(RewriteResult result) throws Exception { List oldFileIds = result.group.files.stream() .map(fc -> fc.fileId).collect(Collectors.toList()); + long retireDeadline = System.currentTimeMillis() + retireDelayMs; try { - metadataService.atomicSwapFiles(result.newFileId, oldFileIds); + metadataService.atomicSwapFiles(result.newFileId, oldFileIds, retireDeadline); } catch (Exception e) { @@ -1200,7 +1201,6 @@ void commitFileGroup(RewriteResult result) throws Exception unregisterDualWrite(result); - long retireDeadline = System.currentTimeMillis() + retireDelayMs; for (FileCandidate fc : result.group.files) { resourceManager.scheduleRetiredFile( diff --git a/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestStorageGarbageCollector.java b/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestStorageGarbageCollector.java index f1d8d43f53..e56b1f1b45 100644 --- a/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestStorageGarbageCollector.java +++ b/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestStorageGarbageCollector.java @@ -1884,8 +1884,8 @@ public void testDualWrite_concurrentPressure() throws Exception /** * Atomicity with multiple old files: one TEMPORARY_GC new file and three REGULAR * old files are swapped in a single call. Verifies that after the call the new - * file is promoted to REGULAR and all old files are removed from the - * catalog—i.e., the UPDATE and DELETE execute as one indivisible transaction. + * file is promoted to REGULAR and all old files are marked RETIRED with + * the same cleanup deadline—i.e., both UPDATE steps execute as one transaction. */ @Test public void testAtomicSwap_multipleOldFilesAtomicity() throws Exception @@ -1902,25 +1902,27 @@ public void testAtomicSwap_multipleOldFilesAtomicity() throws Exception new File.Type[]{File.Type.REGULAR, File.Type.REGULAR, File.Type.REGULAR}, new int[]{1, 1, 1}, new long[]{0, 0, 0}, new long[]{1, 1, 1}); long newFileId = registerTestFile("atom_new.pxl", File.Type.TEMPORARY_GC, 1, 0, 1); + long cleanupAt = 1_700_000_010_000L; File preSwapNew = metadataService.getFileById(newFileId); assertNotNull("New file must exist before swap", preSwapNew); assertEquals("New file should be TEMPORARY_GC before swap", File.Type.TEMPORARY_GC, preSwapNew.getType()); - metadataService.atomicSwapFiles(newFileId, Arrays.asList(oldIds[0], oldIds[1], oldIds[2])); + metadataService.atomicSwapFiles(newFileId, Arrays.asList(oldIds[0], oldIds[1], oldIds[2]), cleanupAt); assertFileRegular(newFileId, "New file should be REGULAR after swap"); for (long oldId : oldIds) { - assertFileGone(oldId, "Old file " + oldId + " should be gone after swap"); + assertFileRetired(oldId, cleanupAt, + "Old file " + oldId + " should be retired after swap"); } } /** * Idempotency: calling {@code atomicSwapFiles} a second time after the swap has - * already committed must not throw. The UPDATE is a no-op (already REGULAR) and - * the DELETE is a no-op (old files already removed). + * already committed must not throw. The new file remains REGULAR and the old file + * remains RETIRED with the retry's cleanup deadline. */ @Test public void testAtomicSwap_idempotent() throws Exception @@ -1928,55 +1930,18 @@ public void testAtomicSwap_idempotent() throws Exception writeTestFile("idem_old.pxl", LONG_ID_SCHEMA, new long[]{0, 1, 2}, true, new long[]{100, 100, 100}); long oldFileId = registerTestFile("idem_old.pxl", File.Type.REGULAR, 1, 0, 2); long newFileId = registerTestFile("idem_new.pxl", File.Type.TEMPORARY_GC, 1, 0, 2); + long firstCleanupAt = 1_700_000_020_000L; + long retryCleanupAt = 1_700_000_030_000L; - metadataService.atomicSwapFiles(newFileId, Collections.singletonList(oldFileId)); + metadataService.atomicSwapFiles(newFileId, Collections.singletonList(oldFileId), firstCleanupAt); assertFileRegular(newFileId, "File should be REGULAR after first swap"); + assertFileRetired(oldFileId, firstCleanupAt, "Old file should be RETIRED after first swap"); - metadataService.atomicSwapFiles(newFileId, Collections.singletonList(oldFileId)); + metadataService.atomicSwapFiles(newFileId, Collections.singletonList(oldFileId), retryCleanupAt); assertFileRegular(newFileId, "File should remain REGULAR after idempotent retry"); - assertFileGone(oldFileId, "Old file should remain absent after idempotent retry"); - } - - /** - * TEMPORARY_GC visibility semantics: before the swap, {@code getRegularFiles(pathId)} must - * not return the TEMPORARY_GC new file (the DAO filters {@code FILE_TYPE = REGULAR}). - * After the swap the promoted file is visible and the old file disappears. - */ - @Test - public void testAtomicSwap_temporaryInvisibleViaGetRegularFiles() throws Exception - { - writeTestFile("vis_old.pxl", LONG_ID_SCHEMA, new long[]{0, 1}, true, new long[]{100, 100}); - long[] fileIds = registerTestFiles( - new String[]{"vis_old.pxl", "vis_new_temp.pxl"}, - new File.Type[]{File.Type.REGULAR, File.Type.TEMPORARY_GC}, - new int[]{1, 1}, new long[]{0, 0}, new long[]{1, 1}); - long oldFileId = fileIds[0]; - long tempFileId = fileIds[1]; - - List beforeSwap = metadataService.getRegularFiles(testPathId); - Set beforeIds = new HashSet<>(); - for (File f : beforeSwap) - { - beforeIds.add(f.getId()); - } - assertTrue("REGULAR old file should be visible via getRegularFiles before swap", - beforeIds.contains(oldFileId)); - assertFalse("TEMPORARY_GC new file must NOT be visible via getRegularFiles before swap", - beforeIds.contains(tempFileId)); - - metadataService.atomicSwapFiles(tempFileId, Collections.singletonList(oldFileId)); - - List afterSwap = metadataService.getRegularFiles(testPathId); - Set afterIds = new HashSet<>(); - for (File f : afterSwap) - { - afterIds.add(f.getId()); - } - assertTrue("Promoted file should be visible via getRegularFiles after swap", - afterIds.contains(tempFileId)); - assertFalse("Old file should NOT be visible via getRegularFiles after swap", - afterIds.contains(oldFileId)); + assertFileRetired(oldFileId, retryCleanupAt, + "Old file should remain RETIRED after idempotent retry"); } // ----------------------------------------------------------------------- @@ -2136,10 +2101,11 @@ public void testGetFiles_retiredCoexistsWithFreshlyPromoted() throws Exception assertFalse("TEMPORARY_GC must be hidden before swap", beforeIds.contains(tempGcId)); - metadataService.atomicSwapFiles(tempGcId, Collections.singletonList(oldRegularId)); + long cleanupAt = 1_700_000_050_000L; + metadataService.atomicSwapFiles(tempGcId, Collections.singletonList(oldRegularId), cleanupAt); - // After swap: tempGcId is now REGULAR (visible); old REGULAR is gone; the - // coexisting RETIRED file must STILL be hidden (the swap did not promote it). + // After swap: tempGcId is now REGULAR (visible); old REGULAR is now RETIRED and + // hidden; the coexisting RETIRED file must STILL be hidden (the swap did not promote it). Set afterIds = new HashSet<>(); for (File f : metadataService.getRegularFiles(testPathId)) { @@ -2149,14 +2115,12 @@ public void testGetFiles_retiredCoexistsWithFreshlyPromoted() throws Exception } assertTrue("freshly-promoted file must be visible after swap", afterIds.contains(tempGcId)); - assertFalse("the deleted old REGULAR must be gone after swap", + assertFalse("the retired old REGULAR must be hidden after swap", afterIds.contains(oldRegularId)); + assertFileRetired(oldRegularId, cleanupAt, + "the old REGULAR must become RETIRED after swap"); assertFalse("the unrelated RETIRED tombstone must remain hidden after swap", afterIds.contains(retiredCoexistingId)); - - // After the promote, the old file ids are deleted — clear the local handle so - // the cleanup block below does not double-delete a non-existent row. - oldRegularId = -1L; } finally { @@ -2330,7 +2294,7 @@ public void testGetFiles_concurrentReaders_consistentRegularOnly() throws Except * thread, so {@code atomicSwapFiles} is never called concurrently in production. * This test reflects that design: N independent (newFile, oldFile) pairs are * swapped one after another, and every new file ends up REGULAR while every - * old file is removed. + * old file is marked RETIRED with its cleanup deadline. */ @Test public void testAtomicSwap_multipleSerialSwaps() throws Exception @@ -2342,6 +2306,7 @@ public void testAtomicSwap_multipleSerialSwaps() throws Exception long[] newFileIds = new long[nPairs]; long[] oldFileIds = new long[nPairs]; + long[] cleanupAts = new long[nPairs]; for (int i = 0; i < nPairs; i++) { @@ -2355,26 +2320,28 @@ public void testAtomicSwap_multipleSerialSwaps() throws Exception new int[]{1, 1}, new long[]{0, 0}, new long[]{0, 0}); oldFileIds[i] = pair[0]; newFileIds[i] = pair[1]; + cleanupAts[i] = 1_700_000_060_000L + i; } for (int i = 0; i < nPairs; i++) { metadataService.atomicSwapFiles(newFileIds[i], - Collections.singletonList(oldFileIds[i])); + Collections.singletonList(oldFileIds[i]), cleanupAts[i]); } for (int i = 0; i < nPairs; i++) { assertFileRegular(newFileIds[i], "Promoted file " + i + " must be REGULAR"); - assertFileGone(oldFileIds[i], "Old file " + i + " should be gone"); + assertFileRetired(oldFileIds[i], cleanupAts[i], + "Old file " + i + " should be RETIRED"); } } /** * Partial old-files-already-gone: one old file is deleted before the swap, but - * {@code atomicSwapFiles} is called with both IDs. The DELETE-WHERE-IN for an - * already-absent row is a no-op; the transaction must still commit, promoting the - * new file and removing the remaining old file. + * {@code atomicSwapFiles} is called with both IDs. The UPDATE for an already-absent + * row is a no-op; the transaction must still commit, promoting the new file and + * retiring the remaining old file. */ @Test public void testAtomicSwap_partialOldFilesAlreadyGone() throws Exception @@ -2391,10 +2358,11 @@ public void testAtomicSwap_partialOldFilesAlreadyGone() throws Exception assertFileGone(oldIds[0], "old1 should be gone before swap"); long newFileId = registerTestFile("partial_new.pxl", File.Type.TEMPORARY_GC, 1, 0, 1); - metadataService.atomicSwapFiles(newFileId, Arrays.asList(oldIds[0], oldIds[1])); + long cleanupAt = 1_700_000_070_000L; + metadataService.atomicSwapFiles(newFileId, Arrays.asList(oldIds[0], oldIds[1]), cleanupAt); assertFileRegular(newFileId, "New file must be REGULAR"); - assertFileGone(oldIds[1], "Remaining old file should be gone"); + assertFileRetired(oldIds[1], cleanupAt, "Remaining old file should be RETIRED"); } /** @@ -2588,7 +2556,8 @@ public void testEndToEnd_fullGcCycle() throws Exception e2eGc.commitFileGroup(result); assertFileRegular(newFileId, "new file should be REGULAR after commit"); - assertFileGone(srcFileId, "old file should be gone from catalog after commit"); + assertFileRetiredWithCleanupAt(srcFileId, + "old file should be RETIRED in catalog after commit"); assertTrue("old physical file should still exist (delayed cleanup, not yet due)", fileStorage.exists(srcPath)); @@ -2873,7 +2842,7 @@ public void testEndToEnd_concurrentCdcAndGc() throws Exception // 3b. Verify catalog state assertFileRegular(newFileId, "new file should be REGULAR"); - assertFileGone(srcFileId, "old file should be gone from catalog"); + assertFileRetiredWithCleanupAt(srcFileId, "old file should be RETIRED in catalog"); // 3c. Forward mapping int[] fwd = result.forwardRgMappings.get(srcFileId).get(0); @@ -3251,10 +3220,10 @@ public void testEndToEnd_multiRoundCdcGcLifecycle() throws Exception assertNotNull("file-B must still exist (not GCed)", metadataService.getFileById(fileIdB)); assertNotNull("file-C must still exist", metadataService.getFileById(fileIdC)); - // Old generations gone from catalog - assertFileGone(fileIdA, "file-A should be gone from catalog"); - assertFileGone(fileIdAprime, "file-A' should be gone from catalog"); - assertFileGone(fileIdAdoubleprime, "file-A'' should be gone from catalog"); + // Old generations are retired in catalog + assertFileRetiredWithCleanupAt(fileIdA, "file-A should be RETIRED in catalog"); + assertFileRetiredWithCleanupAt(fileIdAprime, "file-A' should be RETIRED in catalog"); + assertFileRetiredWithCleanupAt(fileIdAdoubleprime, "file-A'' should be RETIRED in catalog"); // Physical files from generations 1 and 2 cleaned up assertFalse("file-A physical should not exist", fileStorage.exists(pathA)); @@ -3441,6 +3410,22 @@ private void assertFileRegular(long fileId, String msg) throws Exception assertEquals(msg, File.Type.REGULAR, f.getType()); } + private void assertFileRetired(long fileId, long cleanupAt, String msg) throws Exception + { + File f = metadataService.getFileById(fileId); + assertNotNull(msg, f); + assertEquals(msg, File.Type.RETIRED, f.getType()); + assertEquals(msg, Long.valueOf(cleanupAt), f.getCleanupAt()); + } + + private void assertFileRetiredWithCleanupAt(long fileId, String msg) throws Exception + { + File f = metadataService.getFileById(fileId); + assertNotNull(msg, f); + assertEquals(msg, File.Type.RETIRED, f.getType()); + assertNotNull(msg, f.getCleanupAt()); + } + // ======================================================================= // Helpers: GC factory for grouping tests // ======================================================================= diff --git a/proto/metadata.proto b/proto/metadata.proto index 492b497a30..9cf7269944 100644 --- a/proto/metadata.proto +++ b/proto/metadata.proto @@ -746,6 +746,7 @@ message AtomicSwapFilesRequest { RequestHeader header = 1; uint64 newFileId = 2; repeated uint64 oldFileIds = 3; + optional uint64 cleanupAt = 4; } message AtomicSwapFilesResponse { From faff52d5a1ed63941c0e57fa6d565572eb2cb999 Mon Sep 17 00:00:00 2001 From: Dongyang Geng Date: Fri, 22 May 2026 15:26:46 +0800 Subject: [PATCH 11/17] feat: order ingest file publish and centralize through write buffer --- .../io/pixelsdb/pixels/core/PixelsWriter.java | 18 + .../pixels/core/PixelsWriterImpl.java | 62 ++ .../daemon/retina/RetinaServerImpl.java | 120 +++- .../pixels/retina/FileWriterManager.java | 170 +++-- .../retina/IngestFileMetadataRegistry.java | 228 ++++++ .../pixels/retina/IngestFilePublisher.java | 94 +++ .../io/pixelsdb/pixels/retina/MemTable.java | 33 + .../pixels/retina/PixelsWriteBuffer.java | 403 ++++++----- .../pixels/retina/RetinaResourceManager.java | 134 +++- .../retina/StorageGarbageCollector.java | 1 + .../TestIngestFileMetadataRegistry.java | 129 ++++ .../retina/TestIngestFilePublisher.java | 664 ------------------ .../pixels/retina/TestPixelsWriteBuffer.java | 95 ++- 13 files changed, 1176 insertions(+), 975 deletions(-) create mode 100644 pixels-retina/src/main/java/io/pixelsdb/pixels/retina/IngestFileMetadataRegistry.java create mode 100644 pixels-retina/src/main/java/io/pixelsdb/pixels/retina/IngestFilePublisher.java create mode 100644 pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestIngestFileMetadataRegistry.java delete mode 100644 pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestIngestFilePublisher.java diff --git a/pixels-core/src/main/java/io/pixelsdb/pixels/core/PixelsWriter.java b/pixels-core/src/main/java/io/pixelsdb/pixels/core/PixelsWriter.java index c8c394587b..bf21f73dc9 100644 --- a/pixels-core/src/main/java/io/pixelsdb/pixels/core/PixelsWriter.java +++ b/pixels-core/src/main/java/io/pixelsdb/pixels/core/PixelsWriter.java @@ -63,4 +63,22 @@ public interface PixelsWriter extends Closeable int getNumWriteRequests(); long getCompletedBytes(); + + /** + * Release writer resources without writing the file tail. Caller is + * responsible for deleting any partial bytes the underlying physical + * writer may have flushed before abort. + * + *

Aborting after one or more row batches have been added is not + * supported and results in undefined file contents; aborting an + * already-closed writer is a no-op. + * + *

The default implementation falls back to {@link #close()} for + * writers that do not distinguish abort from normal close (e.g. test + * fakes or stream writers that never produce a file tail). + */ + default void abort() throws IOException + { + close(); + } } diff --git a/pixels-core/src/main/java/io/pixelsdb/pixels/core/PixelsWriterImpl.java b/pixels-core/src/main/java/io/pixelsdb/pixels/core/PixelsWriterImpl.java index 9b86e55906..02ae3a8547 100644 --- a/pixels-core/src/main/java/io/pixelsdb/pixels/core/PixelsWriterImpl.java +++ b/pixels-core/src/main/java/io/pixelsdb/pixels/core/PixelsWriterImpl.java @@ -588,6 +588,68 @@ public void close() } } + /** + * Abort the writer: release underlying resources without writing the + * file tail. Caller must ensure no row batches have been added; calling + * abort after data has been written results in undefined file contents + * and the caller should also delete any partial bytes the physical + * writer may have flushed. + * + *

Errors closing component writers are logged and the first failure + * is rethrown after all components have been attempted, so resources are + * released as eagerly as possible. + */ + @Override + public void abort() throws IOException + { + IOException firstFailure = null; + try + { + physicalWriter.close(); + } + catch (IOException e) + { + firstFailure = e; + LOGGER.warn("PixelsWriterImpl.abort: physicalWriter close failed", e); + } + for (ColumnWriter cw : columnWriters) + { + try + { + cw.close(); + } + catch (IOException e) + { + if (firstFailure == null) + { + firstFailure = e; + } + LOGGER.warn("PixelsWriterImpl.abort: columnWriter close failed", e); + } + } + if (hasHiddenColumn) + { + try + { + hiddenColumnWriter.close(); + } + catch (IOException e) + { + if (firstFailure == null) + { + firstFailure = e; + } + LOGGER.warn("PixelsWriterImpl.abort: hiddenColumnWriter close failed", e); + } + } + columnWriterService.shutdown(); + columnWriterService.shutdownNow(); + if (firstFailure != null) + { + throw firstFailure; + } + } + private void writeRowGroup() throws IOException { int rowGroupDataLength = 0; diff --git a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/retina/RetinaServerImpl.java b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/retina/RetinaServerImpl.java index c9be09752c..fe5499072f 100644 --- a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/retina/RetinaServerImpl.java +++ b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/retina/RetinaServerImpl.java @@ -537,24 +537,45 @@ private void processUpdateRequest(RetinaProto.UpdateRecordRequest request) throw { List primaryEntries = new ArrayList<>(subList.size()); List rowIds = new ArrayList<>(subList.size()); + List insertedLocations = new ArrayList<>(subList.size()); - // 2c. Insert records - for (RetinaProto.InsertData data : subList) + try { - byte[][] values = data.getColValuesList().stream().map(ByteString::toByteArray).toArray(byte[][]::new); - IndexProto.PrimaryIndexEntry.Builder builder = retinaResourceManager.insertRecord(schemaName, tableName, values, timestamp, virtualNodeId); - builder.setIndexKey(data.getIndexKeys(0)); - IndexProto.PrimaryIndexEntry entry = builder.build(); - primaryEntries.add(entry); - rowIds.add(entry.getRowId()); - } + // 2b. Insert records + for (RetinaProto.InsertData data : subList) + { + byte[][] values = data.getColValuesList().stream().map(ByteString::toByteArray).toArray(byte[][]::new); + IndexProto.PrimaryIndexEntry.Builder builder = retinaResourceManager.insertRecord(schemaName, tableName, values, timestamp, virtualNodeId); + builder.setIndexKey(data.getIndexKeys(0)); + IndexProto.PrimaryIndexEntry entry = builder.build(); + primaryEntries.add(entry); + rowIds.add(entry.getRowId()); + insertedLocations.add(entry.getRowLocation()); + } - // 2d. Put primary index entries - long tableId = primaryEntries.get(0).getIndexKey().getTableId(); - indexService.putPrimaryIndexEntries(tableId, primaryIndexId, primaryEntries, option); + // 2c. Put primary index entries + long tableId = primaryEntries.get(0).getIndexKey().getTableId(); + indexService.putPrimaryIndexEntries(tableId, primaryIndexId, primaryEntries, option); - // 2e. Put secondary index entries - processSecondaryIndexes(subList, RetinaProto.InsertData::getIndexKeysList, rowIds, option, false); + // 2d. Put secondary index entries + processSecondaryIndexes(subList, RetinaProto.InsertData::getIndexKeysList, rowIds, option, false); + } + catch (Exception e) + { + for (IndexProto.RowLocation loc : insertedLocations) + { + try + { + this.retinaResourceManager.deleteRecord(loc, timestamp); + } + catch (Exception rollbackEx) + { + logger.error("Failed to roll back visibility for inserted row at fileId={}, rgId={}, rgRowOffset={}", + loc.getFileId(), loc.getRgId(), loc.getRgRowOffset(), rollbackEx); + } + } + throw e; + } }); } // ================================================================= @@ -570,40 +591,61 @@ private void processUpdateRequest(RetinaProto.UpdateRecordRequest request) throw { List primaryEntries = new ArrayList<>(subList.size()); List rowIds = new ArrayList<>(subList.size()); + List insertedLocations = new ArrayList<>(subList.size()); - // 3c. Insert new records - for (RetinaProto.UpdateData data : subList) + try { - byte[][] values = data.getColValuesList().stream().map(ByteString::toByteArray).toArray(byte[][]::new); - IndexProto.PrimaryIndexEntry.Builder builder = retinaResourceManager.insertRecord(schemaName, tableName, values, timestamp, virtualNodeId); - builder.setIndexKey(data.getIndexKeys(0)); - IndexProto.PrimaryIndexEntry entry = builder.build(); - primaryEntries.add(entry); - rowIds.add(entry.getRowId()); - } + // 3b. Insert new records + for (RetinaProto.UpdateData data : subList) + { + byte[][] values = data.getColValuesList().stream().map(ByteString::toByteArray).toArray(byte[][]::new); + IndexProto.PrimaryIndexEntry.Builder builder = retinaResourceManager.insertRecord(schemaName, tableName, values, timestamp, virtualNodeId); + builder.setIndexKey(data.getIndexKeys(0)); + IndexProto.PrimaryIndexEntry entry = builder.build(); + primaryEntries.add(entry); + rowIds.add(entry.getRowId()); + insertedLocations.add(entry.getRowLocation()); + } - // 3d. Update primary index entries with fine-grained locking - long tableId = primaryEntries.get(0).getIndexKey().getTableId(); - String lockKey = "v_" + virtualNodeId + "_b_" + bucketId + "_i_" + primaryIndexId; - Lock lock = updateLocks.get(lockKey); + // 3c. Update primary index entries with bucket-level locking + long tableId = primaryEntries.get(0).getIndexKey().getTableId(); + String lockKey = "v_" + virtualNodeId + "_b_" + bucketId + "_i_" + primaryIndexId; + Lock lock = updateLocks.get(lockKey); - lock.lock(); - try - { - List prevLocs = indexService.updatePrimaryIndexEntries(tableId, primaryIndexId, primaryEntries, option); - // 3e. Delete previous records - for (IndexProto.RowLocation loc : prevLocs) + lock.lock(); + try + { + List prevLocs = indexService.updatePrimaryIndexEntries(tableId, primaryIndexId, primaryEntries, option); + // 3d. Delete previous records + for (IndexProto.RowLocation loc : prevLocs) + { + this.retinaResourceManager.deleteRecord(loc, timestamp); + } + } + finally { - this.retinaResourceManager.deleteRecord(loc, timestamp); + lock.unlock(); } + + // 3e. Update secondary index entries + processSecondaryIndexes(subList, RetinaProto.UpdateData::getIndexKeysList, rowIds, option, true); } - finally + catch (Exception e) { - lock.unlock(); + for (IndexProto.RowLocation loc : insertedLocations) + { + try + { + this.retinaResourceManager.deleteRecord(loc, timestamp); + } + catch (Exception rollbackEx) + { + logger.error("Failed to roll back visibility for inserted row at fileId={}, rgId={}, rgRowOffset={}", + loc.getFileId(), loc.getRgId(), loc.getRgRowOffset(), rollbackEx); + } + } + throw e; } - - // 3f. Update secondary index entries - processSecondaryIndexes(subList, RetinaProto.UpdateData::getIndexKeysList, rowIds, option, true); }); } } diff --git a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/FileWriterManager.java b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/FileWriterManager.java index 7c463e0814..3acd97283f 100644 --- a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/FileWriterManager.java +++ b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/FileWriterManager.java @@ -31,17 +31,19 @@ import io.pixelsdb.pixels.core.TypeDescription; import io.pixelsdb.pixels.core.encoding.EncodingLevel; import io.pixelsdb.pixels.core.vector.VectorizedRowBatch; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; -import java.io.IOException; import java.nio.ByteBuffer; import java.util.Collections; -import java.util.concurrent.CompletableFuture; /** * Responsible for managing several blocks of data and writing them to a file. */ public class FileWriterManager { + private static final Logger logger = LogManager.getLogger(FileWriterManager.class); + private final long tableId; private final PixelsWriter writer; private final File file; @@ -50,8 +52,16 @@ public class FileWriterManager private final long firstBlockId; private long lastBlockId = -1; private final int virtualNodeId; - // Initialized by PixelsWriteBuffer's single-threaded file publisher. - private CompletableFuture physicalCloseFuture; + + // [fileMinRowId, fileMaxRowId] is the range of row ids in the file. + private long fileMinRowId = Long.MAX_VALUE; + private long fileMaxRowId = Long.MIN_VALUE; + + private volatile boolean physicalClosed; + private volatile RetinaException physicalCloseFailure; + + // Signals that the index has been flushed. + private volatile boolean indexFlushed; /** * Creating pixelsWriter by passing in parameters avoids the need to read @@ -124,24 +134,32 @@ public FileWriterManager(long tableId, TypeDescription schema, .build(); } catch (Exception e) { + retinaResourceManager.removeVisibility(this.file.getId()); + try + { + if (!MetadataService.Instance().deleteFiles(Collections.singletonList(this.file.getId()))) + { + logger.warn("Failed to delete metadata for ingest file after writer creation failure, fileId={}", + this.file.getId()); + } + } + catch (MetadataException metadataException) + { + logger.warn("Failed to delete metadata for ingest file after writer creation failure, fileId={}", + this.file.getId(), metadataException); + } throw new RetinaException("Failed to create pixels writer", e); } } - FileWriterManager(long tableId, PixelsWriter writer, File file, - long firstBlockId, long lastBlockId, int virtualNodeId) + public long getFileId() { - this.tableId = tableId; - this.writer = writer; - this.file = file; - this.firstBlockId = firstBlockId; - this.lastBlockId = lastBlockId; - this.virtualNodeId = virtualNodeId; + return this.file.getId(); } - public long getFileId() + public String getFileName() { - return this.file.getId(); + return this.file.getName(); } public void setLastBlockId(long lastBlockId) @@ -159,51 +177,76 @@ public long getLastBlockId() return this.lastBlockId; } - File getFileSnapshot() + public int getVirtualNodeId() + { + return this.virtualNodeId; + } + + public synchronized void includeRowId(long rowId) + { + this.fileMinRowId = Math.min(this.fileMinRowId, rowId); + this.fileMaxRowId = Math.max(this.fileMaxRowId, rowId); + } + + public synchronized boolean hasRowIds() + { + return this.fileMinRowId != Long.MAX_VALUE && this.fileMaxRowId != Long.MIN_VALUE; + } + + public boolean isPhysicalClosed() + { + return this.physicalClosed; + } + + public boolean isIndexFlushed() + { + return this.indexFlushed; + } + + void markIndexFlushed() + { + this.indexFlushed = true; + } + + public synchronized File getFileSnapshot() throws RetinaException { + if (!hasRowIds()) + { + throw new RetinaException("Cannot create file snapshot without row-id hull: fileId=" + getFileId()); + } File snapshot = new File(); snapshot.setId(this.file.getId()); snapshot.setName(this.file.getName()); snapshot.setType(this.file.getType()); snapshot.setNumRowGroup(this.file.getNumRowGroup()); - snapshot.setMinRowId(this.file.getMinRowId()); - snapshot.setMaxRowId(this.file.getMaxRowId()); + snapshot.setMinRowId(this.fileMinRowId); + snapshot.setMaxRowId(this.fileMaxRowId); snapshot.setPathId(this.file.getPathId()); return snapshot; } - public void addRowBatch(VectorizedRowBatch rowBatch) throws RetinaException - { - try - { - this.writer.addRowBatch(rowBatch); - } catch (IOException e) - { - throw new RetinaException("Failed to add rowBatch to pixels writer", e); - } - } - /** - * Create a background thread to write the block of data stored in shared storage to a file. - * Metadata publication is handled by {@link PixelsWriteBuffer} after the - * physical close and index flush barrier both complete. + * Replay object blocks and physically close the writer. + * Idempotent after success; failed closes rethrow the cached failure. */ - CompletableFuture finish() + public synchronized void finish() throws RetinaException { - if (physicalCloseFuture != null) + if (this.physicalCloseFailure != null) { - return physicalCloseFuture; + throw this.physicalCloseFailure; + } + if (this.physicalClosed) + { + return; } - CompletableFuture future = new CompletableFuture<>(); - physicalCloseFuture = future; - - new Thread(() -> { - try + try + { + if (this.lastBlockId >= this.firstBlockId) { + ObjectStorageManager objectStorageManager = ObjectStorageManager.Instance(); for (long blockId = firstBlockId; blockId <= lastBlockId; ++blockId) { - ObjectStorageManager objectStorageManager = ObjectStorageManager.Instance(); /* * Issue-1083: Since we obtain a read-only ByteBuffer from the S3 Reader, * we cannot read a byte[]. Instead, we should return the ByteBuffer directly. @@ -211,14 +254,47 @@ CompletableFuture finish() ByteBuffer data = objectStorageManager.read(this.tableId, virtualNodeId, blockId); this.writer.addRowBatch(VectorizedRowBatch.deserialize(data)); } - this.writer.close(); - future.complete(null); - } catch (Exception e) - { - future.completeExceptionally(e); } - }, "pixels-retina-file-finish-" + this.file.getId()).start(); + this.writer.close(); + this.physicalClosed = true; + } catch (Exception e) + { + RetinaException wrapped = new RetinaException( + "Failed to physically close ingest file " + this.file.getId(), e); + this.physicalCloseFailure = wrapped; + throw wrapped; + } + } - return future; + /** + * Discard a zero-data ingest file by aborting the writer and removing metadata. + * The caller deletes any half-written physical bytes before calling this. + * Must not be called after {@link #finish()}. + */ + public synchronized void discard() throws RetinaException + { + if (isPhysicalClosed()) + { + throw new RetinaException( + "Cannot discard a physically closed FileWriterManager, fileId=" + getFileId()); + } + try + { + this.writer.abort(); + } + catch (Exception e) + { + logger.warn("FileWriterManager.discard: writer abort failed, fileId={}", getFileId(), e); + } + try + { + MetadataService.Instance().deleteFiles(Collections.singletonList(this.file.getId())); + } + catch (MetadataException e) + { + throw new RetinaException( + "Failed to delete TEMPORARY_INGEST file metadata, fileId=" + getFileId(), e); + } + RetinaResourceManager.Instance().removeVisibility(this.file.getId()); } } diff --git a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/IngestFileMetadataRegistry.java b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/IngestFileMetadataRegistry.java new file mode 100644 index 0000000000..96b38fad08 --- /dev/null +++ b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/IngestFileMetadataRegistry.java @@ -0,0 +1,228 @@ +/* + * Copyright 2026 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the Affero GNU General Public + * License along with Pixels. If not, see + * . + */ +package io.pixelsdb.pixels.retina; + +import io.pixelsdb.pixels.common.exception.RetinaException; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * In-process source of truth for the ingest-path metadata + * {@code (fileId, tableId, virtualNodeId, firstBlockId)} of published REGULAR + * files. + *

+ * Callers must {@link #register} files of a given {@code (tableId, virtualNodeId)} + * stream in strictly increasing {@code firstBlockId} order. That ordering is the + * stream-append ordering enforced by {@link IngestFilePublisher} at publish time, + * and is what allows {@link #listByStream} to return ingest order via plain + * insertion order without an explicit sort. Out-of-order registration is treated + * as a publisher contract violation and fails closed. + *

+ * Commit timestamp bounds are intentionally omitted. A REGULAR file's commit + * timestamp bounds are persisted by PixelsWriter in + * {@code footer.hiddenColumnStats}, and callers that need per-segment + * timestamps for not-yet-published data consult the source memtable directly + * via {@link MemTable#getMinCommitTs()}. + */ +public class IngestFileMetadataRegistry +{ + private final Map entriesByFileId = new HashMap<>(); + private final Map> entriesByStream = new HashMap<>(); + + IngestFileMetadataRegistry() + { + } + + synchronized void register(long fileId, long tableId, int virtualNodeId, + long firstBlockId) throws RetinaException + { + Entry entry = new Entry(fileId, tableId, virtualNodeId, firstBlockId); + Entry existing = entriesByFileId.get(fileId); + if (existing != null) + { + if (existing.equals(entry)) + { + return; + } + throw new RetinaException("Conflicting ingest metadata registration for fileId=" + fileId); + } + + StreamKey streamKey = new StreamKey(tableId, virtualNodeId); + List streamEntries = entriesByStream.get(streamKey); + if (streamEntries != null && !streamEntries.isEmpty()) + { + Entry tail = streamEntries.get(streamEntries.size() - 1); + if (firstBlockId <= tail.getFirstBlockId()) + { + throw new RetinaException("Out-of-order ingest metadata registration for fileId=" + fileId + + ": firstBlockId=" + firstBlockId + + " must be strictly greater than prior tail firstBlockId=" + tail.getFirstBlockId()); + } + } + if (streamEntries == null) + { + streamEntries = new ArrayList<>(); + entriesByStream.put(streamKey, streamEntries); + } + entriesByFileId.put(fileId, entry); + streamEntries.add(entry); + } + + synchronized void unregister(long fileId) + { + Entry removed = entriesByFileId.remove(fileId); + if (removed == null) + { + return; + } + StreamKey streamKey = new StreamKey(removed.getTableId(), removed.getVirtualNodeId()); + List streamEntries = entriesByStream.get(streamKey); + if (streamEntries == null) + { + return; + } + streamEntries.removeIf(entry -> entry.getFileId() == fileId); + if (streamEntries.isEmpty()) + { + entriesByStream.remove(streamKey); + } + } + + synchronized Entry get(long fileId) throws RetinaException + { + Entry entry = entriesByFileId.get(fileId); + if (entry == null) + { + throw new RetinaException("Missing ingest metadata for fileId=" + fileId); + } + return entry; + } + + synchronized boolean contains(long fileId) + { + return entriesByFileId.containsKey(fileId); + } + + synchronized List listByStream(long tableId, int virtualNodeId) + { + List streamEntries = entriesByStream.get(new StreamKey(tableId, virtualNodeId)); + if (streamEntries == null) + { + return Collections.emptyList(); + } + return Collections.unmodifiableList(new ArrayList<>(streamEntries)); + } + + static final class Entry + { + private final long fileId; + private final long tableId; + private final int virtualNodeId; + private final long firstBlockId; + + Entry(long fileId, long tableId, int virtualNodeId, long firstBlockId) + { + this.fileId = fileId; + this.tableId = tableId; + this.virtualNodeId = virtualNodeId; + this.firstBlockId = firstBlockId; + } + + long getFileId() + { + return this.fileId; + } + + long getTableId() + { + return this.tableId; + } + + int getVirtualNodeId() + { + return this.virtualNodeId; + } + + long getFirstBlockId() + { + return this.firstBlockId; + } + + @Override + public boolean equals(Object o) + { + if (this == o) + { + return true; + } + if (!(o instanceof Entry)) + { + return false; + } + Entry entry = (Entry) o; + return fileId == entry.fileId && tableId == entry.tableId && + virtualNodeId == entry.virtualNodeId && firstBlockId == entry.firstBlockId; + } + + @Override + public int hashCode() + { + return Objects.hash(fileId, tableId, virtualNodeId, firstBlockId); + } + } + + private static final class StreamKey + { + private final long tableId; + private final int virtualNodeId; + + private StreamKey(long tableId, int virtualNodeId) + { + this.tableId = tableId; + this.virtualNodeId = virtualNodeId; + } + + @Override + public boolean equals(Object o) + { + if (this == o) + { + return true; + } + if (!(o instanceof StreamKey)) + { + return false; + } + StreamKey streamKey = (StreamKey) o; + return tableId == streamKey.tableId && virtualNodeId == streamKey.virtualNodeId; + } + + @Override + public int hashCode() + { + return Objects.hash(tableId, virtualNodeId); + } + } +} diff --git a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/IngestFilePublisher.java b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/IngestFilePublisher.java new file mode 100644 index 0000000000..2fa7ccf503 --- /dev/null +++ b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/IngestFilePublisher.java @@ -0,0 +1,94 @@ +/* + * Copyright 2026 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the Affero GNU General Public + * License along with Pixels. If not, see + * . + */ +package io.pixelsdb.pixels.retina; + +import io.pixelsdb.pixels.common.exception.RetinaException; + +import java.util.ArrayList; +import java.util.List; +import java.util.NavigableMap; +import java.util.TreeMap; + +/** + * Publishes prepared ingest files in stream-append order. + *

+ * The scheduled fast-path inside {@link PixelsWriteBuffer} already drains + * {@code fileWriterManagers} in FIFO order on a single thread, so admission + * naturally arrives sorted by {@code firstBlockId}. This class is what keeps + * the ordering invariant intact on the {@code close()} path, where multiple + * drivers (the scheduler and the buffer's close thread) may race to admit + * the same manager: every publish action runs synchronously inside the + * monitor, and admissions whose predecessor has not yet been published are + * parked in {@link #readyFiles} until the head of the run is publishable. + */ +final class IngestFilePublisher +{ + interface PublishAction + { + void publish(FileWriterManager fileWriterManager) throws RetinaException; + } + + private final NavigableMap readyFiles = new TreeMap<>(); + private long nextCommitFirstBlockId; + + IngestFilePublisher(long nextCommitFirstBlockId) + { + this.nextCommitFirstBlockId = nextCommitFirstBlockId; + } + + synchronized List admitReady(FileWriterManager fileWriterManager, + PublishAction publishAction) throws RetinaException + { + long firstBlockId = fileWriterManager.getFirstBlockId(); + if (firstBlockId < this.nextCommitFirstBlockId) + { + // Already published in a previous admission. Re-admission is a + // benign no-op so that callers (the scheduler and the close() + // driver) can both attempt to publish without coordinating. + return new ArrayList<>(); + } + + FileWriterManager existing = this.readyFiles.putIfAbsent(firstBlockId, fileWriterManager); + if (existing != null && existing != fileWriterManager) + { + throw new RetinaException("Conflicting ingest file publisher admission for firstBlockId=" + firstBlockId); + } + + return publishReadyPrefix(publishAction); + } + + private List publishReadyPrefix(PublishAction publishAction) throws RetinaException + { + List published = new ArrayList<>(); + while (true) + { + FileWriterManager next = this.readyFiles.get(this.nextCommitFirstBlockId); + if (next == null) + { + return published; + } + + publishAction.publish(next); + this.readyFiles.remove(this.nextCommitFirstBlockId); + this.nextCommitFirstBlockId = next.getLastBlockId() + 1; + published.add(next); + } + } +} diff --git a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/MemTable.java b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/MemTable.java index e3d3004296..cefa83c90f 100644 --- a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/MemTable.java +++ b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/MemTable.java @@ -21,6 +21,7 @@ import io.pixelsdb.pixels.common.exception.RetinaException; import io.pixelsdb.pixels.core.TypeDescription; +import io.pixelsdb.pixels.core.vector.LongColumnVector; import io.pixelsdb.pixels.core.vector.VectorizedRowBatch; /** @@ -93,6 +94,38 @@ public int getLength() return this.length; } + public synchronized int getSize() + { + return this.rowBatch.size; + } + + /** + * Minimum commit timestamp over the appended rows, derived from the hidden + * timestamp column. Same-stream input is monotonically increasing by + * contract, so the first appended row carries the minimum. + */ + public synchronized long getMinCommitTs() + { + if (this.rowBatch.size == 0) + { + return Long.MAX_VALUE; + } + return ((LongColumnVector) this.rowBatch.cols[this.schema.getChildren().size()]).vector[0]; + } + + /** + * Maximum commit timestamp over the appended rows, derived from the hidden + * timestamp column rather than a separately maintained field. + */ + public synchronized long getMaxCommitTs() + { + if (this.rowBatch.size == 0) + { + return Long.MIN_VALUE; + } + return ((LongColumnVector) this.rowBatch.cols[this.schema.getChildren().size()]).vector[this.rowBatch.size - 1]; + } + public VectorizedRowBatch getRowBatch() { return this.rowBatch; diff --git a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/PixelsWriteBuffer.java b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/PixelsWriteBuffer.java index 1880a2be63..c80d8f9e22 100644 --- a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/PixelsWriteBuffer.java +++ b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/PixelsWriteBuffer.java @@ -38,6 +38,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.io.IOException; import java.util.*; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicLong; @@ -112,6 +113,7 @@ public class PixelsWriteBuffer private int currentMemTableCount; private final Queue fileWriterManagers; private FileWriterManager currentFileWriterManager; + private IngestFilePublisher ingestFilePublisher; /** * Issue #1254: Multi-threaded flush @@ -177,6 +179,7 @@ public PixelsWriteBuffer(long tableId, TypeDescription schema, Path targetOrdere this.targetOrderedStorage, this.memTableSize, this.blockSize, this.replication, this.encodingLevel, this.nullsPadding, idCounter, this.memTableSize * this.maxMemTableCount, retinaHostName, virtualNodeId); + this.ingestFilePublisher = new IngestFilePublisher(this.currentFileWriterManager.getFirstBlockId()); this.activeMemTable = new MemTable(this.idCounter, schema, memTableSize, TypeDescription.Mode.CREATE_INT_VECTOR_FOR_INT, this.currentFileWriterManager.getFileId(), @@ -192,12 +195,17 @@ public PixelsWriteBuffer(long tableId, TypeDescription schema, Path targetOrdere } /** - * Add all column values and timestamp into the buffer. + * Append a row to the active memTable atomically. On return the row is + * query-visible and {@code builder} is populated with its + * {@link IndexProto.RowLocation} for downstream MainIndex / primary index + * writes. If those writes fail, the caller MUST compensate by writing an + * RGVisibility delete on that RowLocation; do not try to rewind the append. * - * @param values - * @param timestamp - * @param builder - * @return the unique row identifier (rowId) allocated for the added row + * @param values the column values of the row. + * @param timestamp the commit timestamp of the row. + * @param builder the builder of the row location, populated on return. + * @return the allocated rowId. + * @throws RetinaException if the buffer is fail-closed or rowId allocation fails. */ public long addRow(byte[][] values, long timestamp, IndexProto.RowLocation.Builder builder) throws RetinaException { @@ -209,15 +217,19 @@ public long addRow(byte[][] values, long timestamp, IndexProto.RowLocation.Build long rowId = -1; while (rowOffset < 0) { - currentMemTable = this.activeMemTable; try { synchronized (rowLock) { - // Ensure rgRowOffset and rowId are allocated synchronously to minimize - // fragmentation after MainIndex flush. + currentMemTable = this.activeMemTable; + FileWriterManager appendFileWriterManager = this.currentFileWriterManager; + // Keep row offsets and row IDs aligned for index flush. rowOffset = currentMemTable.add(values, timestamp); - rowId = rowIdAllocator.getRowId(); + if (rowOffset >= 0) + { + rowId = rowIdAllocator.getRowId(); + appendFileWriterManager.includeRowId(rowId); + } } } catch (NullPointerException e) { @@ -234,7 +246,7 @@ public long addRow(byte[][] values, long timestamp, IndexProto.RowLocation.Build } } int rgRowOffset = currentMemTable.getStartIndex() + rowOffset; - if(rgRowOffset < 0) + if (rgRowOffset < 0) { throw new RetinaException("Expect rgRowOffset >= 0, get " + rgRowOffset); } @@ -253,39 +265,7 @@ private void switchMemTable() throws RetinaException { return; } - - if (this.currentMemTableCount >= this.maxMemTableCount) - { - this.currentMemTableCount = 0; - this.currentFileWriterManager.setLastBlockId(this.activeMemTable.getId()); - this.fileWriterManagers.add(this.currentFileWriterManager); - this.currentFileWriterManager = new FileWriterManager( - this.tableId, this.schema, - this.targetOrderedDirPath, this.targetOrderedStorage, - this.memTableSize, this.blockSize, this.replication, - this.encodingLevel, this.nullsPadding, this.idCounter, - this.memTableSize * this.maxMemTableCount, this.retinaHostName, virtualNodeId); - } - - /* - * For activeMemTable, at initialization the reference count is 2 because of *this and superVersion - * Here only currentVersion is destroyed, *this is still in use, so only one call to unref() is needed. - */ - MemTable oldMemTable = this.activeMemTable; - SuperVersion oldVersion = this.currentVersion; - this.immutableMemTables.add(this.activeMemTable); - this.activeMemTable = new MemTable(this.idCounter, this.schema, - this.memTableSize, TypeDescription.Mode.CREATE_INT_VECTOR_FOR_INT, - this.currentFileWriterManager.getFileId(), - this.currentMemTableCount * this.memTableSize, - this.memTableSize); - this.currentMemTableCount += 1; - this.idCounter++; - - this.currentVersion = new SuperVersion(this.activeMemTable, this.immutableMemTables, this.objectEntries); - oldVersion.unref(); - - triggerFlushToObject(oldMemTable); + retireActiveMemTableLocked(); } catch (Exception e) { throw new RetinaException("Failed to switch memtable", e); @@ -295,6 +275,43 @@ private void switchMemTable() throws RetinaException } } + // Caller must hold versionLock.writeLock(). + private void retireActiveMemTableLocked() throws RetinaException + { + if (this.currentMemTableCount >= this.maxMemTableCount) + { + this.currentMemTableCount = 0; + this.currentFileWriterManager.setLastBlockId(this.activeMemTable.getId()); + this.fileWriterManagers.add(this.currentFileWriterManager); + this.currentFileWriterManager = new FileWriterManager( + this.tableId, this.schema, + this.targetOrderedDirPath, this.targetOrderedStorage, + this.memTableSize, this.blockSize, this.replication, + this.encodingLevel, this.nullsPadding, this.idCounter, + this.memTableSize * this.maxMemTableCount, this.retinaHostName, virtualNodeId); + } + + /* + * For activeMemTable, at initialization the reference count is 2 because of *this and currentVersion + * Here only currentVersion is destroyed, *this is still in use, so only one call to unref() is needed. + */ + MemTable oldMemTable = this.activeMemTable; + SuperVersion oldVersion = this.currentVersion; + this.immutableMemTables.add(this.activeMemTable); + this.activeMemTable = new MemTable(this.idCounter, this.schema, + this.memTableSize, TypeDescription.Mode.CREATE_INT_VECTOR_FOR_INT, + this.currentFileWriterManager.getFileId(), + this.currentMemTableCount * this.memTableSize, + this.memTableSize); + this.currentMemTableCount += 1; + this.idCounter++; + + this.currentVersion = new SuperVersion(this.activeMemTable, this.immutableMemTables, this.objectEntries); + oldVersion.unref(); + + triggerFlushToObject(oldMemTable); + } + private void triggerFlushToObject(MemTable flushMemTable) { flushObjectExecutor.submit(() -> { @@ -305,7 +322,7 @@ private void triggerFlushToObject(MemTable flushMemTable) this.objectStorageManager.write(this.tableId, virtualNodeId, id, flushMemTable.serialize()); ObjectEntry objectEntry = new ObjectEntry(id, flushMemTable.getFileId(), - flushMemTable.getStartIndex(), flushMemTable.getLength()); + flushMemTable.getStartIndex(), flushMemTable.getSize()); objectEntry.ref(); // update watermark @@ -370,30 +387,63 @@ public SuperVersion getCurrentVersion() } } - private void publishFinishedFile(FileWriterManager fileWriterManager) throws RetinaException + private List publishFinishedFile(FileWriterManager fileWriterManager) throws RetinaException { try { - fileWriterManager.finish().get(); + fileWriterManager.finish(); - if (this.index == null) + if (!fileWriterManager.isIndexFlushed()) { - this.index = MetadataService.Instance().getPrimaryIndex(tableId); if (this.index == null) { - throw new RetinaException("Primary index not found for table " + tableId); + this.index = MetadataService.Instance().getPrimaryIndex(tableId); + if (this.index == null) + { + throw new RetinaException("Primary index not found for table " + tableId); + } } + + boolean flushed = IndexServiceProvider.getService(IndexServiceProvider.ServiceMode.local) + .flushIndexEntriesOfFile( + tableId, index.getId(), fileWriterManager.getFileId(), true, indexOption); + if (!flushed) + { + throw new RetinaException("Failed to flush main index for ingest file " + + fileWriterManager.getFileId()); + } + fileWriterManager.markIndexFlushed(); } + } catch (IndexException e) + { + throw new RetinaException("Failed to flush main index for ingest file " + + fileWriterManager.getFileId(), e); + } catch (MetadataException e) + { + throw new RetinaException("Failed to load primary index for table " + tableId, e); + } + return this.ingestFilePublisher.admitReady(fileWriterManager, this::publishPreparedFile); + } - boolean flushed = IndexServiceProvider.getService(IndexServiceProvider.ServiceMode.local) - .flushIndexEntriesOfFile( - tableId, index.getId(), fileWriterManager.getFileId(), true, indexOption); - if (!flushed) + private void publishPreparedFile(FileWriterManager fileWriterManager) throws RetinaException + { + try + { + if (!fileWriterManager.isPhysicalClosed()) { - throw new RetinaException("Failed to flush main index for ingest file " + throw new RetinaException("Cannot publish ingest file before physical close: fileId=" + + fileWriterManager.getFileId()); + } + if (!fileWriterManager.isIndexFlushed()) + { + throw new RetinaException("Cannot publish ingest file before main index flush: fileId=" + + fileWriterManager.getFileId()); + } + if (!fileWriterManager.hasRowIds()) + { + throw new RetinaException("Cannot publish ingest file without row-id hull: fileId=" + fileWriterManager.getFileId()); } - File regularFile = fileWriterManager.getFileSnapshot(); regularFile.setType(File.Type.REGULAR); if (!MetadataService.Instance().updateFile(regularFile)) @@ -401,19 +451,9 @@ private void publishFinishedFile(FileWriterManager fileWriterManager) throws Ret throw new RetinaException("Failed to publish ingest file " + fileWriterManager.getFileId() + " as REGULAR"); } - } catch (InterruptedException e) - { - Thread.currentThread().interrupt(); - throw new RetinaException("Interrupted while publishing ingest file " - + fileWriterManager.getFileId(), e); - } catch (ExecutionException e) - { - throw new RetinaException("Failed to physically close ingest file " - + fileWriterManager.getFileId(), e.getCause()); - } catch (IndexException e) - { - throw new RetinaException("Failed to flush main index for ingest file " - + fileWriterManager.getFileId(), e); + RetinaResourceManager.Instance().registerIngestFileMetadata( + fileWriterManager.getFileId(), tableId, fileWriterManager.getVirtualNodeId(), + fileWriterManager.getFirstBlockId()); } catch (MetadataException e) { throw new RetinaException("Failed to publish ingest file " @@ -435,45 +475,15 @@ private void startFlushObjectToFileScheduler(long intervalSeconds) while (iterator.hasNext()) { FileWriterManager fileWriterManager = iterator.next(); - if (fileWriterManager.getLastBlockId() <= this.continuousFlushedId.get()) + if (fileWriterManager.getLastBlockId() > this.continuousFlushedId.get()) { - publishFinishedFile(fileWriterManager); - - /* - * Detach only the current write-buffer view while holding versionLock. - * Physical object deletion stays outside the lock so storage I/O does - * not run under the SuperVersion write lock. - */ - List toRemove; - this.versionLock.writeLock().lock(); - try - { - long firstBlockId = fileWriterManager.getFirstBlockId(); - long lastBlockId = fileWriterManager.getLastBlockId(); - toRemove = this.objectEntries.stream() - .filter(objectEntry -> - objectEntry.getId() >= firstBlockId && objectEntry.getId() <= lastBlockId) - .collect(Collectors.toList()); - - this.objectEntries.removeAll(toRemove); - - SuperVersion oldVersion = this.currentVersion; - this.currentVersion = new SuperVersion( - this.activeMemTable, this.immutableMemTables, this.objectEntries); - oldVersion.unref(); - } finally - { - this.versionLock.writeLock().unlock(); - } - - iterator.remove(); - for (ObjectEntry objectEntry : toRemove) - { - if (objectEntry.unref()) - { - this.objectStorageManager.delete(this.tableId, virtualNodeId, objectEntry.getId()); - } - } + break; + } + List publishedFiles = publishFinishedFile(fileWriterManager); + for (FileWriterManager publishedFile : publishedFiles) + { + this.fileWriterManagers.remove(publishedFile); + cleanupPublishedObjects(publishedFile.getFirstBlockId(), publishedFile.getLastBlockId()); } } } catch (Exception e) @@ -483,25 +493,46 @@ private void startFlushObjectToFileScheduler(long intervalSeconds) }, 0, intervalSeconds, TimeUnit.SECONDS); } - /** - * Gracefully close the writer buffer, ensuring all in-memory data is persisted. - */ - public void close() throws RetinaException + private void cleanupPublishedObjects(long firstBlockId, long lastBlockId) throws RetinaException { - // First, shut down the flush process to prevent changes to the data view. - this.flushObjectExecutor.shutdown(); + if (lastBlockId < firstBlockId) + { + return; + } + + List toRemove; + this.versionLock.writeLock().lock(); try { - if (!this.flushObjectExecutor.awaitTermination(60, TimeUnit.SECONDS)) + toRemove = this.objectEntries.stream() + .filter(objectEntry -> objectEntry.getId() >= firstBlockId && objectEntry.getId() <= lastBlockId) + .collect(Collectors.toList()); + this.objectEntries.removeAll(toRemove); + + SuperVersion oldVersion = this.currentVersion; + this.currentVersion = new SuperVersion( + this.activeMemTable, this.immutableMemTables, this.objectEntries); + oldVersion.unref(); + } finally + { + this.versionLock.writeLock().unlock(); + } + + for (ObjectEntry objectEntry : toRemove) + { + if (objectEntry.unref()) { - this.flushObjectExecutor.shutdownNow(); + this.objectStorageManager.delete(this.tableId, virtualNodeId, objectEntry.getId()); } - } catch (InterruptedException e) - { - this.flushObjectExecutor.shutdownNow(); - Thread.currentThread().interrupt(); - throw new RetinaException("Close process was interrupted while waiting for flushObjectExecutor", e); } + } + + public void close() throws RetinaException + { + // The caller (RetinaServer / RetinaResourceManager shutdown path) is + // responsible for quiescing append traffic before invoking close(). + // There is no buffer-internal "append-to-publish" window to drain. + // Stop scheduled publishing before the driver thread publishes leftovers. if (this.flushFileFuture != null) { this.flushFileFuture.cancel(false); @@ -511,88 +542,102 @@ public void close() throws RetinaException { if (!this.flushFileExecutor.awaitTermination(60, TimeUnit.SECONDS)) { - this.flushFileExecutor.shutdownNow(); + logger.warn("Close timed out waiting for flushFileExecutor to drain; proceeding"); } - } catch (InterruptedException e) + } + catch (InterruptedException e) { - this.flushFileExecutor.shutdownNow(); Thread.currentThread().interrupt(); - throw new RetinaException("Close process was interrupted while waiting for flushDiskExecutor", e); + throw new RetinaException("Close process was interrupted while waiting for flushFileExecutor", e); } - SuperVersion sv = getCurrentVersion(); - boolean completed = false; + // Retire non-empty active data so file close only replays ObjectEntry bytes. + this.versionLock.writeLock().lock(); try { - long maxObjectKey = this.continuousFlushedId.get(); - - // process current fileWriterManager - this.currentFileWriterManager.setLastBlockId(maxObjectKey); - this.currentFileWriterManager.addRowBatch(sv.getActiveMemTable().getRowBatch()); - long firstBlockId = this.currentFileWriterManager.getFirstBlockId(); - Iterator iterator = sv.getImmutableMemTables().iterator(); - while (iterator.hasNext()) + if (!this.activeMemTable.isEmpty()) { - MemTable immutableMemtable = iterator.next(); - if (immutableMemtable.getId() >= firstBlockId) - { - this.currentFileWriterManager.addRowBatch(immutableMemtable.getRowBatch()); - iterator.remove(); - } + retireActiveMemTableLocked(); } - publishFinishedFile(this.currentFileWriterManager); + } + finally + { + this.versionLock.writeLock().unlock(); + } - // process the remaining fileWriterManager - for (FileWriterManager fileWriterManager : this.fileWriterManagers) + // Let submitted object flushes finish; never interrupt in-flight uploads. + this.flushObjectExecutor.shutdown(); + try + { + if (!this.flushObjectExecutor.awaitTermination(60, TimeUnit.SECONDS)) { - firstBlockId = fileWriterManager.getFirstBlockId(); - long lastBlockId = fileWriterManager.getLastBlockId(); + logger.warn("Close timed out waiting for flushObjectExecutor to drain; proceeding"); + } + } + catch (InterruptedException e) + { + Thread.currentThread().interrupt(); + throw new RetinaException("Close process was interrupted while waiting for flushObjectExecutor", e); + } - // all written to object - if (lastBlockId <= maxObjectKey) - { - publishFinishedFile(fileWriterManager); - } else + // Publish files with rows; discard an empty current ingest file. + if (this.currentFileWriterManager != null) + { + if (this.currentFileWriterManager.hasRowIds()) + { + this.currentFileWriterManager.setLastBlockId(this.continuousFlushedId.get()); + this.fileWriterManagers.add(this.currentFileWriterManager); + } + else + { + FileWriterManager zeroDataFwm = this.currentFileWriterManager; + String filePath = this.targetOrderedDirPath.getUri() + "/" + + zeroDataFwm.getFileName(); + try { - // process elements in immutable memTable - iterator = sv.getImmutableMemTables().iterator(); - while (iterator.hasNext()) + if (this.targetOrderedStorage.exists(filePath)) { - MemTable immutableMemtable = iterator.next(); - long id = immutableMemtable.getId(); - if (id >= firstBlockId && id <= lastBlockId) - { - fileWriterManager.addRowBatch(immutableMemtable.getRowBatch()); - iterator.remove(); - } + this.targetOrderedStorage.delete(filePath, false); } - - // elements in object will be processed in finish() later - fileWriterManager.setLastBlockId(maxObjectKey); - publishFinishedFile(fileWriterManager); + } + catch (IOException e) + { + logger.warn("Close failed to delete half-written bytes of empty FileWriterManager fileId={}, path={}; continuing", + zeroDataFwm.getFileId(), filePath, e); + } + try + { + zeroDataFwm.discard(); + } + catch (RetinaException e) + { + logger.warn("Close failed to discard empty current FileWriterManager fileId={}; continuing", + zeroDataFwm.getFileId(), e); } } - completed = true; - } catch (Exception e) - { - throw new RetinaException("Failed to persist data during close operation. Data may be lost", e); - } finally - { - sv.unref(); - currentVersion.unref(); - activeMemTable.unref(); - for (MemTable immutableMemTable: sv.getImmutableMemTables()) - { - immutableMemTable.unref(); - } + this.currentFileWriterManager = null; + } - for (ObjectEntry objectEntry : sv.getObjectEntries()) + SuperVersion sv = getCurrentVersion(); + try + { + for (FileWriterManager fwm : new ArrayList<>(this.fileWriterManagers)) { - if (objectEntry.unref() && completed) + List published = publishFinishedFile(fwm); + for (FileWriterManager publishedFile : published) { - this.objectStorageManager.delete(this.tableId, virtualNodeId, objectEntry.getId()); + this.fileWriterManagers.remove(publishedFile); + cleanupPublishedObjects(publishedFile.getFirstBlockId(), publishedFile.getLastBlockId()); } } } + catch (Exception e) + { + throw new RetinaException("Failed to publish ingest files during close", e); + } + finally + { + sv.unref(); + } } } diff --git a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/RetinaResourceManager.java b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/RetinaResourceManager.java index 900d907b63..857f12b5d8 100644 --- a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/RetinaResourceManager.java +++ b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/RetinaResourceManager.java @@ -60,9 +60,11 @@ public class RetinaResourceManager { private static final Logger logger = LogManager.getLogger(RetinaResourceManager.class); + private final MetadataService metadataService; private final Map rgVisibilityMap; private final Map> pixelsWriteBufferMap; + private final IngestFileMetadataRegistry ingestFileMetadataRegistry; private String retinaHostName; // GC related fields @@ -136,6 +138,7 @@ private RetinaResourceManager() this.metadataService = MetadataService.Instance(); this.rgVisibilityMap = new ConcurrentHashMap<>(); this.pixelsWriteBufferMap = new ConcurrentHashMap<>(); + this.ingestFileMetadataRegistry = new IngestFileMetadataRegistry(); this.offloadedCheckpoints = new ConcurrentHashMap<>(); this.checkpointFutures = new ConcurrentHashMap<>(); @@ -200,12 +203,14 @@ public static RetinaResourceManager Instance() } /** - * Starts the periodic Retina GC scheduler after the service has reached the - * lifecycle point where background cleanup is safe to run. + * Starts the periodic Retina GC scheduler after the service has reached + * the lifecycle point where background cleanup is safe to run. * - *

The constructor intentionally does not schedule GC: recovery-capable - * startup must stay fail-closed until initialization succeeds. This method is - * idempotent so future lifecycle READY hooks can call it safely.

+ *

The constructor intentionally does not schedule GC: startup must + * stay fail-closed until initialization succeeds, otherwise a background + * GC tick could observe partially constructed state. This method is + * idempotent so callers that wire it into a service-ready hook can + * invoke it more than once safely.

* * @throws RetinaException if GC configuration is invalid or the scheduler cannot be started. */ @@ -298,6 +303,24 @@ public void addVisibility(String filePath) throws RetinaException } } + public void removeVisibility(long fileId) + { + String prefix = fileId + "_"; + this.rgVisibilityMap.entrySet().removeIf(entry -> + { + if (!entry.getKey().startsWith(prefix)) + { + return false; + } + RGVisibility rgVisibility = entry.getValue(); + if (rgVisibility != null) + { + rgVisibility.close(); + } + return true; + }); + } + public long[] queryVisibility(long fileId, int rgId, long timestamp, long transId) throws RetinaException { // read from memory @@ -412,6 +435,36 @@ private CompletableFuture createCheckpoint(long timestamp, CheckpointType return createCheckpoint(timestamp, type, null); } + void registerIngestFileMetadata(long fileId, long tableId, int virtualNodeId, + long firstBlockId) throws RetinaException + { + this.ingestFileMetadataRegistry.register(fileId, tableId, virtualNodeId, firstBlockId); + } + + void unregisterIngestFileMetadata(long fileId) + { + this.ingestFileMetadataRegistry.unregister(fileId); + } + + IngestFileMetadataRegistry.Entry getIngestFileMetadata(long fileId) throws RetinaException + { + return this.ingestFileMetadataRegistry.get(fileId); + } + + List listIngestFileMetadataByStream(long tableId, int virtualNodeId) + { + return this.ingestFileMetadataRegistry.listByStream(tableId, virtualNodeId); + } + + void validateRgVisibilityFileRegistered(long fileId) throws RetinaException + { + if (!this.ingestFileMetadataRegistry.contains(fileId)) + { + throw new RetinaException("RGVisibilityIndex contains fileId=" + fileId + + " but registry has no entry, indicating publisher/retire ordering bug"); + } + } + private CompletableFuture createCheckpoint( long timestamp, CheckpointType type, Map precomputedBitmaps) throws RetinaException { @@ -420,6 +473,11 @@ private CompletableFuture createCheckpoint( // 1. Capture current entries to ensure we process a consistent set of RGs List> entries = new ArrayList<>(this.rgVisibilityMap.entrySet()); + for (Map.Entry entry : entries) + { + long fileId = RetinaUtils.parseFileIdFromRgKey(entry.getKey()); + validateRgVisibilityFileRegistered(fileId); + } int totalRgs = entries.size(); logger.info("Starting {} checkpoint for {} RGs at timestamp {}", type, totalRgs, timestamp); @@ -779,7 +837,9 @@ public void addWriteBuffer(String schemaName, String tableName) throws RetinaExc } } - public IndexProto.PrimaryIndexEntry.Builder insertRecord(String schemaName, String tableName, byte[][] colValues, long timestamp, int vNodeId) throws RetinaException + public IndexProto.PrimaryIndexEntry.Builder insertRecord(String schemaName, String tableName, + byte[][] colValues, long timestamp, + int vNodeId) throws RetinaException { IndexProto.PrimaryIndexEntry.Builder builder = IndexProto.PrimaryIndexEntry.newBuilder(); PixelsWriteBuffer writeBuffer = checkPixelsWriteBuffer(schemaName, tableName, vNodeId); @@ -789,17 +849,18 @@ public IndexProto.PrimaryIndexEntry.Builder insertRecord(String schemaName, Stri private RetinaProto.VisibilityBitmap getVisibilityBitmapSlice(long[] visibilityBitmap, long startIndex, int length) throws RetinaException { - if (startIndex % 64 != 0 || length % 64 != 0) + if (startIndex % 64 != 0) { - throw new RetinaException("StartIndex and length must be multiple of 64"); + throw new RetinaException("StartIndex must be multiple of 64"); } - if (length == 0) + if (length <= 0) { return RetinaProto.VisibilityBitmap.newBuilder().build(); } + int alignedLength = ((length + 63) / 64) * 64; int startLongIndex = (int) (startIndex / 64); - int endLongIndex = startLongIndex + (length / 64); + int endLongIndex = startLongIndex + (alignedLength / 64); if (visibilityBitmap == null || endLongIndex > visibilityBitmap.length) { @@ -825,10 +886,12 @@ public RetinaProto.GetWriteBufferResponse.Builder getWriteBuffer(String schemaNa Set fileIds = new HashSet<>(); - // active memTable returns directly - if (!activeMemtable.getRowBatch().isEmpty()) + // Active memTable returns its full appended rows; visibility is masked + // downstream by the RGVisibility bitmap slice below. + int activeSize = activeMemtable.getSize(); + if (activeSize > 0) { - ByteString data = ByteString.copyFrom(activeMemtable.getRowBatch().serialize()); + ByteString data = ByteString.copyFrom(activeMemtable.serialize()); responseBuilder.setData(data); fileIds.add(activeMemtable.getFileId()); @@ -842,8 +905,11 @@ public RetinaProto.GetWriteBufferResponse.Builder getWriteBuffer(String schemaNa fileIds.add(activeMemtable.getFileId()); for (MemTable immutableMemtable : immutableMemTables) { - fileIds.add(immutableMemtable.getFileId()); - ids.add(immutableMemtable.getId()); + if (!immutableMemtable.isEmpty()) + { + fileIds.add(immutableMemtable.getFileId()); + ids.add(immutableMemtable.getId()); + } } for (ObjectEntry objectEntry : objectEntries) { @@ -860,21 +926,25 @@ public RetinaProto.GetWriteBufferResponse.Builder getWriteBuffer(String schemaNa fileIdToVisibility.put(fileId, visibility); } - // only return the corresponding part of bitmap - if (!activeMemtable.getRowBatch().isEmpty()) + // only return the corresponding visible part of bitmap + if (activeSize > 0) { responseBuilder.addBitmaps(getVisibilityBitmapSlice( fileIdToVisibility.get(activeMemtable.getFileId()), - activeMemtable.getStartIndex(), activeMemtable.getLength())); + activeMemtable.getStartIndex(), activeSize)); } else { responseBuilder.addBitmaps(RetinaProto.VisibilityBitmap.newBuilder()); } for (MemTable immutableMemtable : immutableMemTables) { - responseBuilder.addBitmaps(getVisibilityBitmapSlice( - fileIdToVisibility.get(immutableMemtable.getFileId()), - immutableMemtable.getStartIndex(), immutableMemtable.getLength())); + int immutableSize = immutableMemtable.getSize(); + if (immutableSize > 0) + { + responseBuilder.addBitmaps(getVisibilityBitmapSlice( + fileIdToVisibility.get(immutableMemtable.getFileId()), + immutableMemtable.getStartIndex(), immutableSize)); + } } for (ObjectEntry objectEntry : objectEntries) { @@ -958,16 +1028,18 @@ private PixelsWriteBuffer checkPixelsWriteBuffer(String schema, String table, in * checkpoint bitmap serialisation significantly cheaper. *
  • Checkpoint second, unconditional and blocking: written regardless of whether * Storage GC finds any candidate files. The {@code .join()} ensures the checkpoint - * file is fully on disk before Storage GC begins rewriting any files, so crash - * recovery can always restore the post-Memory-GC visibility state independently of - * any in-progress Storage GC rewrite. {@code gcExecutor} is single-threaded, so the - * blocking join is also the simplest way to guarantee no two GC cycles overlap.
    • + * file is fully on disk before Storage GC begins rewriting any files, so after a + * crash the post-Memory-GC visibility state can be rebuilt from the checkpoint + * independently of any in-progress Storage GC rewrite. {@code gcExecutor} is + * single-threaded, so the blocking join is also the simplest way to guarantee no + * two GC cycles overlap. *
  • Storage GC third: requires an up-to-date {@code baseBitmap} (hence after - * Memory GC) and its own WAL for crash recovery. Placing it after the checkpoint - * keeps the two recovery paths independent: on restart, the GC checkpoint restores - * the post-Memory-GC visibility state, and the GcWal resumes any in-progress Storage - * GC task separately. Once scan completes, bitmaps for non-candidate files are - * immediately released from memory (they are no longer needed by subsequent phases).
    • + * Memory GC) and its own WAL to resume in-progress tasks after a crash. Placing + * it after the checkpoint keeps the two restart paths independent: the GC checkpoint + * rebuilds the post-Memory-GC visibility state, and the GcWal resumes any + * in-progress Storage GC task separately. Once scan completes, bitmaps for + * non-candidate files are immediately released from memory (they are no longer + * needed by subsequent phases). *
  • Advance {@code latestGcTimestamp} last: updated only after the entire cycle * succeeds (Memory GC + checkpoint + Storage GC). If any step throws, the timestamp * is not advanced and the next scheduled invocation will retry the full cycle.
    • @@ -1007,6 +1079,8 @@ private void runGC() long fileId = RetinaUtils.parseFileIdFromRgKey(rgKey); int rgId = RetinaUtils.parseRgIdFromRgKey(rgKey); + validateRgVisibilityFileRegistered(fileId); + long[] bitmap = entry.getValue().garbageCollect(timestamp); gcSnapshotBitmaps.put(rgKey, bitmap); diff --git a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/StorageGarbageCollector.java b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/StorageGarbageCollector.java index d72ef5aaa9..bd973cf7a4 100644 --- a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/StorageGarbageCollector.java +++ b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/StorageGarbageCollector.java @@ -1203,6 +1203,7 @@ void commitFileGroup(RewriteResult result) throws Exception for (FileCandidate fc : result.group.files) { + resourceManager.unregisterIngestFileMetadata(fc.fileId); resourceManager.scheduleRetiredFile( new RetinaResourceManager.RetiredFile( fc.fileId, fc.rgCount, fc.filePath, retireDeadline, result.oldRowIds)); diff --git a/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestIngestFileMetadataRegistry.java b/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestIngestFileMetadataRegistry.java new file mode 100644 index 0000000000..ea3f55a9e3 --- /dev/null +++ b/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestIngestFileMetadataRegistry.java @@ -0,0 +1,129 @@ +/* + * Copyright 2026 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the Affero GNU General Public + * License along with Pixels. If not, see + * . + */ +package io.pixelsdb.pixels.retina; + +import io.pixelsdb.pixels.common.exception.RetinaException; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class TestIngestFileMetadataRegistry +{ + @Test + public void tracksMetadataByFileIdAndStream() throws Exception + { + IngestFileMetadataRegistry registry = new IngestFileMetadataRegistry(); + + registry.register(100L, 7L, 3, 0L); + registry.register(100L, 7L, 3, 0L); + registry.register(200L, 7L, 3, 10L); + + IngestFileMetadataRegistry.Entry entry = registry.get(100L); + assertEquals(100L, entry.getFileId()); + assertEquals(7L, entry.getTableId()); + assertEquals(3, entry.getVirtualNodeId()); + assertEquals(0L, entry.getFirstBlockId()); + + List streamEntries = registry.listByStream(7L, 3); + assertEquals(2, streamEntries.size()); + assertEquals(100L, streamEntries.get(0).getFileId()); + assertEquals(200L, streamEntries.get(1).getFileId()); + } + + @Test + public void rejectsConflictsAndUnregisters() throws Exception + { + IngestFileMetadataRegistry registry = new IngestFileMetadataRegistry(); + registry.register(100L, 7L, 3, 0L); + + try + { + registry.register(100L, 7L, 3, 1L); + fail("Expected conflicting registration to fail"); + } catch (RetinaException expected) + { + assertTrue(expected.getMessage().contains("Conflicting")); + } + + registry.unregister(100L); + assertTrue(registry.listByStream(7L, 3).isEmpty()); + assertFalse(registry.contains(100L)); + + try + { + registry.get(100L); + fail("Expected unregistered file metadata lookup to fail"); + } catch (RetinaException expected) + { + assertTrue(expected.getMessage().contains("Missing ingest metadata")); + } + } + + @Test + public void unregisterRemovesOnlyMatchingStreamEntry() throws Exception + { + IngestFileMetadataRegistry registry = new IngestFileMetadataRegistry(); + + registry.register(100L, 7L, 3, 0L); + registry.register(200L, 7L, 3, 10L); + registry.register(300L, 7L, 4, 0L); + + registry.unregister(100L); + registry.unregister(999L); + + List streamEntries = registry.listByStream(7L, 3); + assertEquals(1, streamEntries.size()); + assertEquals(200L, streamEntries.get(0).getFileId()); + assertEquals(1, registry.listByStream(7L, 4).size()); + } + + @Test + public void rejectsOutOfOrderRegistrationWithinStream() throws Exception + { + IngestFileMetadataRegistry registry = new IngestFileMetadataRegistry(); + registry.register(200L, 7L, 3, 10L); + + try + { + registry.register(100L, 7L, 3, 0L); + fail("Expected out-of-order registration to fail"); + } catch (RetinaException expected) + { + assertTrue(expected.getMessage().contains("Out-of-order")); + } + + try + { + registry.register(300L, 7L, 3, 10L); + fail("Expected non-strictly-increasing firstBlockId to fail"); + } catch (RetinaException expected) + { + assertTrue(expected.getMessage().contains("Out-of-order")); + } + + registry.register(300L, 7L, 4, 0L); + assertEquals(1, registry.listByStream(7L, 4).size()); + } +} diff --git a/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestIngestFilePublisher.java b/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestIngestFilePublisher.java deleted file mode 100644 index 817b27a1c2..0000000000 --- a/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestIngestFilePublisher.java +++ /dev/null @@ -1,664 +0,0 @@ -/* - * Copyright 2026 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ -package io.pixelsdb.pixels.retina; - -import io.pixelsdb.pixels.common.metadata.domain.File; -import io.pixelsdb.pixels.common.exception.RetinaException; -import io.pixelsdb.pixels.core.PixelsWriter; -import io.pixelsdb.pixels.core.TypeDescription; -import io.pixelsdb.pixels.core.vector.VectorizedRowBatch; -import org.junit.Test; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotSame; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -public class TestIngestFilePublisher -{ - @Test - public void finishClosesPhysicalFileOnlyOnceAndLeavesMetadataTemporary() throws Exception - { - CountingPixelsWriter writer = new CountingPixelsWriter(); - File file = temporaryFile(101L); - FileWriterManager fileWriterManager = testFileWriterManager(writer, file); - - CompletableFuture firstFinish = fileWriterManager.finish(); - CompletableFuture secondFinish = fileWriterManager.finish(); - firstFinish.get(5, TimeUnit.SECONDS); - secondFinish.get(5, TimeUnit.SECONDS); - - assertSame(firstFinish, secondFinish); - assertEquals(1, writer.closeCount.get()); - assertEquals(File.Type.TEMPORARY_INGEST, file.getType()); - assertEquals(File.Type.TEMPORARY_INGEST, fileWriterManager.getFileSnapshot().getType()); - assertTrue(firstFinish.isDone()); - assertFalse(firstFinish.isCompletedExceptionally()); - } - - @Test - public void finishFailureIsPropagatedAndDoesNotPublishMetadata() throws Exception - { - IOException closeFailure = new IOException("close failed"); - CountingPixelsWriter writer = new CountingPixelsWriter(null, null, closeFailure, null); - File file = temporaryFile(103L); - FileWriterManager fileWriterManager = testFileWriterManager(writer, file); - - CompletableFuture firstFinish = fileWriterManager.finish(); - try - { - firstFinish.get(5, TimeUnit.SECONDS); - fail("Expected physical close failure"); - } catch (ExecutionException e) - { - assertSame(closeFailure, e.getCause()); - } - - CompletableFuture secondFinish = fileWriterManager.finish(); - assertSame(firstFinish, secondFinish); - assertTrue(secondFinish.isCompletedExceptionally()); - assertEquals(1, writer.closeCount.get()); - assertEquals(File.Type.TEMPORARY_INGEST, file.getType()); - assertEquals(File.Type.TEMPORARY_INGEST, fileWriterManager.getFileSnapshot().getType()); - } - - @Test - public void fileSnapshotCopiesCurrentFileMetadata() - { - File file = temporaryFile(202L); - CountingPixelsWriter writer = new CountingPixelsWriter(); - FileWriterManager fileWriterManager = testFileWriterManager(writer, file); - - File snapshot = fileWriterManager.getFileSnapshot(); - - assertEquals(file.getId(), snapshot.getId()); - assertEquals(file.getName(), snapshot.getName()); - assertEquals(file.getType(), snapshot.getType()); - assertEquals(file.getNumRowGroup(), snapshot.getNumRowGroup()); - assertEquals(file.getMinRowId(), snapshot.getMinRowId()); - assertEquals(file.getMaxRowId(), snapshot.getMaxRowId()); - assertEquals(file.getPathId(), snapshot.getPathId()); - } - - @Test - public void fileSnapshotDoesNotExposeInternalFileState() - { - File file = temporaryFile(203L); - CountingPixelsWriter writer = new CountingPixelsWriter(); - FileWriterManager fileWriterManager = testFileWriterManager(writer, file); - File snapshot = fileWriterManager.getFileSnapshot(); - - snapshot.setName("published.pxl"); - snapshot.setType(File.Type.REGULAR); - snapshot.setNumRowGroup(99); - snapshot.setMinRowId(1000); - snapshot.setMaxRowId(2000); - snapshot.setPathId(88L); - - File freshSnapshot = fileWriterManager.getFileSnapshot(); - assertEquals("ingest_203.pxl", file.getName()); - assertEquals(File.Type.TEMPORARY_INGEST, file.getType()); - assertEquals(1, file.getNumRowGroup()); - assertEquals(0, file.getMinRowId()); - assertEquals(63, file.getMaxRowId()); - assertEquals(9L, file.getPathId()); - assertEquals(file.getName(), freshSnapshot.getName()); - assertEquals(file.getType(), freshSnapshot.getType()); - assertEquals(file.getNumRowGroup(), freshSnapshot.getNumRowGroup()); - assertEquals(file.getMinRowId(), freshSnapshot.getMinRowId()); - assertEquals(file.getMaxRowId(), freshSnapshot.getMaxRowId()); - assertEquals(file.getPathId(), freshSnapshot.getPathId()); - } - - @Test - public void fileSnapshotReflectsMutationsOnUnderlyingFile() - { - File file = temporaryFile(205L); - CountingPixelsWriter writer = new CountingPixelsWriter(); - FileWriterManager fileWriterManager = testFileWriterManager(writer, file); - - File before = fileWriterManager.getFileSnapshot(); - assertEquals(File.Type.TEMPORARY_INGEST, before.getType()); - assertEquals(63L, before.getMaxRowId()); - - // Mutations on the underlying file (e.g. visibility/row id updates) must be observed - // by snapshots taken afterwards. Snapshots taken earlier must remain unchanged. - file.setMaxRowId(127L); - file.setNumRowGroup(2); - - File after = fileWriterManager.getFileSnapshot(); - assertEquals(127L, after.getMaxRowId()); - assertEquals(2, after.getNumRowGroup()); - // The previously taken snapshot must keep its original values. - assertEquals(63L, before.getMaxRowId()); - assertEquals(1, before.getNumRowGroup()); - } - - @Test - public void gettersExposeConstructorArguments() - { - File file = temporaryFile(301L); - CountingPixelsWriter writer = new CountingPixelsWriter(); - FileWriterManager fileWriterManager = new FileWriterManager(7L, writer, file, 5L, 10L, 0); - - assertEquals(file.getId(), fileWriterManager.getFileId()); - assertEquals(5L, fileWriterManager.getFirstBlockId()); - assertEquals(10L, fileWriterManager.getLastBlockId()); - } - - @Test - public void setLastBlockIdUpdatesGetter() - { - File file = temporaryFile(302L); - CountingPixelsWriter writer = new CountingPixelsWriter(); - FileWriterManager fileWriterManager = new FileWriterManager(1L, writer, file, 0L, 0L, 0); - - fileWriterManager.setLastBlockId(42L); - assertEquals(42L, fileWriterManager.getLastBlockId()); - - // Allow lowering as well, e.g. when shrinking the range during close(). - fileWriterManager.setLastBlockId(-1L); - assertEquals(-1L, fileWriterManager.getLastBlockId()); - } - - @Test - public void addRowBatchSucceedsAndForwardsToWriter() throws Exception - { - CountingPixelsWriter writer = new CountingPixelsWriter(); - File file = temporaryFile(401L); - FileWriterManager fileWriterManager = testFileWriterManager(writer, file); - - fileWriterManager.addRowBatch(null); - fileWriterManager.addRowBatch(null); - fileWriterManager.addRowBatch(null); - - assertEquals(3, writer.addRowBatchCount.get()); - assertEquals(0, writer.closeCount.get()); - assertEquals(File.Type.TEMPORARY_INGEST, file.getType()); - } - - @Test - public void addRowBatchFailureLeavesManagerUsableForFinish() throws Exception - { - IOException writeFailure = new IOException("write failed"); - CountingPixelsWriter writer = new CountingPixelsWriter(null, null, null, writeFailure); - File file = temporaryFile(402L); - FileWriterManager fileWriterManager = testFileWriterManager(writer, file); - - try - { - fileWriterManager.addRowBatch(null); - fail("Expected row batch write failure"); - } catch (RetinaException e) - { - assertSame(writeFailure, e.getCause()); - } - - // After a failed addRowBatch, finish() must still close the underlying writer exactly once - // and keep the file in TEMPORARY_INGEST state (publication is the buffer's responsibility). - fileWriterManager.finish().get(5, TimeUnit.SECONDS); - assertEquals(1, writer.closeCount.get()); - assertEquals(File.Type.TEMPORARY_INGEST, file.getType()); - } - - @SuppressWarnings("unchecked") - @Test - public void finishIsIdempotentUnderConcurrentCallers() throws Exception - { - CountDownLatch closeStarted = new CountDownLatch(1); - CountDownLatch allowClose = new CountDownLatch(1); - CountingPixelsWriter writer = new CountingPixelsWriter(closeStarted, allowClose, null, null); - File file = temporaryFile(501L); - FileWriterManager fileWriterManager = testFileWriterManager(writer, file); - - // Start the first finish() so the close thread is parked inside writer.close(). - CompletableFuture firstFinish = fileWriterManager.finish(); - assertTrue(closeStarted.await(5, TimeUnit.SECONDS)); - - int callerCount = 8; - ExecutorService callers = Executors.newFixedThreadPool(callerCount); - try - { - CountDownLatch readyLatch = new CountDownLatch(callerCount); - CountDownLatch startLatch = new CountDownLatch(1); - Future>[] results = new Future[callerCount]; - for (int i = 0; i < callerCount; ++i) - { - results[i] = callers.submit(() -> { - readyLatch.countDown(); - startLatch.await(); - return fileWriterManager.finish(); - }); - } - assertTrue(readyLatch.await(5, TimeUnit.SECONDS)); - startLatch.countDown(); - - for (Future> result : results) - { - CompletableFuture observed = result.get(5, TimeUnit.SECONDS); - assertSame(firstFinish, observed); - assertFalse(observed.isDone()); - } - } finally - { - allowClose.countDown(); - callers.shutdownNow(); - } - - firstFinish.get(5, TimeUnit.SECONDS); - assertEquals("writer.close() must run at most once even under concurrent finish() calls", - 1, writer.closeCount.get()); - assertEquals(File.Type.TEMPORARY_INGEST, file.getType()); - } - - @Test - public void finishRunsCloseOnDedicatedNamedThread() throws Exception - { - CountDownLatch closeStarted = new CountDownLatch(1); - CountDownLatch allowClose = new CountDownLatch(1); - ThreadCapturingPixelsWriter writer = new ThreadCapturingPixelsWriter(closeStarted, allowClose); - File file = temporaryFile(601L); - FileWriterManager fileWriterManager = new FileWriterManager(1L, writer, file, 1L, 0L, 0); - - Thread caller = Thread.currentThread(); - CompletableFuture finishFuture = fileWriterManager.finish(); - assertTrue(closeStarted.await(5, TimeUnit.SECONDS)); - - Thread closeThread = writer.closeThread; - assertNotSame("close() must run off the caller thread", caller, closeThread); - assertEquals("pixels-retina-file-finish-" + file.getId(), closeThread.getName()); - - allowClose.countDown(); - finishFuture.get(5, TimeUnit.SECONDS); - } - - @Test - public void finishPropagatesRuntimeExceptionFromClose() throws Exception - { - RuntimeException closeFailure = new RuntimeException("boom"); - CountingPixelsWriter writer = new CountingPixelsWriter(null, null, null, null, - closeFailure); - File file = temporaryFile(701L); - FileWriterManager fileWriterManager = testFileWriterManager(writer, file); - - CompletableFuture firstFinish = fileWriterManager.finish(); - try - { - firstFinish.get(5, TimeUnit.SECONDS); - fail("Expected runtime close failure"); - } catch (ExecutionException e) - { - assertSame(closeFailure, e.getCause()); - } - - // Subsequent calls must keep returning the same failed future and must not retry close(). - CompletableFuture secondFinish = fileWriterManager.finish(); - assertSame(firstFinish, secondFinish); - assertTrue(secondFinish.isCompletedExceptionally()); - assertEquals(1, writer.closeCount.get()); - assertEquals(File.Type.TEMPORARY_INGEST, file.getType()); - } - - @Test(timeout = 10_000L) - public void finishDoesNotBlockCallerThread() throws Exception - { - CountDownLatch closeStarted = new CountDownLatch(1); - CountDownLatch allowClose = new CountDownLatch(1); - CountingPixelsWriter writer = new CountingPixelsWriter(closeStarted, allowClose, null, null); - File file = temporaryFile(801L); - FileWriterManager fileWriterManager = testFileWriterManager(writer, file); - - long start = System.nanoTime(); - CompletableFuture finishFuture = fileWriterManager.finish(); - long elapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start); - - // The caller thread must return promptly; the actual close() runs on the named thread. - assertTrue("finish() must not block on writer.close(); elapsedMillis=" + elapsedMillis, - elapsedMillis < 2_000L); - assertTrue(closeStarted.await(5, TimeUnit.SECONDS)); - assertFalse(finishFuture.isDone()); - try - { - finishFuture.get(200, TimeUnit.MILLISECONDS); - fail("finish() future must not complete before writer.close() returns"); - } catch (TimeoutException expected) - { - // expected: still in progress - } - allowClose.countDown(); - finishFuture.get(5, TimeUnit.SECONDS); - } - - @Test - public void concurrentAddRowBatchesAreAllForwardedToWriter() throws Exception - { - // FileWriterManager does not perform internal locking around addRowBatch; verify it does - // not lose calls or throw NPEs when several threads forward row batches concurrently. - CountingPixelsWriter writer = new CountingPixelsWriter(); - File file = temporaryFile(1601L); - FileWriterManager fileWriterManager = testFileWriterManager(writer, file); - - int callerCount = 16; - int callsPerCaller = 25; - ExecutorService callers = Executors.newFixedThreadPool(callerCount); - try - { - CountDownLatch startLatch = new CountDownLatch(1); - List> results = new ArrayList<>(callerCount); - for (int i = 0; i < callerCount; ++i) - { - results.add(callers.submit(() -> { - startLatch.await(); - for (int j = 0; j < callsPerCaller; ++j) - { - fileWriterManager.addRowBatch(null); - } - return null; - })); - } - startLatch.countDown(); - for (Future result : results) - { - result.get(10, TimeUnit.SECONDS); - } - } finally - { - callers.shutdownNow(); - } - - assertEquals(callerCount * callsPerCaller, writer.addRowBatchCount.get()); - assertEquals(0, writer.closeCount.get()); - assertEquals(File.Type.TEMPORARY_INGEST, file.getType()); - } - - @Test - public void finishIsRobustAgainstFileMetadataMutationsBeforeReturn() throws Exception - { - // Mutations on the underlying file (e.g. visibility/row id updates by other components) - // performed while finish() is in progress must not affect the success of physical close, - // and the post-close snapshot must reflect the mutated state because publication has - // not yet rewritten file.type. - CountDownLatch closeStarted = new CountDownLatch(1); - CountDownLatch allowClose = new CountDownLatch(1); - CountingPixelsWriter writer = new CountingPixelsWriter(closeStarted, allowClose, null, null); - File file = temporaryFile(2001L); - FileWriterManager fileWriterManager = testFileWriterManager(writer, file); - - CompletableFuture finishFuture = fileWriterManager.finish(); - assertTrue(closeStarted.await(5, TimeUnit.SECONDS)); - - // Concurrently update row id bookkeeping; this is what the visibility layer does. - file.setMaxRowId(255L); - file.setNumRowGroup(3); - - allowClose.countDown(); - finishFuture.get(5, TimeUnit.SECONDS); - - File snapshot = fileWriterManager.getFileSnapshot(); - assertEquals(255L, snapshot.getMaxRowId()); - assertEquals(3, snapshot.getNumRowGroup()); - assertEquals(File.Type.TEMPORARY_INGEST, snapshot.getType()); - assertEquals(1, writer.closeCount.get()); - } - - @Test - public void addRowBatchPropagatesWriterRuntimeExceptionWithoutWrapping() throws Exception - { - // FileWriterManager only wraps IOException into RetinaException; unchecked exceptions - // (e.g. format-corruption indicators thrown by the underlying writer as RuntimeException) - // must propagate to the caller as-is so they are not silently masked. After such a failure - // the manager must remain usable and finish() must still close the writer exactly once. - RuntimeException formatFailure = new IllegalStateException("corrupted column vector"); - CountingPixelsWriter writer = new CountingPixelsWriter() - { - @Override - public boolean addRowBatch(VectorizedRowBatch rowBatch) throws IOException - { - addRowBatchCount.incrementAndGet(); - throw formatFailure; - } - }; - File file = temporaryFile(2101L); - FileWriterManager fileWriterManager = testFileWriterManager(writer, file); - - try - { - fileWriterManager.addRowBatch(null); - fail("Runtime exception from writer must propagate without being wrapped"); - } catch (RetinaException e) - { - fail("Runtime exception must not be wrapped as RetinaException, got: " + e); - } catch (IllegalStateException expected) - { - assertSame(formatFailure, expected); - } - assertEquals(1, writer.addRowBatchCount.get()); - - // After a runtime failure inside the writer, finish() must still be able to close it. - fileWriterManager.finish().get(5, TimeUnit.SECONDS); - assertEquals(1, writer.closeCount.get()); - assertEquals(File.Type.TEMPORARY_INGEST, file.getType()); - } - - private static File temporaryFile(long id) - { - File file = new File(); - file.setId(id); - file.setName("ingest_" + id + ".pxl"); - file.setType(File.Type.TEMPORARY_INGEST); - file.setNumRowGroup(1); - file.setMinRowId(0); - file.setMaxRowId(63); - file.setPathId(9L); - return file; - } - - private static FileWriterManager testFileWriterManager(CountingPixelsWriter writer, File file) - { - return new FileWriterManager(1L, writer, file, 1L, 0L, 0); - } - - private static class CountingPixelsWriter implements PixelsWriter - { - // Package-private so anonymous subclasses defined inside this test can observe call counts. - final AtomicInteger closeCount = new AtomicInteger(0); - final AtomicInteger addRowBatchCount = new AtomicInteger(0); - private final CountDownLatch closeStarted; - private final CountDownLatch allowClose; - private final IOException closeFailure; - private final IOException addRowBatchFailure; - private final RuntimeException closeRuntimeFailure; - - private CountingPixelsWriter() - { - this(null, null, null, null, null); - } - - private CountingPixelsWriter(CountDownLatch closeStarted, CountDownLatch allowClose, - IOException closeFailure, IOException addRowBatchFailure) - { - this(closeStarted, allowClose, closeFailure, addRowBatchFailure, null); - } - - private CountingPixelsWriter(CountDownLatch closeStarted, CountDownLatch allowClose, - IOException closeFailure, IOException addRowBatchFailure, - RuntimeException closeRuntimeFailure) - { - this.closeStarted = closeStarted; - this.allowClose = allowClose; - this.closeFailure = closeFailure; - this.addRowBatchFailure = addRowBatchFailure; - this.closeRuntimeFailure = closeRuntimeFailure; - } - - @Override - public boolean addRowBatch(VectorizedRowBatch rowBatch) throws IOException - { - addRowBatchCount.incrementAndGet(); - if (addRowBatchFailure != null) - { - throw addRowBatchFailure; - } - return true; - } - - @Override - public void addRowBatch(VectorizedRowBatch rowBatch, int hashValue) throws IOException - { - } - - @Override - public TypeDescription getSchema() - { - return null; - } - - @Override - public int getNumRowGroup() - { - return 0; - } - - @Override - public int getNumWriteRequests() - { - return 0; - } - - @Override - public long getCompletedBytes() - { - return 0; - } - - @Override - public void close() throws IOException - { - closeCount.incrementAndGet(); - if (closeStarted != null) - { - closeStarted.countDown(); - } - if (allowClose != null) - { - try - { - assertTrue(allowClose.await(5, TimeUnit.SECONDS)); - } catch (InterruptedException e) - { - Thread.currentThread().interrupt(); - throw new IOException("Interrupted while waiting to close", e); - } - } - if (closeFailure != null) - { - throw closeFailure; - } - if (closeRuntimeFailure != null) - { - throw closeRuntimeFailure; - } - } - } - - private static class ThreadCapturingPixelsWriter implements PixelsWriter - { - private final CountDownLatch closeStarted; - private final CountDownLatch allowClose; - private volatile Thread closeThread; - - private ThreadCapturingPixelsWriter(CountDownLatch closeStarted, CountDownLatch allowClose) - { - this.closeStarted = closeStarted; - this.allowClose = allowClose; - } - - @Override - public boolean addRowBatch(VectorizedRowBatch rowBatch) - { - return true; - } - - @Override - public void addRowBatch(VectorizedRowBatch rowBatch, int hashValue) - { - } - - @Override - public TypeDescription getSchema() - { - return null; - } - - @Override - public int getNumRowGroup() - { - return 0; - } - - @Override - public int getNumWriteRequests() - { - return 0; - } - - @Override - public long getCompletedBytes() - { - return 0; - } - - @Override - public void close() throws IOException - { - this.closeThread = Thread.currentThread(); - if (closeStarted != null) - { - closeStarted.countDown(); - } - if (allowClose != null) - { - try - { - assertTrue(allowClose.await(5, TimeUnit.SECONDS)); - } catch (InterruptedException e) - { - Thread.currentThread().interrupt(); - throw new IOException("Interrupted while waiting to close", e); - } - } - } - } -} diff --git a/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestPixelsWriteBuffer.java b/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestPixelsWriteBuffer.java index 3e6b4d9b6c..a7b392df8e 100644 --- a/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestPixelsWriteBuffer.java +++ b/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestPixelsWriteBuffer.java @@ -19,20 +19,27 @@ */ package io.pixelsdb.pixels.retina; +import io.pixelsdb.pixels.common.exception.RetinaException; import io.pixelsdb.pixels.common.metadata.domain.Path; import io.pixelsdb.pixels.core.TypeDescription; +import io.pixelsdb.pixels.core.vector.VectorizedRowBatch; import io.pixelsdb.pixels.index.IndexProto; import org.junit.Before; import org.junit.Test; -import java.lang.management.ManagementFactory; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicBoolean; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + public class TestPixelsWriteBuffer { private List columnNames = new ArrayList<>(); @@ -44,29 +51,32 @@ public class TestPixelsWriteBuffer @Before public void setup() { + columnNames.clear(); + columnTypes.clear(); + columnNames.add("id"); + columnNames.add("name"); + columnTypes.add("int"); + columnTypes.add("int"); + schema = TypeDescription.createSchemaFromStrings(columnNames, columnTypes); + targetOrderDirPath = new Path(); targetOrderDirPath.setUri("file:///home/gengdy/data/tpch/1g/customer/v-0-ordered"); targetOrderDirPath.setId(1); // path id get from mysql `PATHS` table targetCompactDirPath = new Path(); targetCompactDirPath.setUri("file:///home/gengdy/data/tpch/1g/customer/v-0-compact"); targetCompactDirPath.setId(2); // get from mysql `PATHS` table + } + + @Test + public void testConcurrentWriteOperations() + { try { - columnNames.add("id"); - columnNames.add("name"); - columnTypes.add("int"); - columnTypes.add("int"); - - schema = TypeDescription.createSchemaFromStrings(columnNames, columnTypes); buffer = new PixelsWriteBuffer(0L, schema, targetOrderDirPath, targetCompactDirPath, "localhost", 0); // table id get from mysql `TBLS` table } catch (Exception e) { System.out.println("setup error: " + e); } - } - @Test - public void testConcurrentWriteOperations() - { // // print pid if you want to attach a profiler like async-profiler or YourKit // try @@ -114,10 +124,63 @@ public void testConcurrentWriteOperations() { completionLatch.await(); Thread.sleep(10000); // wait for async flush to complete - buffer.close(); - } catch (Exception e) - { - System.out.println("error: " + e); - } + buffer.close(); + } catch (Exception e) + { + System.out.println("error: " + e); } } + + @Test + public void rgVisibilityRegistryValidationFailsClosed() throws Exception + { + RetinaResourceManager resourceManager = RetinaResourceManager.Instance(); + + try + { + resourceManager.validateRgVisibilityFileRegistered(500L); + fail("Expected missing RGVisibility registry entry to fail closed"); + } catch (RetinaException expected) + { + assertTrue(expected.getMessage().contains("RGVisibilityIndex contains fileId=500")); + } + + resourceManager.registerIngestFileMetadata(500L, 7L, 3, 0L); + resourceManager.validateRgVisibilityFileRegistered(500L); + } + + @Test + public void appendedRowsAreImmediatelyVisibleAndAdvanceCommitTsBounds() throws Exception + { + // After removing the two-phase publish, append is the only step and a + // row is query-visible as soon as it returns. The hidden ts column + // bounds therefore cover all appended rows immediately, and serialize() + // returns the full row batch with no truncation. + MemTable memTable = newMemTable(4); + + memTable.add(row(1), 10L); + assertEquals(1, memTable.getSize()); + assertEquals(1, VectorizedRowBatch.deserialize(memTable.serialize()).size); + assertEquals(10L, memTable.getMinCommitTs()); + assertEquals(10L, memTable.getMaxCommitTs()); + + memTable.add(row(2), 20L); + assertEquals(2, memTable.getSize()); + assertEquals(2, VectorizedRowBatch.deserialize(memTable.serialize()).size); + assertEquals(10L, memTable.getMinCommitTs()); + assertEquals(20L, memTable.getMaxCommitTs()); + } + + private static MemTable newMemTable(int size) + { + TypeDescription schema = TypeDescription.createSchemaFromStrings( + Arrays.asList("id"), Arrays.asList("int")); + return new MemTable(0L, schema, size, + TypeDescription.Mode.CREATE_INT_VECTOR_FOR_INT, 100L, 0, size); + } + + private static byte[][] row(int value) + { + return new byte[][] {ByteBuffer.allocate(Integer.BYTES).putInt(value).array()}; + } +} From e3228b48dedd8ba920f8ec1fa18bec4fc3673f7a Mon Sep 17 00:00:00 2001 From: Dongyang Geng Date: Sat, 23 May 2026 11:11:37 +0800 Subject: [PATCH 12/17] feat: optimize Retina visibility replay delete modes --- cpp/pixels-retina/include/RGVisibility.h | 3 +- cpp/pixels-retina/include/RGVisibilityJni.h | 4 +- cpp/pixels-retina/include/TileVisibility.h | 26 ++- cpp/pixels-retina/lib/RGVisibility.cpp | 5 +- cpp/pixels-retina/lib/RGVisibilityJni.cpp | 17 +- cpp/pixels-retina/lib/TileVisibility.cpp | 66 +++++++- cpp/pixels-retina/test/RGVisibilityTest.cpp | 72 ++++++++ cpp/pixels-retina/test/TileVisibilityTest.cpp | 159 ++++++++++++++++++ .../pixelsdb/pixels/retina/RGVisibility.java | 58 ++++++- 9 files changed, 398 insertions(+), 12 deletions(-) diff --git a/cpp/pixels-retina/include/RGVisibility.h b/cpp/pixels-retina/include/RGVisibility.h index 144cb4833a..88eda0c775 100644 --- a/cpp/pixels-retina/include/RGVisibility.h +++ b/cpp/pixels-retina/include/RGVisibility.h @@ -31,7 +31,8 @@ class RGVisibility : public pixels::RetinaBase> { const std::vector* initialBitmap = nullptr); ~RGVisibility() override; - void deleteRGRecord(uint32_t rowId, uint64_t timestamp); + void deleteRGRecord(uint32_t rowId, uint64_t timestamp, + ReplayMode replayMode = ReplayMode::NORMAL); uint64_t* getRGVisibilityBitmap(uint64_t timestamp); std::vector collectRGGarbage(uint64_t timestamp); diff --git a/cpp/pixels-retina/include/RGVisibilityJni.h b/cpp/pixels-retina/include/RGVisibilityJni.h index c8bb1fc3a5..79e82e16b6 100644 --- a/cpp/pixels-retina/include/RGVisibilityJni.h +++ b/cpp/pixels-retina/include/RGVisibilityJni.h @@ -26,10 +26,10 @@ JNIEXPORT void JNICALL Java_io_pixelsdb_pixels_retina_RGVisibility_destroyNative /* * Class: io_pixelsdb_pixels_retina_RGVisibility * Method: deleteRecord - * Signature: (IJJ)V + * Signature: (IJJI)V */ JNIEXPORT void JNICALL Java_io_pixelsdb_pixels_retina_RGVisibility_deleteRecord - (JNIEnv *, jobject, jint, jlong, jlong); + (JNIEnv *, jobject, jint, jlong, jlong, jint); /* * Class: io_pixelsdb_pixels_retina_RGVisibility diff --git a/cpp/pixels-retina/include/TileVisibility.h b/cpp/pixels-retina/include/TileVisibility.h index ef9bd59143..fae7665dee 100644 --- a/cpp/pixels-retina/include/TileVisibility.h +++ b/cpp/pixels-retina/include/TileVisibility.h @@ -48,6 +48,22 @@ inline uint64_t extractTimestamp(uint64_t raw) { return (raw & 0x0000FFFFFFFFFFFFULL); } +/** + * Controls how DELETE replay interacts with the compacted base bitmap. + * + * NORMAL is the live append path: the caller provides a current delete + * timestamp and the record is appended to the chain. VERSIONED is used when + * replay may race with READY readers; historical deletes publish a new + * VersionedData with a folded baseBitmap. EXCLUSIVE is used only while recovery + * blocks readers and GC; historical deletes may update baseBitmap in place, but + * concurrent recovery writers still need tile-level synchronization. + */ +enum class ReplayMode : uint8_t { + NORMAL = 0, + VERSIONED = 1, + EXCLUSIVE = 2 +}; + struct DeleteIndexBlock : public pixels::RetinaBase { static constexpr size_t BLOCK_CAPACITY = 8; uint64_t items[BLOCK_CAPACITY] = {0}; @@ -96,7 +112,7 @@ class TileVisibility : public pixels::RetinaBase> { // timestamp defaults to 0; bitmap defaults to all-zeros. explicit TileVisibility(uint64_t timestamp = 0, const uint64_t* bitmap = nullptr); ~TileVisibility() override; - void deleteTileRecord(uint16_t rowId, uint64_t ts); + void deleteTileRecord(uint16_t rowId, uint64_t ts, ReplayMode replayMode = ReplayMode::NORMAL); void getTileVisibilityBitmap(uint64_t ts, uint64_t* outBitmap) const; void collectTileGarbage(uint64_t ts, uint64_t* gcSnapshotBitmap); void exportChainItemsAfter(uint32_t tileId, uint64_t safeGcTs, @@ -109,6 +125,14 @@ class TileVisibility : public pixels::RetinaBase> { void reclaimRetiredVersions(); + void appendDeleteChain(uint16_t rowId, uint64_t ts); + + // VERSIONED: replay with possible readers; historical deletes use COW fold. + void deleteTileRecordVersioned(uint16_t rowId, uint64_t ts); + + // EXCLUSIVE: recovery replay without readers; historical deletes fold in place. + void deleteTileRecordExclusive(uint16_t rowId, uint64_t ts); + std::atomic*> currentVersion; std::atomic tail; std::atomic tailUsed; diff --git a/cpp/pixels-retina/lib/RGVisibility.cpp b/cpp/pixels-retina/lib/RGVisibility.cpp index d1609535f0..289de9e0d3 100644 --- a/cpp/pixels-retina/lib/RGVisibility.cpp +++ b/cpp/pixels-retina/lib/RGVisibility.cpp @@ -70,9 +70,10 @@ TileVisibility* RGVisibility::getTileVisibility(uint32_t row } template -void RGVisibility::deleteRGRecord(uint32_t rowId, uint64_t timestamp) { +void RGVisibility::deleteRGRecord(uint32_t rowId, uint64_t timestamp, + ReplayMode replayMode) { TileVisibility* tileVisibility = getTileVisibility(rowId); - tileVisibility->deleteTileRecord(rowId % VISIBILITY_RECORD_CAPACITY, timestamp); + tileVisibility->deleteTileRecord(rowId % VISIBILITY_RECORD_CAPACITY, timestamp, replayMode); } template diff --git a/cpp/pixels-retina/lib/RGVisibilityJni.cpp b/cpp/pixels-retina/lib/RGVisibilityJni.cpp index fdcbeaa328..b6293366ca 100644 --- a/cpp/pixels-retina/lib/RGVisibilityJni.cpp +++ b/cpp/pixels-retina/lib/RGVisibilityJni.cpp @@ -23,6 +23,17 @@ #include "RGVisibility.h" #include +namespace { +ReplayMode toReplayMode(jint mode) { + switch (mode) { + case 0: return ReplayMode::NORMAL; + case 1: return ReplayMode::VERSIONED; + case 2: return ReplayMode::EXCLUSIVE; + default: throw std::invalid_argument("unknown ReplayMode"); + } +} +} + /* * Class: io_pixelsdb_pixels_retina_RGVisibility * Method: createNativeObject @@ -72,13 +83,13 @@ JNIEXPORT void JNICALL Java_io_pixelsdb_pixels_retina_RGVisibility_destroyNative /* * Class: io_pixelsdb_pixels_retina_RGVisibility * Method: deleteRecord - * Signature: (JJJ)V + * Signature: (IJJI)V */ JNIEXPORT void JNICALL Java_io_pixelsdb_pixels_retina_RGVisibility_deleteRecord - (JNIEnv* env, jobject, jint rowId, jlong timestamp, jlong handle) { + (JNIEnv* env, jobject, jint rowId, jlong timestamp, jlong handle, jint replayMode) { try { auto* rgVisibility = reinterpret_cast(handle); - rgVisibility->deleteRGRecord(rowId, timestamp); + rgVisibility->deleteRGRecord(rowId, timestamp, toReplayMode(replayMode)); } catch (const std::exception& e) { env->ThrowNew(env->FindClass("java/lang/RuntimeException"), e.what()); } diff --git a/cpp/pixels-retina/lib/TileVisibility.cpp b/cpp/pixels-retina/lib/TileVisibility.cpp index f4fcdcb429..49710c71b5 100644 --- a/cpp/pixels-retina/lib/TileVisibility.cpp +++ b/cpp/pixels-retina/lib/TileVisibility.cpp @@ -68,7 +68,71 @@ TileVisibility::~TileVisibility() { } template -void TileVisibility::deleteTileRecord(uint16_t rowId, uint64_t ts) { +void TileVisibility::deleteTileRecord(uint16_t rowId, uint64_t ts, + ReplayMode replayMode) { + switch (replayMode) { + case ReplayMode::NORMAL: + appendDeleteChain(rowId, ts); + return; + case ReplayMode::VERSIONED: + deleteTileRecordVersioned(rowId, ts); + return; + case ReplayMode::EXCLUSIVE: + deleteTileRecordExclusive(rowId, ts); + return; + default: + throw std::invalid_argument("unknown ReplayMode"); + } +} + +template +void TileVisibility::deleteTileRecordVersioned(uint16_t rowId, uint64_t ts) { + // READY backlog replay can race with getTileVisibilityBitmap readers. Fold + // historical deletes by publishing a new VersionedData instead of mutating + // baseBitmap observed by an existing reader. + // Keep ts=0 out of this path because item=0 is the chain-slot sentinel. + while (ts > 0) { + VersionedData* cur = currentVersion.load(std::memory_order_acquire); + if (ts > cur->baseTimestamp) { + break; + } + if ((cur->baseBitmap[rowId / 64] & (1ULL << (rowId % 64))) != 0) { + return; + } + uint64_t newBaseBitmap[NUM_WORDS]; + std::memcpy(newBaseBitmap, cur->baseBitmap, NUM_WORDS * sizeof(uint64_t)); + SET_BITMAP_BIT(newBaseBitmap, rowId); + VersionedData* newVer = + new VersionedData(cur->baseTimestamp, newBaseBitmap, cur->head); + if (currentVersion.compare_exchange_strong(cur, newVer, std::memory_order_acq_rel)) { + pendingRetire.store(cur, std::memory_order_release); + return; + } + delete newVer; + } + + appendDeleteChain(rowId, ts); +} + +template +void TileVisibility::deleteTileRecordExclusive(uint16_t rowId, uint64_t ts) { + // RECOVERING replay blocks readers and GC, so historical deletes can fold + // into baseBitmap in place. Atomic OR prevents lost updates when concurrent + // recovery writers touch the same bitmap word. + VersionedData* cur = currentVersion.load(std::memory_order_acquire); + if (ts > 0 && ts <= cur->baseTimestamp) { + uint64_t mask = 1ULL << (rowId % 64); + __atomic_fetch_or(&cur->baseBitmap[rowId / 64], mask, __ATOMIC_RELAXED); + return; + } + + appendDeleteChain(rowId, ts); +} + +template +void TileVisibility::appendDeleteChain(uint16_t rowId, uint64_t ts) { + // Normal live apply assumes a current timestamp and records the delete in + // the append-only chain, leaving baseBitmap untouched for the hot path. uint64_t item = makeDeleteIndex(rowId, ts); while (true) { DeleteIndexBlock *curTail = tail.load(std::memory_order_acquire); diff --git a/cpp/pixels-retina/test/RGVisibilityTest.cpp b/cpp/pixels-retina/test/RGVisibilityTest.cpp index 8d8b135eee..145a9918f3 100644 --- a/cpp/pixels-retina/test/RGVisibilityTest.cpp +++ b/cpp/pixels-retina/test/RGVisibilityTest.cpp @@ -49,6 +49,50 @@ class RGVisibilityTest : public ::testing::Test { RGVisibilityInstance* rgVisibility; }; +static bool rgBitSet(const uint64_t* bitmap, uint32_t rowId) { + return ((bitmap[rowId / 64] >> (rowId % 64)) & 1ULL) != 0; +} + +static void runConcurrentRGDeletes(RGVisibilityInstance* visibility, + ReplayMode mode, + uint64_t ts, + int rowCount = 64, + int threadCount = 8) { + ASSERT_EQ(rowCount % threadCount, 0); + std::atomic start{false}; + std::vector threads; + int rowsPerThread = rowCount / threadCount; + + for (int t = 0; t < threadCount; t++) { + threads.emplace_back([&, t]() { + while (!start.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + for (int i = 0; i < rowsPerThread; i++) { + uint32_t rowId = static_cast(t * rowsPerThread + i); + visibility->deleteRGRecord(rowId, ts, mode); + } + }); + } + + start.store(true, std::memory_order_release); + for (auto& thread : threads) { + thread.join(); + } +} + +static void expectRGRows(RGVisibilityInstance* visibility, + uint64_t queryTs, + int rowCount, + bool expectedSet) { + uint64_t* bitmap = visibility->getRGVisibilityBitmap(queryTs); + for (int row = 0; row < rowCount; row++) { + EXPECT_EQ(expectedSet, rgBitSet(bitmap, static_cast(row))) + << "row=" << row << " queryTs=" << queryTs; + } + delete[] bitmap; +} + TEST_F(RGVisibilityTest, BasicDeleteAndVisibility) { uint64_t timestamp1 = 100; uint64_t timestamp2 = 200; @@ -67,6 +111,34 @@ TEST_F(RGVisibilityTest, BasicDeleteAndVisibility) { delete[] bitmap2; } +TEST_F(RGVisibilityTest, ConcurrentNormalModeAppendsDeleteChain) { + constexpr uint64_t baseTs = 100; + RGVisibilityInstance visibility(ROW_COUNT, baseTs, nullptr); + + runConcurrentRGDeletes(&visibility, ReplayMode::NORMAL, baseTs + 1); + + expectRGRows(&visibility, baseTs, 64, false); + expectRGRows(&visibility, baseTs + 1, 64, true); +} + +TEST_F(RGVisibilityTest, ConcurrentVersionedModeFoldsWithCow) { + constexpr uint64_t baseTs = 100; + RGVisibilityInstance visibility(ROW_COUNT, baseTs, nullptr); + + runConcurrentRGDeletes(&visibility, ReplayMode::VERSIONED, baseTs - 1); + + expectRGRows(&visibility, baseTs, 64, true); +} + +TEST_F(RGVisibilityTest, ConcurrentExclusiveModeFoldsWithAtomicOr) { + constexpr uint64_t baseTs = 100; + RGVisibilityInstance visibility(ROW_COUNT, baseTs, nullptr); + + runConcurrentRGDeletes(&visibility, ReplayMode::EXCLUSIVE, baseTs - 1); + + expectRGRows(&visibility, baseTs, 64, true); +} + TEST_F(RGVisibilityTest, MultiThread) { struct DeleteRecord { uint64_t timestamp; diff --git a/cpp/pixels-retina/test/TileVisibilityTest.cpp b/cpp/pixels-retina/test/TileVisibilityTest.cpp index 0a84b806f9..7994f62e4d 100644 --- a/cpp/pixels-retina/test/TileVisibilityTest.cpp +++ b/cpp/pixels-retina/test/TileVisibilityTest.cpp @@ -695,3 +695,162 @@ TEST_F(TileVisibilityTest, ImportDeletionItems_EmptyChainTailClaim) { v->getTileVisibilityBitmap(500, actualBitmap); EXPECT_TRUE(checkBitmap(actualBitmap, expectedBitmap)); } + +// ========================================================================= +// COW fold of `ts <= baseTimestamp` deletes into baseBitmap. +// Three ts relations plus duplicate replay. +// ========================================================================= + +class TileVisibilityCowFoldTest : public ::testing::Test { +protected: + static constexpr uint64_t kBaseTimestamp = 100; + TileVisibility* v; + + void SetUp() override { + // Start with a non-zero baseTimestamp so the fold guard is exercised. + v = new TileVisibility(kBaseTimestamp, nullptr); + } + + void TearDown() override { + delete v; + } + + bool bitSet(const uint64_t* bitmap, uint16_t rowId) { + return ((bitmap[rowId / 64] >> (rowId % 64)) & 1ULL) != 0; + } + + void runConcurrentDeletes(ReplayMode mode, uint64_t ts, int rowCount = 64, int threadCount = 8) { + ASSERT_EQ(rowCount % threadCount, 0); + std::atomic start{false}; + std::vector threads; + int rowsPerThread = rowCount / threadCount; + + for (int t = 0; t < threadCount; t++) { + threads.emplace_back([&, t]() { + while (!start.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + for (int i = 0; i < rowsPerThread; i++) { + uint16_t rowId = static_cast(t * rowsPerThread + i); + v->deleteTileRecord(rowId, ts, mode); + } + }); + } + + start.store(true, std::memory_order_release); + for (auto& thread : threads) { + thread.join(); + } + } + + void expectRows(uint64_t queryTs, int rowCount, bool expectedSet) { + uint64_t bitmap[BITMAP_SIZE] = {0}; + v->getTileVisibilityBitmap(queryTs, bitmap); + for (int row = 0; row < rowCount; row++) { + EXPECT_EQ(expectedSet, bitSet(bitmap, static_cast(row))) + << "row=" << row << " queryTs=" << queryTs; + } + } +}; + +TEST_F(TileVisibilityCowFoldTest, FoldsWhenTsLessThanBaseTimestamp) { + // ts < baseTimestamp: row must be folded into baseBitmap and visible at any + // snap_ts >= baseTimestamp. + v->deleteTileRecord(7, kBaseTimestamp - 50, ReplayMode::VERSIONED); + + uint64_t bitmap[BITMAP_SIZE] = {0}; + v->getTileVisibilityBitmap(kBaseTimestamp, bitmap); + EXPECT_TRUE(bitSet(bitmap, 7)); + + // Even at a much later snap_ts the row should still be visible-as-deleted. + uint64_t bitmap2[BITMAP_SIZE] = {0}; + v->getTileVisibilityBitmap(kBaseTimestamp + 1000, bitmap2); + EXPECT_TRUE(bitSet(bitmap2, 7)); +} + +TEST_F(TileVisibilityCowFoldTest, FoldsWhenTsEqualsBaseTimestamp) { + v->deleteTileRecord(9, kBaseTimestamp, ReplayMode::VERSIONED); + + uint64_t bitmap[BITMAP_SIZE] = {0}; + v->getTileVisibilityBitmap(kBaseTimestamp, bitmap); + EXPECT_TRUE(bitSet(bitmap, 9)); +} + +TEST_F(TileVisibilityCowFoldTest, NormalModeDoesNotFoldHistoricalTimestamp) { + v->deleteTileRecord(10, kBaseTimestamp - 1, ReplayMode::NORMAL); + + uint64_t bitmap[BITMAP_SIZE] = {0}; + v->getTileVisibilityBitmap(kBaseTimestamp, bitmap); + EXPECT_FALSE(bitSet(bitmap, 10)); +} + +TEST_F(TileVisibilityCowFoldTest, ExclusiveModeFoldsHistoricalTimestamp) { + v->deleteTileRecord(12, kBaseTimestamp - 1, ReplayMode::EXCLUSIVE); + + uint64_t bitmap[BITMAP_SIZE] = {0}; + v->getTileVisibilityBitmap(kBaseTimestamp, bitmap); + EXPECT_TRUE(bitSet(bitmap, 12)); +} + +TEST_F(TileVisibilityCowFoldTest, ConcurrentNormalModeAppendsDeleteChain) { + runConcurrentDeletes(ReplayMode::NORMAL, kBaseTimestamp + 1); + + expectRows(kBaseTimestamp, 64, false); + expectRows(kBaseTimestamp + 1, 64, true); +} + +TEST_F(TileVisibilityCowFoldTest, ConcurrentVersionedModeFoldsWithCow) { + runConcurrentDeletes(ReplayMode::VERSIONED, kBaseTimestamp - 1); + + expectRows(kBaseTimestamp, 64, true); +} + +TEST_F(TileVisibilityCowFoldTest, ConcurrentExclusiveModeFoldsWithAtomicOr) { + runConcurrentDeletes(ReplayMode::EXCLUSIVE, kBaseTimestamp - 1); + + expectRows(kBaseTimestamp, 64, true); +} + +TEST_F(TileVisibilityCowFoldTest, AppendsToChainWhenTsGreaterThanBaseTimestamp) { + // ts > baseTimestamp: should take the append-to-chain path. The row must be + // invisible at snap_ts < ts and visible at snap_ts >= ts. + v->deleteTileRecord(11, kBaseTimestamp + 50, ReplayMode::VERSIONED); + + uint64_t before[BITMAP_SIZE] = {0}; + v->getTileVisibilityBitmap(kBaseTimestamp + 49, before); + EXPECT_FALSE(bitSet(before, 11)); + + uint64_t after[BITMAP_SIZE] = {0}; + v->getTileVisibilityBitmap(kBaseTimestamp + 50, after); + EXPECT_TRUE(bitSet(after, 11)); +} + +TEST_F(TileVisibilityCowFoldTest, DuplicateFoldOnAlreadyDeletedRowIsIdempotent) { + // A replayed historical DELETE for a row already folded into baseBitmap should + // remain a no-op semantically. This guards the fast path that returns before + // cloning another VersionedData when the base bit is already set. + v->deleteTileRecord(13, kBaseTimestamp - 10, ReplayMode::VERSIONED); + for (int i = 0; i < 32; i++) { + v->deleteTileRecord(13, kBaseTimestamp - 20, ReplayMode::VERSIONED); + } + + uint64_t atBase[BITMAP_SIZE] = {0}; + v->getTileVisibilityBitmap(kBaseTimestamp, atBase); + EXPECT_TRUE(bitSet(atBase, 13)); + EXPECT_FALSE(bitSet(atBase, 14)); + + // The duplicate fold must not corrupt the append-to-chain path or later GC. + v->deleteTileRecord(14, kBaseTimestamp + 5, ReplayMode::VERSIONED); + uint64_t beforeAppendTs[BITMAP_SIZE] = {0}; + v->getTileVisibilityBitmap(kBaseTimestamp + 4, beforeAppendTs); + EXPECT_TRUE(bitSet(beforeAppendTs, 13)); + EXPECT_FALSE(bitSet(beforeAppendTs, 14)); + + uint64_t gcBitmap[BITMAP_SIZE] = {0}; + v->collectTileGarbage(kBaseTimestamp + 5, gcBitmap); + + uint64_t afterGc[BITMAP_SIZE] = {0}; + v->getTileVisibilityBitmap(kBaseTimestamp + 5, afterGc); + EXPECT_TRUE(bitSet(afterGc, 13)); + EXPECT_TRUE(bitSet(afterGc, 14)); +} diff --git a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/RGVisibility.java b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/RGVisibility.java index 1816f262d5..6b4696e7d1 100644 --- a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/RGVisibility.java +++ b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/RGVisibility.java @@ -34,6 +34,54 @@ public class RGVisibility implements AutoCloseable { private static final Logger logger = LogManager.getLogger(RGVisibility.class); + + /** + * Selects how a visibility DELETE should be applied. + * + *

    The modes separate the timestamp semantics from the lifecycle concurrency + * guarantees. NORMAL is the live fast path and only appends to the delete chain. + * VERSIONED is for replay while READY readers may be active, so historical + * deletes fold into baseBitmap through copy-on-write. EXCLUSIVE is for the + * RECOVERING replay window where readers and GC are blocked; historical deletes + * may fold into baseBitmap in place, with native writer synchronization.

    + */ + public enum ReplayMode + { + /** + * Normal live apply. The caller is expected to provide delete timestamps + * newer than the current baseTimestamp, so native code appends the delete + * record to the timestamped chain and does not inspect baseBitmap first. + */ + NORMAL(0), + + /** + * Replay while concurrent readers may exist, for example READY backlog + * catchup. Deletes with timestamp <= baseTimestamp are folded into + * baseBitmap by publishing a new version; newer deletes append to the chain. + */ + VERSIONED(1), + + /** + * Replay in an exclusive recovery window. Query and GC readers must be + * blocked, but multiple recovery writers may still run; native code uses a + * tile-level writer lock and folds historical deletes into baseBitmap in + * place. + */ + EXCLUSIVE(2); + + private final int code; + + ReplayMode(int code) + { + this.code = code; + } + + int code() + { + return code; + } + } + static { String pixelsHome = System.getenv("PIXELS_HOME"); @@ -93,7 +141,7 @@ public void close() // native methods private native long createNativeObject(long rgRecordNum, long timestamp, long[] bitmap); private native void destroyNativeObject(long nativeHandle); - private native void deleteRecord(int rgRowOffset, long timestamp, long nativeHandle); + private native void deleteRecord(int rgRowOffset, long timestamp, long nativeHandle, int replayMode); private native long[] getVisibilityBitmap(long timestamp, long nativeHandle); private native long[] garbageCollect(long timestamp, long nativeHandle); private native long[] exportChainItemsAfter(long safeGcTs, long nativeHandle); @@ -103,10 +151,16 @@ public void close() private static native long getRetinaObjectCount(); public void deleteRecord(int rgRowOffset, long timestamp) + { + deleteRecord(rgRowOffset, timestamp, ReplayMode.NORMAL); + } + + public void deleteRecord(int rgRowOffset, long timestamp, ReplayMode replayMode) { long handle = nativeHandle.get(); if (handle == 0) throw new IllegalStateException("RGVisibility is closed"); - deleteRecord(rgRowOffset, timestamp, handle); + if (replayMode == null) throw new IllegalArgumentException("replayMode is null"); + deleteRecord(rgRowOffset, timestamp, handle, replayMode.code()); } public long[] getVisibilityBitmap(long timestamp) From 08189d71edc71b11af14428c68890dfbc34a88c9 Mon Sep 17 00:00:00 2001 From: Dongyang Geng Date: Sat, 23 May 2026 11:16:28 +0800 Subject: [PATCH 13/17] fix: propagate replay mode in Retina deletes --- .../pixels/retina/RetinaResourceManager.java | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/RetinaResourceManager.java b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/RetinaResourceManager.java index 857f12b5d8..f38fc4dd37 100644 --- a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/RetinaResourceManager.java +++ b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/RetinaResourceManager.java @@ -692,7 +692,13 @@ public String getCheckpointPath(long timestamp) public void deleteRecord(long fileId, int rgId, int rgRowOffset, long timestamp) throws RetinaException { - checkRGVisibility(fileId, rgId).deleteRecord(rgRowOffset, timestamp); + deleteRecord(fileId, rgId, rgRowOffset, timestamp, RGVisibility.ReplayMode.NORMAL); + } + + public void deleteRecord(long fileId, int rgId, int rgRowOffset, long timestamp, + RGVisibility.ReplayMode replayMode) throws RetinaException + { + checkRGVisibility(fileId, rgId).deleteRecord(rgRowOffset, timestamp, replayMode); if (!isDualWriteActive) { @@ -719,7 +725,7 @@ public void deleteRecord(long fileId, int rgId, int rgRowOffset, long timestamp) int oldGlobal = bwdMapping[rgRowOffset]; int oldRgId = rgIdForGlobalRowOffset(oldGlobal, bwd.oldFileRgRowStart); int oldRgOff = oldGlobal - bwd.oldFileRgRowStart[oldRgId]; - checkRGVisibility(bwd.oldFileId, oldRgId).deleteRecord(oldRgOff, timestamp); + checkRGVisibility(bwd.oldFileId, oldRgId).deleteRecord(oldRgOff, timestamp, replayMode); } } } @@ -733,7 +739,7 @@ public void deleteRecord(long fileId, int rgId, int rgRowOffset, long timestamp) int newGlobal = fwdMapping[rgRowOffset]; int newRgId = rgIdForGlobalRowOffset(newGlobal, result.newFileRgRowStart); int newRgOff = newGlobal - result.newFileRgRowStart[newRgId]; - checkRGVisibility(result.newFileId, newRgId).deleteRecord(newRgOff, timestamp); + checkRGVisibility(result.newFileId, newRgId).deleteRecord(newRgOff, timestamp, replayMode); } } } @@ -748,6 +754,14 @@ public void deleteRecord(IndexProto.RowLocation rowLocation, long timestamp) thr deleteRecord(rowLocation.getFileId(), rowLocation.getRgId(), rowLocation.getRgRowOffset(), timestamp); } + public void deleteRecord(IndexProto.RowLocation rowLocation, long timestamp, + RGVisibility.ReplayMode replayMode) + throws RetinaException + { + deleteRecord(rowLocation.getFileId(), rowLocation.getRgId(), rowLocation.getRgRowOffset(), + timestamp, replayMode); + } + /** * Registers dual-write redirection so that {@link #deleteRecord} propagates * deletes between old and new files. The write lock acts as a barrier: all From e8f05a991e904bda7021e4b03ece1ec50cd37eac Mon Sep 17 00:00:00 2001 From: Dongyang Geng Date: Sun, 24 May 2026 14:06:56 +0800 Subject: [PATCH 14/17] fix: stage Retina primary index updates --- .../pixels/common/index/ResolvedPrimary.java | 52 ++ .../pixels/common/index/RollbackEntry.java | 59 +++ .../common/index/service/IndexService.java | 117 ++++- .../index/service/LocalIndexService.java | 307 ++++++++--- .../common/index/TestLocalIndexService.java | 178 +++++++ .../daemon/retina/RetinaServerImpl.java | 395 +++++++++----- .../daemon/retina/TestRetinaServer.java | 497 +++++++++++++++++- .../index/main/sqlite/SqliteMainIndex.java | 23 +- .../main/sqlite/TestSqliteMainIndex.java | 6 +- 9 files changed, 1388 insertions(+), 246 deletions(-) create mode 100644 pixels-common/src/main/java/io/pixelsdb/pixels/common/index/ResolvedPrimary.java create mode 100644 pixels-common/src/main/java/io/pixelsdb/pixels/common/index/RollbackEntry.java diff --git a/pixels-common/src/main/java/io/pixelsdb/pixels/common/index/ResolvedPrimary.java b/pixels-common/src/main/java/io/pixelsdb/pixels/common/index/ResolvedPrimary.java new file mode 100644 index 0000000000..4587a6fb63 --- /dev/null +++ b/pixels-common/src/main/java/io/pixelsdb/pixels/common/index/ResolvedPrimary.java @@ -0,0 +1,52 @@ +/* + * Copyright 2026 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the Affero GNU General Public + * License along with Pixels. If not, see + * . + */ +package io.pixelsdb.pixels.common.index; + +import io.pixelsdb.pixels.index.IndexProto; + +import java.util.Objects; + +/** + * Result of a successful primary index resolution, returned wrapped in + * {@link java.util.Optional}: present = key is live; empty = key missing or + * maps to an orphan / non-baseline-visible location; backend failures surface + * as {@link io.pixelsdb.pixels.common.exception.IndexException}. + */ +public final class ResolvedPrimary +{ + private final long rowId; + private final IndexProto.RowLocation rowLocation; + + public ResolvedPrimary(long rowId, IndexProto.RowLocation rowLocation) + { + this.rowId = rowId; + this.rowLocation = Objects.requireNonNull(rowLocation, "rowLocation"); + } + + public long getRowId() + { + return rowId; + } + + public IndexProto.RowLocation getRowLocation() + { + return rowLocation; + } +} diff --git a/pixels-common/src/main/java/io/pixelsdb/pixels/common/index/RollbackEntry.java b/pixels-common/src/main/java/io/pixelsdb/pixels/common/index/RollbackEntry.java new file mode 100644 index 0000000000..20780aa2a0 --- /dev/null +++ b/pixels-common/src/main/java/io/pixelsdb/pixels/common/index/RollbackEntry.java @@ -0,0 +1,59 @@ +/* + * Copyright 2026 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the Affero GNU General Public + * License along with Pixels. If not, see + * . + */ +package io.pixelsdb.pixels.common.index; + +import io.pixelsdb.pixels.index.IndexProto; + +import java.util.Objects; + +/** + * Journal record for restoring one primary index pointer from newRowId + * back to oldRowId. restorePrimaryIndexEntries writes back oldRowId only when + * the current pointer still equals newRowId, skipping entries that have + * been tombstoned or moved on to a third rowId. + */ +public final class RollbackEntry +{ + private final IndexProto.IndexKey indexKey; + private final long oldRowId; + private final long newRowId; + + public RollbackEntry(IndexProto.IndexKey indexKey, long oldRowId, long newRowId) + { + this.indexKey = Objects.requireNonNull(indexKey, "indexKey"); + this.oldRowId = oldRowId; + this.newRowId = newRowId; + } + + public IndexProto.IndexKey getIndexKey() + { + return indexKey; + } + + public long getOldRowId() + { + return oldRowId; + } + + public long getNewRowId() + { + return newRowId; + } +} diff --git a/pixels-common/src/main/java/io/pixelsdb/pixels/common/index/service/IndexService.java b/pixels-common/src/main/java/io/pixelsdb/pixels/common/index/service/IndexService.java index 627f340207..3fe2f257f0 100644 --- a/pixels-common/src/main/java/io/pixelsdb/pixels/common/index/service/IndexService.java +++ b/pixels-common/src/main/java/io/pixelsdb/pixels/common/index/service/IndexService.java @@ -21,8 +21,12 @@ import io.pixelsdb.pixels.common.exception.IndexException; import io.pixelsdb.pixels.common.index.IndexOption; +import io.pixelsdb.pixels.common.index.ResolvedPrimary; +import io.pixelsdb.pixels.common.index.RollbackEntry; import io.pixelsdb.pixels.index.IndexProto; + import java.util.List; +import java.util.Optional; public interface IndexService { @@ -40,7 +44,7 @@ public interface IndexService /** * Lookup a unique index. * @param key the index key - * @return the row location or null if the index entry is not found + * @return the row location, or null if the key is missing or maps to an orphan */ IndexProto.RowLocation lookupUniqueIndex(IndexProto.IndexKey key, IndexOption indexOption) throws IndexException; @@ -87,6 +91,7 @@ boolean putSecondaryIndexEntries(long tableId, long indexId, /** * Delete an entry from the primary index. The deleted index entry is marked as deleted using a tombstone. + * Crash-unsafe; prefer {@link #resolvePrimary} + {@link #deletePrimaryIndexEntriesOnly}. * @param key the index key * @return the row location of the deleted index entry * @throws IndexException if no existing entry to delete @@ -103,6 +108,7 @@ boolean putSecondaryIndexEntries(long tableId, long indexId, /** * Delete entries from the primary index. Each deleted index entry is marked as deleted using a tombstone. + * Crash-unsafe; prefer {@link #resolvePrimary} + {@link #deletePrimaryIndexEntriesOnly}. * @param tableId the table id of the index * @param indexId the index id of the index * @param keys the keys of the entries to delete @@ -126,6 +132,7 @@ List deleteSecondaryIndexEntries(long tableId, long indexId, /** * Update the entry of a primary index. + * Crash-unsafe; prefer DELETE + INSERT. * @param indexEntry the index entry to update * @return the previous row location of the index entry * @throws IndexException if no existing entry to update @@ -142,6 +149,7 @@ List deleteSecondaryIndexEntries(long tableId, long indexId, /** * Update the entries of a primary index. + * Crash-unsafe; prefer DELETE + INSERT. * @param tableId the table id of the primary index * @param indexId the index id of the primary index * @param indexEntries the index entries to update @@ -215,5 +223,112 @@ boolean flushIndexEntriesOfFile(long tableId, long indexId, * @return true on success */ boolean removeIndex(long tableId, long indexId, boolean isPrimary, IndexOption option) throws IndexException; + + // ================================================================================== + // Staged primary-index APIs. Default implementations throw UnsupportedOperationException; + // LocalIndexService provides the in-process implementation. + // ================================================================================== + + /** + * Resolve a batch of primary index keys to {@link ResolvedPrimary} (rowId + RowLocation), + * positionally aligned with keys. Returns Optional.empty() for keys + * that are missing, tombstoned, orphan in MainIndex, or filtered out by the + * baseline visible file set; throws on backend error. + * + * @param tableId the table id of the primary index + * @param indexId the index id of the primary index + * @param keys the primary index keys to resolve + * @param indexOption optional index option + * @return positional list of resolved primaries + * @throws IndexException on backend error + */ + default List> resolvePrimary(long tableId, long indexId, + List keys, IndexOption indexOption) throws IndexException + { + throw new UnsupportedOperationException( + "resolvePrimary is not supported by this IndexService scheme"); + } + + /** + * Write rowId -> RowLocation entries into the main index. + * + * @param tableId the table id of the main index + * @param entries the entries to persist + * @throws IndexException on backend error + */ + default void putMainIndexEntriesOnly(long tableId, + List entries) throws IndexException + { + throw new UnsupportedOperationException( + "putMainIndexEntriesOnly is not supported by this IndexService scheme"); + } + + /** + * Write IndexKey -> rowId entries into the primary single point index. + * + * @param tableId the table id of the primary index + * @param indexId the index id of the primary index + * @param entries the entries to persist + * @param indexOption optional index option + * @throws IndexException on backend error + */ + default void putPrimaryIndexEntriesOnly(long tableId, long indexId, + List entries, IndexOption indexOption) throws IndexException + { + throw new UnsupportedOperationException( + "putPrimaryIndexEntriesOnly is not supported by this IndexService scheme"); + } + + /** + * Delete primary index entries for keys already resolved by {@link #resolvePrimary}. + * Repeating on an already-deleted key is a no-op. + * + * @param tableId the table id of the primary index + * @param indexId the index id of the primary index + * @param resolvedKeys the keys to delete + * @param indexOption optional index option + * @throws IndexException on backend error + */ + default void deletePrimaryIndexEntriesOnly(long tableId, long indexId, + List resolvedKeys, IndexOption indexOption) throws IndexException + { + throw new UnsupportedOperationException( + "deletePrimaryIndexEntriesOnly is not supported by this IndexService scheme"); + } + + /** + * Update primary index entries to the new IndexKey -> rowId mapping; + * does not look up the previous rowId. + * + * @param tableId the table id of the primary index + * @param indexId the index id of the primary index + * @param entries the new IndexKey -> rowId mappings + * @param indexOption optional index option + * @throws IndexException on backend error + */ + default void updatePrimaryIndexEntriesOnly(long tableId, long indexId, + List entries, IndexOption indexOption) throws IndexException + { + throw new UnsupportedOperationException( + "updatePrimaryIndexEntriesOnly is not supported by this IndexService scheme"); + } + + /** + * Restore primary index entries to oldRowId where the current pointer + * still equals newRowId; skip otherwise. Intended for single-threaded + * rollback windows and does not require atomic conditional update from the backend. + * + * @param tableId the table id of the primary index + * @param indexId the index id of the primary index + * @param entries rollback entries describing each oldRowId -> newRowId transition + * @param indexOption optional index option + * @throws IndexException on backend error + */ + default void restorePrimaryIndexEntries(long tableId, long indexId, + List entries, IndexOption indexOption) throws IndexException + { + throw new UnsupportedOperationException( + "restorePrimaryIndexEntries is not supported by this IndexService scheme"); + } } diff --git a/pixels-common/src/main/java/io/pixelsdb/pixels/common/index/service/LocalIndexService.java b/pixels-common/src/main/java/io/pixelsdb/pixels/common/index/service/LocalIndexService.java index 7577036278..2e4be1f1bd 100644 --- a/pixels-common/src/main/java/io/pixelsdb/pixels/common/index/service/LocalIndexService.java +++ b/pixels-common/src/main/java/io/pixelsdb/pixels/common/index/service/LocalIndexService.java @@ -28,12 +28,25 @@ import io.pixelsdb.pixels.common.utils.ConfigFactory; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.function.Supplier; public class LocalIndexService implements IndexService { private static final LocalIndexService defaultInstance = new LocalIndexService(); private static boolean upsertMode; + + /** + * Visible file-id set supplier used by {@link #resolvePrimary} to filter + * out RowLocations whose fileId is outside the set. Default returns null, + * which disables the filter; install a real supplier via + * {@link #setBaselineVisibleFilesSupplier}. + */ + private volatile Supplier> baselineVisibleFilesSupplier = () -> null; + public static LocalIndexService Instance() { return defaultInstance; @@ -44,6 +57,22 @@ private LocalIndexService() upsertMode = Boolean.parseBoolean(ConfigFactory.Instance().getProperty("retina.upsert-mode.enabled")); } + /** + * Install the visible file-id set supplier. Polled on every + * {@link #resolvePrimary} call; a null return disables the filter. + * Node-local; not exposed on the {@link IndexService} interface. + * + * @param supplier non-null; use {@code () -> null} to disable + */ + public void setBaselineVisibleFilesSupplier(Supplier> supplier) + { + if (supplier == null) + { + throw new IllegalArgumentException("supplier must not be null; use () -> null to disable"); + } + this.baselineVisibleFilesSupplier = supplier; + } + @Override public IndexProto.RowIdBatch allocateRowIdBatch(long tableId, int numRowIds) throws IndexException { @@ -60,34 +89,10 @@ public IndexProto.RowIdBatch allocateRowIdBatch(long tableId, int numRowIds) thr @Override public IndexProto.RowLocation lookupUniqueIndex(IndexProto.IndexKey key, IndexOption indexOption) throws IndexException { - try - { - long tableId = key.getTableId(); - long indexId = key.getIndexId(); - MainIndex mainIndex = MainIndexFactory.Instance().getMainIndex(tableId); - SinglePointIndex singlePointIndex = SinglePointIndexFactory.Instance().getSinglePointIndex(tableId, indexId, indexOption); - long rowId = singlePointIndex.getUniqueRowId(key); - if (rowId >= 0) - { - IndexProto.RowLocation rowLocation = mainIndex.getLocation(rowId); - if (rowLocation != null) - { - return rowLocation; - } - else - { - throw new IndexException("Failed to get row location for rowId=" + rowId); - } - } - else - { - return null; - } - } - catch (SinglePointIndexException | MainIndexException e) - { - throw new IndexException("Failed to lookup unique index for key=" + key, e); - } + // Delegates to resolvePrimary; only backend errors throw, everything else returns null. + List> resolved = resolvePrimary( + key.getTableId(), key.getIndexId(), Collections.singletonList(key), indexOption); + return resolved.get(0).map(ResolvedPrimary::getRowLocation).orElse(null); } @Override @@ -134,71 +139,23 @@ public List lookupNonUniqueIndex(IndexProto.IndexKey key @Override public boolean putPrimaryIndexEntry(IndexProto.PrimaryIndexEntry entry, IndexOption indexOption) throws IndexException { - try - { - IndexProto.IndexKey key = entry.getIndexKey(); - long tableId = key.getTableId(); - long indexId = key.getIndexId(); - MainIndex mainIndex = MainIndexFactory.Instance().getMainIndex(tableId); - SinglePointIndex singlePointIndex = SinglePointIndexFactory.Instance().getSinglePointIndex(tableId, indexId, indexOption); - // Insert into single point index - boolean spSuccess = singlePointIndex.putEntry(entry.getIndexKey(), entry.getRowId()); - if (!spSuccess) - { - throw new IndexException("Failed to put entry into single point index for key=" + key); - } - // Insert into main index - boolean mainSuccess = mainIndex.putEntry(entry.getRowId(), entry.getRowLocation()); - if (!mainSuccess) - { - throw new IndexException("Failed to put entry into main index for rowId=" + entry.getRowId()); - } - return true; - } - catch (SinglePointIndexException e) - { - throw new IndexException("Failed to put entry into single point index for key=" + entry.getIndexKey(), e); - } - catch (MainIndexException e) - { - throw new IndexException("Failed to put entry into main index for rowId=" + entry.getRowId(), e); - } + // Delegates to putPrimaryIndexEntries. + IndexProto.IndexKey key = entry.getIndexKey(); + return putPrimaryIndexEntries(key.getTableId(), key.getIndexId(), + Collections.singletonList(entry), indexOption); } @Override public boolean putPrimaryIndexEntries(long tableId, long indexId, List entries, IndexOption indexOption) throws IndexException { - try + if (entries == null || entries.isEmpty()) { - SinglePointIndex singlePointIndex = SinglePointIndexFactory.Instance().getSinglePointIndex(tableId, indexId, indexOption); - // Batch insert into single point index - boolean success = singlePointIndex.putPrimaryEntries(entries); - if (!success) - { - throw new IndexException("Failed to put primary entries into single point index, tableId=" - + tableId + ", indexId=" + indexId); - } - MainIndex mainIndex = MainIndexFactory.Instance().getMainIndex(tableId); - for (Boolean mainSuccess : mainIndex.putEntries(entries)) - { - if(!mainSuccess) - { - throw new MainIndexException("Failed to put entry into main index, tableId: " + tableId); - } - } return true; } - catch (SinglePointIndexException e) - { - throw new IndexException("Failed to put primary entries into single point index, tableId=" - + tableId + ", indexId=" + indexId, e); - } - catch (MainIndexException e) - { - // Retained for consistency with original code, though normally not expected here - throw new IndexException("Failed to put primary entries into main index, tableId=" - + tableId + ", indexId=" + indexId, e); - } + // Crash-safe order: MainIndex first (rowId -> RowLocation), then primary (IndexKey -> rowId). + putMainIndexEntriesOnly(tableId, entries); + putPrimaryIndexEntriesOnly(tableId, indexId, entries, indexOption); + return true; } @Override @@ -633,4 +590,184 @@ public boolean removeIndex(long tableId, long indexId, boolean isPrimary, IndexO throw new IndexException("Failed to remove index for tableId=" + tableId + ", indexId=" + indexId, e); } } + + // ================================================================================== + // Staged primary-index APIs. Contracts live on the matching IndexService methods. + // ================================================================================== + + @Override + public List> resolvePrimary(long tableId, long indexId, + List keys, IndexOption indexOption) throws IndexException + { + if (keys == null || keys.isEmpty()) + { + return Collections.emptyList(); + } + // null = filter disabled + Set visibleFiles = baselineVisibleFilesSupplier.get(); + try + { + SinglePointIndex sp = SinglePointIndexFactory.Instance().getSinglePointIndex(tableId, indexId, indexOption); + MainIndex mi = MainIndexFactory.Instance().getMainIndex(tableId); + List> result = new ArrayList<>(keys.size()); + for (IndexProto.IndexKey key : keys) + { + long rowId = sp.getUniqueRowId(key); + if (rowId < 0) + { + // missing or tombstoned in primary + result.add(Optional.empty()); + continue; + } + IndexProto.RowLocation location = mi.getLocation(rowId); + if (location == null) + { + // MainIndex orphan rowId + result.add(Optional.empty()); + continue; + } + if (visibleFiles != null && !visibleFiles.contains(location.getFileId())) + { + // fileId outside baseline visible set + result.add(Optional.empty()); + continue; + } + result.add(Optional.of(new ResolvedPrimary(rowId, location))); + } + return result; + } + catch (SinglePointIndexException | MainIndexException e) + { + throw new IndexException("Failed to resolve primary for tableId=" + tableId + + ", indexId=" + indexId, e); + } + } + + @Override + public void putMainIndexEntriesOnly(long tableId, List entries) throws IndexException + { + if (entries == null || entries.isEmpty()) + { + return; + } + try + { + MainIndex mainIndex = MainIndexFactory.Instance().getMainIndex(tableId); + List results = mainIndex.putEntries(entries); + for (Boolean ok : results) + { + if (ok == null || !ok) + { + throw new IndexException("Failed to put main index entry, tableId=" + tableId); + } + } + } + catch (MainIndexException e) + { + throw new IndexException("Failed to put main index entries for tableId=" + tableId, e); + } + } + + @Override + public void putPrimaryIndexEntriesOnly(long tableId, long indexId, + List entries, IndexOption indexOption) throws IndexException + { + if (entries == null || entries.isEmpty()) + { + return; + } + try + { + SinglePointIndex sp = SinglePointIndexFactory.Instance().getSinglePointIndex(tableId, indexId, indexOption); + if (!sp.putPrimaryEntries(entries)) + { + throw new IndexException("Failed to put primary entries into single point index for tableId=" + + tableId + ", indexId=" + indexId); + } + } + catch (SinglePointIndexException e) + { + throw new IndexException("Failed to put primary entries into single point index for tableId=" + + tableId + ", indexId=" + indexId, e); + } + } + + @Override + public void deletePrimaryIndexEntriesOnly(long tableId, long indexId, + List resolvedKeys, IndexOption indexOption) throws IndexException + { + if (resolvedKeys == null || resolvedKeys.isEmpty()) + { + return; + } + try + { + SinglePointIndex sp = SinglePointIndexFactory.Instance().getSinglePointIndex(tableId, indexId, indexOption); + // TODO: avoid the repeated primary lookup by adding a tombstone-only index API. + sp.deleteEntries(resolvedKeys); + } + catch (SinglePointIndexException e) + { + throw new IndexException("Failed to delete primary entries for tableId=" + + tableId + ", indexId=" + indexId, e); + } + } + + @Override + public void updatePrimaryIndexEntriesOnly(long tableId, long indexId, + List entries, IndexOption indexOption) throws IndexException + { + if (entries == null || entries.isEmpty()) + { + return; + } + try + { + SinglePointIndex sp = SinglePointIndexFactory.Instance().getSinglePointIndex(tableId, indexId, indexOption); + // TODO: avoid the repeated primary lookup by adding an update API that accepts resolved rowIds. + sp.updatePrimaryEntries(entries); + } + catch (SinglePointIndexException e) + { + throw new IndexException("Failed to update primary entries for tableId=" + + tableId + ", indexId=" + indexId, e); + } + } + + @Override + public void restorePrimaryIndexEntries(long tableId, long indexId, + List entries, IndexOption indexOption) throws IndexException + { + if (entries == null || entries.isEmpty()) + { + return; + } + // RECOVERING is single-threaded for these entries; read-then-write needs no CAS. + try + { + SinglePointIndex sp = SinglePointIndexFactory.Instance().getSinglePointIndex(tableId, indexId, indexOption); + List toRestore = new ArrayList<>(); + for (RollbackEntry entry : entries) + { + long current = sp.getUniqueRowId(entry.getIndexKey()); + if (current == entry.getNewRowId()) + { + toRestore.add(IndexProto.PrimaryIndexEntry.newBuilder() + .setIndexKey(entry.getIndexKey()) + .setRowId(entry.getOldRowId()) + .build()); + } + // else: primary already tombstoned, reverted, or moved on; skip. + } + if (!toRestore.isEmpty()) + { + sp.updatePrimaryEntries(toRestore); + } + } + catch (SinglePointIndexException e) + { + throw new IndexException("Failed to restore primary entries for tableId=" + + tableId + ", indexId=" + indexId, e); + } + } } diff --git a/pixels-common/src/test/java/io/pixelsdb/pixels/common/index/TestLocalIndexService.java b/pixels-common/src/test/java/io/pixelsdb/pixels/common/index/TestLocalIndexService.java index a76b723904..117cad16c5 100644 --- a/pixels-common/src/test/java/io/pixelsdb/pixels/common/index/TestLocalIndexService.java +++ b/pixels-common/src/test/java/io/pixelsdb/pixels/common/index/TestLocalIndexService.java @@ -24,8 +24,13 @@ import io.pixelsdb.pixels.index.IndexProto; import org.junit.jupiter.api.*; +import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.function.Supplier; import static org.junit.jupiter.api.Assertions.*; @@ -161,4 +166,177 @@ void testCloseAndRemoveIndex() throws Exception assertTrue(indexService.removeIndex(TABLE_ID, PRIMARY_INDEX_ID, true, indexOption)); assertTrue(indexService.removeIndex(TABLE_ID, SECONDARY_INDEX_ID, false, indexOption)); } + + // ===================================================================== + // Staged primary-index API tests. These run after the legacy tests have + // closed/removed the index, so each test re-opens its own (tableId, indexId) + // pair to stay isolated. + // ===================================================================== + + private static final long STAGED_TABLE_ID = 9001L; + private static final long STAGED_PRIMARY_INDEX_ID = 9002L; + + private static IndexProto.PrimaryIndexEntry stagedEntry(String keyStr, long rowId, long fileId, int rgId, int rgOffset) + { + return IndexProto.PrimaryIndexEntry.newBuilder() + .setRowId(rowId) + .setIndexKey(IndexProto.IndexKey.newBuilder() + .setTableId(STAGED_TABLE_ID) + .setIndexId(STAGED_PRIMARY_INDEX_ID) + .setKey(ByteString.copyFromUtf8(keyStr)) + .setTimestamp(1000L)) + .setRowLocation(IndexProto.RowLocation.newBuilder() + .setFileId(fileId).setRgId(rgId).setRgRowOffset(rgOffset)) + .build(); + } + + @Test + @Order(10) + void testStagedPutMainIndexThenPutPrimaryRoundTrip() throws Exception + { + IndexOption opt = IndexOption.builder().vNodeId(0).build(); + assertTrue(indexService.openIndex(STAGED_TABLE_ID, STAGED_PRIMARY_INDEX_ID, true, opt)); + + IndexProto.RowIdBatch batch = indexService.allocateRowIdBatch(STAGED_TABLE_ID, 2); + long row0 = batch.getRowIdStart(); + long row1 = row0 + 1; + IndexProto.PrimaryIndexEntry e0 = stagedEntry("staged-k0", row0, 100L, 0, 0); + IndexProto.PrimaryIndexEntry e1 = stagedEntry("staged-k1", row1, 100L, 0, 1); + + indexService.putMainIndexEntriesOnly(STAGED_TABLE_ID, Arrays.asList(e0, e1)); + indexService.putPrimaryIndexEntriesOnly(STAGED_TABLE_ID, STAGED_PRIMARY_INDEX_ID, Arrays.asList(e0, e1), opt); + + List> resolved = indexService.resolvePrimary( + STAGED_TABLE_ID, STAGED_PRIMARY_INDEX_ID, + Arrays.asList(e0.getIndexKey(), e1.getIndexKey()), opt); + assertEquals(2, resolved.size()); + assertTrue(resolved.get(0).isPresent()); + assertEquals(row0, resolved.get(0).get().getRowId()); + assertEquals(100L, resolved.get(0).get().getRowLocation().getFileId()); + assertTrue(resolved.get(1).isPresent()); + assertEquals(row1, resolved.get(1).get().getRowId()); + } + + @Test + @Order(11) + void testStagedResolvePrimaryReturnsEmptyForUnknownKey() throws Exception + { + IndexOption opt = IndexOption.builder().vNodeId(0).build(); + IndexProto.IndexKey unknown = IndexProto.IndexKey.newBuilder() + .setTableId(STAGED_TABLE_ID) + .setIndexId(STAGED_PRIMARY_INDEX_ID) + .setKey(ByteString.copyFromUtf8("staged-not-there")) + .setTimestamp(1000L) + .build(); + + List> resolved = indexService.resolvePrimary( + STAGED_TABLE_ID, STAGED_PRIMARY_INDEX_ID, Collections.singletonList(unknown), opt); + assertEquals(1, resolved.size()); + assertFalse(resolved.get(0).isPresent()); + } + + @Test + @Order(12) + void testStagedResolvePrimaryAppliesBaselineVisibleFilesFilter() throws Exception + { + IndexOption opt = IndexOption.builder().vNodeId(0).build(); + // Install a baseline visible set that EXCLUDES fileId=100 (the one populated above). + Set visible = new HashSet<>(Collections.singletonList(999L)); + Supplier> originalSupplier = () -> null; + indexService.setBaselineVisibleFilesSupplier(() -> visible); + try + { + IndexProto.IndexKey k0 = stagedEntry("staged-k0", 0L, 100L, 0, 0).getIndexKey(); + List> resolved = indexService.resolvePrimary( + STAGED_TABLE_ID, STAGED_PRIMARY_INDEX_ID, Collections.singletonList(k0), opt); + assertEquals(1, resolved.size()); + assertFalse(resolved.get(0).isPresent(), + "RowLocation.fileId=100 must be filtered out by baseline visible set {999}"); + } + finally + { + // Reset to the default (no filtering) so subsequent tests see a clean state. + indexService.setBaselineVisibleFilesSupplier(originalSupplier); + } + } + + @Test + @Order(13) + void testStagedTombstonePrimaryResolvedIsIdempotent() throws Exception + { + IndexOption opt = IndexOption.builder().vNodeId(0).build(); + IndexProto.IndexKey k0 = stagedEntry("staged-k0", 0L, 100L, 0, 0).getIndexKey(); + + // First tombstone removes the live primary entry. + indexService.deletePrimaryIndexEntriesOnly(STAGED_TABLE_ID, STAGED_PRIMARY_INDEX_ID, + Collections.singletonList(k0), opt); + + List> resolved = indexService.resolvePrimary( + STAGED_TABLE_ID, STAGED_PRIMARY_INDEX_ID, Collections.singletonList(k0), opt); + assertFalse(resolved.get(0).isPresent()); + + // Repeated tombstone of an already-tombstoned key must be a no-op (idempotency invariant). + assertDoesNotThrow(() -> indexService.deletePrimaryIndexEntriesOnly( + STAGED_TABLE_ID, STAGED_PRIMARY_INDEX_ID, Collections.singletonList(k0), opt)); + } + + @Test + @Order(14) + void testStagedUpdateResolvedThenRestorePrimaryEntries() throws Exception + { + IndexOption opt = IndexOption.builder().vNodeId(0).build(); + IndexProto.IndexKey k1 = stagedEntry("staged-k1", 0L, 100L, 0, 1).getIndexKey(); + long oldRowId = indexService.resolvePrimary(STAGED_TABLE_ID, STAGED_PRIMARY_INDEX_ID, + Collections.singletonList(k1), opt).get(0).get().getRowId(); + + long newRowId = oldRowId + 100; + IndexProto.PrimaryIndexEntry newEntry = stagedEntry("staged-k1", newRowId, 101L, 0, 0); + indexService.putMainIndexEntriesOnly(STAGED_TABLE_ID, Collections.singletonList(newEntry)); + indexService.updatePrimaryIndexEntriesOnly(STAGED_TABLE_ID, STAGED_PRIMARY_INDEX_ID, + Collections.singletonList(newEntry), opt); + + Optional updated = indexService.resolvePrimary( + STAGED_TABLE_ID, STAGED_PRIMARY_INDEX_ID, Collections.singletonList(k1), opt).get(0); + assertTrue(updated.isPresent()); + assertEquals(newRowId, updated.get().getRowId()); + + indexService.restorePrimaryIndexEntries(STAGED_TABLE_ID, STAGED_PRIMARY_INDEX_ID, + Collections.singletonList(new RollbackEntry(k1, oldRowId, newRowId)), opt); + + Optional restored = indexService.resolvePrimary( + STAGED_TABLE_ID, STAGED_PRIMARY_INDEX_ID, Collections.singletonList(k1), opt).get(0); + assertTrue(restored.isPresent()); + assertEquals(oldRowId, restored.get().getRowId()); + } + + @Test + @Order(15) + void testStagedRestorePrimaryEntriesSkipsNonMatchingCurrent() throws Exception + { + IndexOption opt = IndexOption.builder().vNodeId(0).build(); + IndexProto.IndexKey k1 = stagedEntry("staged-k1", 0L, 100L, 0, 1).getIndexKey(); + long currentRowId = indexService.resolvePrimary(STAGED_TABLE_ID, STAGED_PRIMARY_INDEX_ID, + Collections.singletonList(k1), opt).get(0).get().getRowId(); + + // Rollback entry says: switch from newRowId=currentRowId+5 back to oldRowId=currentRowId-7. + // Since the actual current pointer is `currentRowId` (not newRowId=currentRowId+5), the + // restore must be a no-op. + RollbackEntry entry = new RollbackEntry(k1, currentRowId - 7, currentRowId + 5); + indexService.restorePrimaryIndexEntries(STAGED_TABLE_ID, STAGED_PRIMARY_INDEX_ID, + Collections.singletonList(entry), opt); + + // Verify primary still points at the original rowId, not the spurious oldRowId. + Optional after = indexService.resolvePrimary( + STAGED_TABLE_ID, STAGED_PRIMARY_INDEX_ID, Collections.singletonList(k1), opt).get(0); + assertTrue(after.isPresent()); + assertEquals(currentRowId, after.get().getRowId()); + } + + @Test + @Order(16) + void testStagedSetBaselineVisibleFilesSupplierRejectsNull() + { + assertThrows(IllegalArgumentException.class, + () -> indexService.setBaselineVisibleFilesSupplier(null)); + } } diff --git a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/retina/RetinaServerImpl.java b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/retina/RetinaServerImpl.java index fe5499072f..9f5434c566 100644 --- a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/retina/RetinaServerImpl.java +++ b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/retina/RetinaServerImpl.java @@ -27,8 +27,10 @@ import io.pixelsdb.pixels.common.exception.IndexException; import io.pixelsdb.pixels.common.exception.RetinaException; import io.pixelsdb.pixels.common.index.IndexOption; +import io.pixelsdb.pixels.common.index.ResolvedPrimary; import io.pixelsdb.pixels.common.index.service.IndexService; import io.pixelsdb.pixels.common.index.service.IndexServiceProvider; +import io.pixelsdb.pixels.common.index.service.LocalIndexService; import io.pixelsdb.pixels.common.metadata.MetadataService; import io.pixelsdb.pixels.common.metadata.domain.*; import io.pixelsdb.pixels.common.physical.Storage; @@ -288,10 +290,18 @@ public void updateRecord(RetinaProto.UpdateRecordRequest request, .setHeader(headerBuilder.build()) .build()); } - catch (RetinaException | IndexException e) + catch (RetinaException e) { - logger.error("updateRecord failed for schema={}", request.getSchemaName(), e); - headerBuilder.setErrorCode(1).setErrorMsg(e.getMessage()); + logger.error("updateRecord failed for schema={} (retina)", request.getSchemaName(), e); + headerBuilder.setErrorCode(1).setErrorMsg("Retina: " + e.getMessage()); + responseObserver.onNext(RetinaProto.UpdateRecordResponse.newBuilder() + .setHeader(headerBuilder.build()) + .build()); + } + catch (IndexException e) + { + logger.error("updateRecord failed for schema={} (index)", request.getSchemaName(), e); + headerBuilder.setErrorCode(2).setErrorMsg("Index: " + e.getMessage()); responseObserver.onNext(RetinaProto.UpdateRecordResponse.newBuilder() .setHeader(headerBuilder.build()) .build()); @@ -399,7 +409,7 @@ private List> transposeIndexKeys(List dataList, private void executeParallelByBucket( List dataList, java.util.function.Function keyExtractor, - BucketProcessor processor) throws RetinaException + BucketProcessor processor) throws RetinaException, IndexException { if (dataList == null || dataList.isEmpty()) { @@ -411,27 +421,47 @@ private void executeParallelByBucket( .collect(Collectors.groupingBy(d -> IndexUtils.getBucketIdFromByteBuffer(keyExtractor.apply(d).getKey()))); - // 2. Parallel Execution: Process each bucket in parallel + // 2. Parallel Execution: Process each bucket in parallel // This utilizes the common ForkJoinPool to execute RPCs and logic simultaneously - bucketMap.entrySet().parallelStream().forEach(entry -> + try { - int bucketId = entry.getKey(); - List subList = entry.getValue(); + bucketMap.entrySet().parallelStream().forEach(entry -> + { + int bucketId = entry.getKey(); + List subList = entry.getValue(); // Fetch the pre-initialized IndexOption from the pool (Zero allocation) - IndexOption option = this.indexOptionPool[bucketId]; + IndexOption option = this.indexOptionPool[bucketId]; - try - { + try + { // Execute the specific Delete/Insert/Update logic - processor.process(bucketId, subList, option); + processor.process(bucketId, subList, option); + } + catch (Exception e) + { + // Wrap checked exceptions to propagate through the parallel stream + throw new RuntimeException("Failure during parallel index processing for Bucket: " + bucketId, e); + } + }); + } + catch (RuntimeException e) + { + Throwable cause = e; + while (cause instanceof RuntimeException && cause.getCause() != null) + { + cause = cause.getCause(); } - catch (Exception e) + if (cause instanceof RetinaException) { - // Wrap checked exceptions to propagate through the parallel stream - throw new RuntimeException("Failure during parallel index processing for Bucket: " + bucketId, e); + throw (RetinaException) cause; } - }); + if (cause instanceof IndexException) + { + throw (IndexException) cause; + } + throw e; + } } /** @@ -469,6 +499,200 @@ private void processSecondaryIndexes( } } + /** + * Delete phase for one bucket. Hide existing rows before removing primary entries; + * secondary cleanup is best effort. + */ + private void executeStagedDeletePhase( + List subList, + java.util.function.Function> keyListExtractor, + long primaryIndexId, long timestamp, IndexOption option) throws IndexException, RetinaException + { + List> keysList = transposeIndexKeys(subList, keyListExtractor::apply); + List primaryKeys = keysList.get(0); + long tableId = primaryKeys.get(0).getTableId(); + + List> resolved = + indexService.resolvePrimary(tableId, primaryIndexId, primaryKeys, option); + List foundKeys = new ArrayList<>(primaryKeys.size()); + for (int i = 0; i < primaryKeys.size(); i++) + { + Optional r = resolved.get(i); + if (r.isPresent()) + { + this.retinaResourceManager.deleteRecord(r.get().getRowLocation(), timestamp); + foundKeys.add(primaryKeys.get(i)); + } + // Missing primary keys are no-op deletes. + } + if (!foundKeys.isEmpty()) + { + indexService.deletePrimaryIndexEntriesOnly(tableId, primaryIndexId, foundKeys, option); + } + + for (int i = 1; i < keysList.size(); ++i) + { + try + { + indexService.deleteSecondaryIndexEntries(tableId, + keysList.get(i).get(0).getIndexId(), keysList.get(i), option); + } + catch (IndexException e) + { + logger.warn("Best-effort staged secondary delete failed for tableId={}, indexId={}", + tableId, keysList.get(i).get(0).getIndexId(), e); + } + } + } + + /** + * Insert phase for one bucket. Write main index entries before primary entries + * so new primary mappings point to resolvable row locations. + */ + private void executeStagedInsertPhase( + String schemaName, String tableName, int virtualNodeId, + List subList, + java.util.function.Function> keyListExtractor, + java.util.function.Function> colValuesExtractor, + long primaryIndexId, long timestamp, IndexOption option) throws Exception + { + List primaryEntries = new ArrayList<>(subList.size()); + List rowIds = new ArrayList<>(subList.size()); + List insertedLocations = new ArrayList<>(subList.size()); + + try + { + for (T data : subList) + { + byte[][] values = colValuesExtractor.apply(data).stream() + .map(ByteString::toByteArray).toArray(byte[][]::new); + IndexProto.PrimaryIndexEntry.Builder builder = retinaResourceManager.insertRecord( + schemaName, tableName, values, timestamp, virtualNodeId); + builder.setIndexKey(keyListExtractor.apply(data).get(0)); + IndexProto.PrimaryIndexEntry entry = builder.build(); + primaryEntries.add(entry); + rowIds.add(entry.getRowId()); + insertedLocations.add(entry.getRowLocation()); + } + + long tableId = primaryEntries.get(0).getIndexKey().getTableId(); + indexService.putMainIndexEntriesOnly(tableId, primaryEntries); + indexService.putPrimaryIndexEntriesOnly(tableId, primaryIndexId, primaryEntries, option); + + processSecondaryIndexes(subList, keyListExtractor::apply, rowIds, option, false); + } + catch (Exception e) + { + for (IndexProto.RowLocation loc : insertedLocations) + { + try + { + this.retinaResourceManager.deleteRecord(loc, timestamp); + } + catch (Exception rollbackEx) + { + logger.error("Failed to roll back visibility for inserted row at fileId={}, rgId={}, rgRowOffset={}", + loc.getFileId(), loc.getRgId(), loc.getRgRowOffset(), rollbackEx); + } + } + throw e; + } + } + + /** + * Update phase for one bucket. Resolve current rows, append replacements, + * write main index entries, switch primary entries, then hide old rows. + */ + private void executeStagedUpdatePhase( + String schemaName, String tableName, int virtualNodeId, + int bucketId, + List subList, + java.util.function.Function> keyListExtractor, + java.util.function.Function> colValuesExtractor, + long primaryIndexId, long timestamp, IndexOption option) throws Exception + { + List primaryEntries = new ArrayList<>(subList.size()); + List rowIds = new ArrayList<>(subList.size()); + List insertedLocations = new ArrayList<>(subList.size()); + String lockKey = "v_" + virtualNodeId + "_b_" + bucketId + "_i_" + primaryIndexId; + Lock lock = updateLocks.get(lockKey); + + try + { + lock.lock(); + try + { + List> keysList = transposeIndexKeys(subList, keyListExtractor::apply); + List primaryKeys = keysList.get(0); + long tableId = primaryKeys.get(0).getTableId(); + + List> resolved = + indexService.resolvePrimary(tableId, primaryIndexId, primaryKeys, option); + if (resolved.size() != primaryKeys.size()) + { + throw new IndexException("Resolved primary count mismatch for tableId=" + + tableId + ", indexId=" + primaryIndexId); + } + + List previousLocations = new ArrayList<>(primaryKeys.size()); + for (int i = 0; i < primaryKeys.size(); i++) + { + Optional r = resolved.get(i); + if (!r.isPresent()) + { + throw new IndexException("Primary index entry not found for update, tableId=" + + tableId + ", indexId=" + primaryIndexId); + } + previousLocations.add(r.get().getRowLocation()); + } + + for (T data : subList) + { + byte[][] values = colValuesExtractor.apply(data).stream() + .map(ByteString::toByteArray).toArray(byte[][]::new); + IndexProto.PrimaryIndexEntry.Builder builder = retinaResourceManager.insertRecord( + schemaName, tableName, values, timestamp, virtualNodeId); + builder.setIndexKey(keyListExtractor.apply(data).get(0)); + IndexProto.PrimaryIndexEntry entry = builder.build(); + primaryEntries.add(entry); + rowIds.add(entry.getRowId()); + insertedLocations.add(entry.getRowLocation()); + } + + // TODO: replace this JVM-local lock with an index API that updates only when the + // resolved old rowIds still match, so concurrent writers can avoid bucket serialization. + indexService.putMainIndexEntriesOnly(tableId, primaryEntries); + indexService.updatePrimaryIndexEntriesOnly(tableId, primaryIndexId, primaryEntries, option); + for (IndexProto.RowLocation loc : previousLocations) + { + this.retinaResourceManager.deleteRecord(loc, timestamp); + } + } + finally + { + lock.unlock(); + } + + processSecondaryIndexes(subList, keyListExtractor::apply, rowIds, option, true); + } + catch (Exception e) + { + for (IndexProto.RowLocation loc : insertedLocations) + { + try + { + this.retinaResourceManager.deleteRecord(loc, timestamp); + } + catch (Exception rollbackEx) + { + logger.error("Failed to roll back visibility for inserted row at fileId={}, rgId={}, rgRowOffset={}", + loc.getFileId(), loc.getRgId(), loc.getRgRowOffset(), rollbackEx); + } + } + throw e; + } + } + /** * Common method to process updates for both normal and streaming rpc. * @@ -497,31 +721,11 @@ private void processUpdateRequest(RetinaProto.UpdateRecordRequest request) throw List deleteDataList = tableUpdateData.getDeleteDataList(); if (!deleteDataList.isEmpty()) { - // 1a. Validate the delete data validateIndexData(deleteDataList, d -> d.getIndexKeysList(), primaryIndexId, "Delete"); executeParallelByBucket(deleteDataList, d -> d.getIndexKeys(0), (bucketId, subList, option) -> - { - // 1b. Transpose the index keys - List> keysList = transposeIndexKeys(subList, RetinaProto.DeleteData::getIndexKeysList); - List primaryKeys = keysList.get(0); - long tableId = primaryKeys.get(0).getTableId(); - - // 1c. Delete primary index entries - List rowLocations = indexService.deletePrimaryIndexEntries(tableId, primaryIndexId, primaryKeys, option); - - // 1d. Delete records - for (IndexProto.RowLocation loc : rowLocations) - { - this.retinaResourceManager.deleteRecord(loc, timestamp); - } - - // 1e. Delete secondary index entries - for (int i = 1; i < keysList.size(); ++i) - { - indexService.deleteSecondaryIndexEntries(tableId, keysList.get(i).get(0).getIndexId(), keysList.get(i), option); - } - }); + executeStagedDeletePhase(subList, RetinaProto.DeleteData::getIndexKeysList, + primaryIndexId, timestamp, option)); } // ================================================================= @@ -530,123 +734,30 @@ private void processUpdateRequest(RetinaProto.UpdateRecordRequest request) throw List insertDataList = tableUpdateData.getInsertDataList(); if (!insertDataList.isEmpty()) { - // 2a. Validate the insert data validateIndexData(insertDataList, d -> d.getIndexKeysList(), primaryIndexId, "Insert"); executeParallelByBucket(insertDataList, d -> d.getIndexKeys(0), (bucketId, subList, option) -> - { - List primaryEntries = new ArrayList<>(subList.size()); - List rowIds = new ArrayList<>(subList.size()); - List insertedLocations = new ArrayList<>(subList.size()); - - try - { - // 2b. Insert records - for (RetinaProto.InsertData data : subList) - { - byte[][] values = data.getColValuesList().stream().map(ByteString::toByteArray).toArray(byte[][]::new); - IndexProto.PrimaryIndexEntry.Builder builder = retinaResourceManager.insertRecord(schemaName, tableName, values, timestamp, virtualNodeId); - builder.setIndexKey(data.getIndexKeys(0)); - IndexProto.PrimaryIndexEntry entry = builder.build(); - primaryEntries.add(entry); - rowIds.add(entry.getRowId()); - insertedLocations.add(entry.getRowLocation()); - } - - // 2c. Put primary index entries - long tableId = primaryEntries.get(0).getIndexKey().getTableId(); - indexService.putPrimaryIndexEntries(tableId, primaryIndexId, primaryEntries, option); - - // 2d. Put secondary index entries - processSecondaryIndexes(subList, RetinaProto.InsertData::getIndexKeysList, rowIds, option, false); - } - catch (Exception e) - { - for (IndexProto.RowLocation loc : insertedLocations) - { - try - { - this.retinaResourceManager.deleteRecord(loc, timestamp); - } - catch (Exception rollbackEx) - { - logger.error("Failed to roll back visibility for inserted row at fileId={}, rgId={}, rgRowOffset={}", - loc.getFileId(), loc.getRgId(), loc.getRgRowOffset(), rollbackEx); - } - } - throw e; - } - }); + executeStagedInsertPhase(schemaName, tableName, virtualNodeId, subList, + RetinaProto.InsertData::getIndexKeysList, + RetinaProto.InsertData::getColValuesList, + primaryIndexId, timestamp, option)); } // ================================================================= // 3. Process Update Data + // + // UpdateData keeps primary-index update semantics; new row locations + // are written before primary entries are switched. // ================================================================= List updateDataList = tableUpdateData.getUpdateDataList(); if (!updateDataList.isEmpty()) { - // 3a. Validate the update data validateIndexData(updateDataList, d -> d.getIndexKeysList(), primaryIndexId, "Update"); executeParallelByBucket(updateDataList, d -> d.getIndexKeys(0), (bucketId, subList, option) -> - { - List primaryEntries = new ArrayList<>(subList.size()); - List rowIds = new ArrayList<>(subList.size()); - List insertedLocations = new ArrayList<>(subList.size()); - - try - { - // 3b. Insert new records - for (RetinaProto.UpdateData data : subList) - { - byte[][] values = data.getColValuesList().stream().map(ByteString::toByteArray).toArray(byte[][]::new); - IndexProto.PrimaryIndexEntry.Builder builder = retinaResourceManager.insertRecord(schemaName, tableName, values, timestamp, virtualNodeId); - builder.setIndexKey(data.getIndexKeys(0)); - IndexProto.PrimaryIndexEntry entry = builder.build(); - primaryEntries.add(entry); - rowIds.add(entry.getRowId()); - insertedLocations.add(entry.getRowLocation()); - } - - // 3c. Update primary index entries with bucket-level locking - long tableId = primaryEntries.get(0).getIndexKey().getTableId(); - String lockKey = "v_" + virtualNodeId + "_b_" + bucketId + "_i_" + primaryIndexId; - Lock lock = updateLocks.get(lockKey); - - lock.lock(); - try - { - List prevLocs = indexService.updatePrimaryIndexEntries(tableId, primaryIndexId, primaryEntries, option); - // 3d. Delete previous records - for (IndexProto.RowLocation loc : prevLocs) - { - this.retinaResourceManager.deleteRecord(loc, timestamp); - } - } - finally - { - lock.unlock(); - } - - // 3e. Update secondary index entries - processSecondaryIndexes(subList, RetinaProto.UpdateData::getIndexKeysList, rowIds, option, true); - } - catch (Exception e) - { - for (IndexProto.RowLocation loc : insertedLocations) - { - try - { - this.retinaResourceManager.deleteRecord(loc, timestamp); - } - catch (Exception rollbackEx) - { - logger.error("Failed to roll back visibility for inserted row at fileId={}, rgId={}, rgRowOffset={}", - loc.getFileId(), loc.getRgId(), loc.getRgRowOffset(), rollbackEx); - } - } - throw e; - } - }); + executeStagedUpdatePhase(schemaName, tableName, virtualNodeId, bucketId, subList, + RetinaProto.UpdateData::getIndexKeysList, + RetinaProto.UpdateData::getColValuesList, + primaryIndexId, timestamp, option)); } } } diff --git a/pixels-daemon/src/test/java/io/pixelsdb/pixels/daemon/retina/TestRetinaServer.java b/pixels-daemon/src/test/java/io/pixelsdb/pixels/daemon/retina/TestRetinaServer.java index 43e6de6bac..2f0ffe8bb7 100644 --- a/pixels-daemon/src/test/java/io/pixelsdb/pixels/daemon/retina/TestRetinaServer.java +++ b/pixels-daemon/src/test/java/io/pixelsdb/pixels/daemon/retina/TestRetinaServer.java @@ -19,9 +19,14 @@ */ package io.pixelsdb.pixels.daemon.retina; +import com.google.protobuf.ByteString; +import io.grpc.stub.StreamObserver; +import io.pixelsdb.pixels.common.exception.IndexException; import io.pixelsdb.pixels.common.exception.MetadataException; import io.pixelsdb.pixels.common.exception.RetinaException; +import io.pixelsdb.pixels.common.index.ResolvedPrimary; import io.pixelsdb.pixels.common.index.service.IndexService; +import io.pixelsdb.pixels.common.index.service.LocalIndexService; import io.pixelsdb.pixels.common.metadata.MetadataService; import io.pixelsdb.pixels.common.metadata.domain.File; import io.pixelsdb.pixels.common.metadata.domain.Layout; @@ -31,19 +36,30 @@ import io.pixelsdb.pixels.common.metadata.domain.Table; import io.pixelsdb.pixels.daemon.ServerContainer; import io.pixelsdb.pixels.daemon.metadata.MetadataServer; +import io.pixelsdb.pixels.index.IndexProto; +import io.pixelsdb.pixels.retina.RetinaProto; import io.pixelsdb.pixels.retina.RetinaResourceManager; import org.junit.Ignore; import org.junit.Test; +import org.mockito.ArgumentMatchers; import org.mockito.InOrder; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicReference; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyLong; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.eq; import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -67,7 +83,7 @@ public void testRetinaServer() public void testRetinaServerImplInitializationFailureIsFailClosed() throws Exception { MetadataService metadataService = mock(MetadataService.class); - IndexService indexService = mock(IndexService.class); + IndexService indexService = mock(LocalIndexService.class); RetinaResourceManager resourceManager = mock(RetinaResourceManager.class); when(metadataService.getSchemas()).thenThrow(new MetadataException("metadata unavailable")); @@ -90,7 +106,7 @@ public void testRetinaServerImplInitializationFailureIsFailClosed() throws Excep public void testRetinaServerImplStartsBackgroundGcAfterSuccessfulInitialization() throws Exception { MetadataService metadataService = mock(MetadataService.class); - IndexService indexService = mock(IndexService.class); + IndexService indexService = mock(LocalIndexService.class); RetinaResourceManager resourceManager = mock(RetinaResourceManager.class); Schema schema = new Schema(); @@ -154,7 +170,7 @@ public void testRetinaServerImplStartsBackgroundGcAfterSuccessfulInitialization( public void testRetinaServerImplBackgroundGcStartFailureIsFailClosed() throws Exception { MetadataService metadataService = mock(MetadataService.class); - IndexService indexService = mock(IndexService.class); + IndexService indexService = mock(LocalIndexService.class); RetinaResourceManager resourceManager = mock(RetinaResourceManager.class); when(metadataService.getSchemas()).thenReturn(Collections.emptyList()); @@ -175,4 +191,479 @@ public void testRetinaServerImplBackgroundGcStartFailureIsFailClosed() throws Ex inOrder.verify(resourceManager).recoverCheckpoints(); inOrder.verify(resourceManager).startBackgroundGc(); } + + // ===================================================================== + // UpdateRecord write paths. + // ===================================================================== + + /** + * Build a RetinaServerImpl with the bare-minimum mocks needed to reach updateRecord + * without performing real metadata work or any background initialisation. + */ + private RetinaServerImpl buildServerWithLocalIndex(LocalIndexService localIndex, + RetinaResourceManager rm) throws Exception + { + MetadataService metadataService = mock(MetadataService.class); + when(metadataService.getSchemas()).thenReturn(Collections.emptyList()); + return new RetinaServerImpl(metadataService, localIndex, rm); + } + + private static IndexProto.IndexKey makeKey(long tableId, long indexId, String key, long ts) + { + return IndexProto.IndexKey.newBuilder() + .setTableId(tableId).setIndexId(indexId) + .setKey(ByteString.copyFromUtf8(key)) + .setTimestamp(ts) + .build(); + } + + private static IndexProto.RowLocation makeLoc(long fileId, int rgId, int rgRowOffset) + { + return IndexProto.RowLocation.newBuilder() + .setFileId(fileId).setRgId(rgId).setRgRowOffset(rgRowOffset) + .build(); + } + + private static RetinaProto.UpdateRecordRequest makeDeleteRequest(long tableId, long indexId, + String schema, String table, + long ts, String... keys) + { + RetinaProto.TableUpdateData.Builder tud = RetinaProto.TableUpdateData.newBuilder() + .setTableName(table) + .setPrimaryIndexId(indexId) + .setTimestamp(ts); + for (String k : keys) + { + tud.addDeleteData(RetinaProto.DeleteData.newBuilder() + .addIndexKeys(makeKey(tableId, indexId, k, ts))); + } + return RetinaProto.UpdateRecordRequest.newBuilder() + .setHeader(RetinaProto.RequestHeader.newBuilder().setToken("t")) + .setSchemaName(schema) + .addTableUpdateData(tud) + .build(); + } + + private static RetinaProto.UpdateRecordRequest makeInsertRequest(long tableId, long indexId, + String schema, String table, + long ts, String... keys) + { + RetinaProto.TableUpdateData.Builder tud = RetinaProto.TableUpdateData.newBuilder() + .setTableName(table) + .setPrimaryIndexId(indexId) + .setTimestamp(ts); + for (String k : keys) + { + tud.addInsertData(RetinaProto.InsertData.newBuilder() + .addIndexKeys(makeKey(tableId, indexId, k, ts)) + .addColValues(ByteString.copyFromUtf8("v-" + k))); + } + return RetinaProto.UpdateRecordRequest.newBuilder() + .setHeader(RetinaProto.RequestHeader.newBuilder().setToken("t")) + .setSchemaName(schema) + .addTableUpdateData(tud) + .build(); + } + + private static RetinaProto.UpdateRecordRequest makeDeleteWithSecondaryRequest( + long tableId, long primaryIndexId, long secondaryIndexId, + String schema, String table, long ts, String key) + { + RetinaProto.TableUpdateData.Builder tud = RetinaProto.TableUpdateData.newBuilder() + .setTableName(table) + .setPrimaryIndexId(primaryIndexId) + .setTimestamp(ts) + .addDeleteData(RetinaProto.DeleteData.newBuilder() + .addIndexKeys(makeKey(tableId, primaryIndexId, key, ts)) + .addIndexKeys(makeKey(tableId, secondaryIndexId, "sec-" + key, ts))); + return RetinaProto.UpdateRecordRequest.newBuilder() + .setHeader(RetinaProto.RequestHeader.newBuilder().setToken("t")) + .setSchemaName(schema) + .addTableUpdateData(tud) + .build(); + } + + private static IndexProto.PrimaryIndexEntry.Builder makePrimaryEntryBuilder( + IndexProto.IndexKey key, long rowId, IndexProto.RowLocation location) + { + return IndexProto.PrimaryIndexEntry.newBuilder() + .setIndexKey(key) + .setRowId(rowId) + .setRowLocation(location); + } + + @Test + public void testStagedDeleteCallsResolveBeforeDeleteRecordThenTombstone() throws Exception + { + LocalIndexService localIndex = mock(LocalIndexService.class); + RetinaResourceManager rm = mock(RetinaResourceManager.class); + + long tableId = 1L; + long indexId = 100L; + long ts = 12345L; + IndexProto.IndexKey foundKey = makeKey(tableId, indexId, "k-found", ts); + IndexProto.IndexKey missKey = makeKey(tableId, indexId, "k-miss", ts); + IndexProto.RowLocation foundLoc = makeLoc(7L, 0, 3); + + when(localIndex.resolvePrimary(eq(tableId), eq(indexId), + ArgumentMatchers.>any(), any())) + .thenReturn(Arrays.asList( + Optional.of(new ResolvedPrimary(42L, foundLoc)), + Optional.empty())); + + RetinaServerImpl server = buildServerWithLocalIndex(localIndex, rm); + + AtomicReference respHolder = new AtomicReference<>(); + StreamObserver observer = new StreamObserver() + { + @Override public void onNext(RetinaProto.UpdateRecordResponse v) { respHolder.set(v); } + @Override public void onError(Throwable t) { fail(t.getMessage()); } + @Override public void onCompleted() { } + }; + + server.updateRecord(makeDeleteRequest(tableId, indexId, "s", "tbl", ts, "k-found", "k-miss"), observer); + assertNotNull(respHolder.get()); + assertEquals(0, respHolder.get().getHeader().getErrorCode()); + + InOrder inOrder = inOrder(localIndex, rm); + inOrder.verify(localIndex).resolvePrimary(eq(tableId), eq(indexId), + ArgumentMatchers.>any(), any()); + // Only the FOUND key triggers deleteRecord and contributes to the tombstone list. + inOrder.verify(rm).deleteRecord(eq(foundLoc), eq(ts)); + inOrder.verify(localIndex).deletePrimaryIndexEntriesOnly(eq(tableId), eq(indexId), + eq(Collections.singletonList(foundKey)), any()); + + verify(localIndex, never()).deletePrimaryIndexEntries(anyLong(), anyLong(), + ArgumentMatchers.>any(), any()); + } + + @Test + public void testStagedDeleteAllNotFoundProducesNoTombstone() throws Exception + { + LocalIndexService localIndex = mock(LocalIndexService.class); + RetinaResourceManager rm = mock(RetinaResourceManager.class); + + long tableId = 1L; + long indexId = 100L; + long ts = 1L; + when(localIndex.resolvePrimary(eq(tableId), eq(indexId), + ArgumentMatchers.>any(), any())) + .thenReturn(Collections.singletonList(Optional.empty())); + + RetinaServerImpl server = buildServerWithLocalIndex(localIndex, rm); + + AtomicReference respHolder = new AtomicReference<>(); + server.updateRecord(makeDeleteRequest(tableId, indexId, "s", "tbl", ts, "absent"), + new StreamObserver() + { + @Override public void onNext(RetinaProto.UpdateRecordResponse v) { respHolder.set(v); } + @Override public void onError(Throwable t) { fail(t.getMessage()); } + @Override public void onCompleted() { } + }); + + assertNotNull(respHolder.get()); + assertEquals(0, respHolder.get().getHeader().getErrorCode()); + verify(rm, never()).deleteRecord(any(IndexProto.RowLocation.class), anyLong()); + verify(localIndex, never()).deletePrimaryIndexEntriesOnly(anyLong(), anyLong(), + ArgumentMatchers.>any(), any()); + } + + @Test + public void testStagedDeleteSecondaryFailureIsBestEffort() throws Exception + { + LocalIndexService localIndex = mock(LocalIndexService.class); + RetinaResourceManager rm = mock(RetinaResourceManager.class); + + long tableId = 1L; + long primaryIndexId = 100L; + long secondaryIndexId = 200L; + long ts = 9L; + IndexProto.RowLocation loc = makeLoc(7L, 0, 3); + when(localIndex.resolvePrimary(eq(tableId), eq(primaryIndexId), + ArgumentMatchers.>any(), any())) + .thenReturn(Collections.singletonList(Optional.of(new ResolvedPrimary(42L, loc)))); + doThrow(new IndexException("secondary already tombstoned")) + .when(localIndex).deleteSecondaryIndexEntries(eq(tableId), eq(secondaryIndexId), + ArgumentMatchers.>any(), any()); + + RetinaServerImpl server = buildServerWithLocalIndex(localIndex, rm); + AtomicReference respHolder = new AtomicReference<>(); + server.updateRecord(makeDeleteWithSecondaryRequest(tableId, primaryIndexId, secondaryIndexId, + "s", "tbl", ts, "k"), + new StreamObserver() + { + @Override public void onNext(RetinaProto.UpdateRecordResponse v) { respHolder.set(v); } + @Override public void onError(Throwable t) { fail(t.getMessage()); } + @Override public void onCompleted() { } + }); + + assertNotNull(respHolder.get()); + assertEquals(0, respHolder.get().getHeader().getErrorCode()); + verify(rm).deleteRecord(eq(loc), eq(ts)); + verify(localIndex).deletePrimaryIndexEntriesOnly(eq(tableId), eq(primaryIndexId), + ArgumentMatchers.>any(), any()); + } + + @Test + public void testStagedInsertWritesMainBeforePrimary() throws Exception + { + LocalIndexService localIndex = mock(LocalIndexService.class); + RetinaResourceManager rm = mock(RetinaResourceManager.class); + + long tableId = 1L; + long indexId = 100L; + long ts = 123L; + IndexProto.IndexKey key = makeKey(tableId, indexId, "k-insert", ts); + IndexProto.RowLocation loc = makeLoc(70L, 0, 4); + when(rm.insertRecord(eq("s"), eq("tbl"), ArgumentMatchers.any(), eq(ts), eq(0))) + .thenReturn(makePrimaryEntryBuilder(key, 51L, loc)); + + RetinaServerImpl server = buildServerWithLocalIndex(localIndex, rm); + AtomicReference respHolder = new AtomicReference<>(); + server.updateRecord(makeInsertRequest(tableId, indexId, "s", "tbl", ts, "k-insert"), + new StreamObserver() + { + @Override public void onNext(RetinaProto.UpdateRecordResponse v) { respHolder.set(v); } + @Override public void onError(Throwable t) { fail(t.getMessage()); } + @Override public void onCompleted() { } + }); + + assertNotNull(respHolder.get()); + assertEquals(0, respHolder.get().getHeader().getErrorCode()); + InOrder inOrder = inOrder(localIndex); + inOrder.verify(localIndex).putMainIndexEntriesOnly(eq(tableId), + ArgumentMatchers.>any()); + inOrder.verify(localIndex).putPrimaryIndexEntriesOnly(eq(tableId), eq(indexId), + ArgumentMatchers.>any(), any()); + } + + @Test + public void testStagedInsertPrimaryFailureMasksInsertedRows() throws Exception + { + LocalIndexService localIndex = mock(LocalIndexService.class); + RetinaResourceManager rm = mock(RetinaResourceManager.class); + + long tableId = 1L; + long indexId = 100L; + long ts = 124L; + IndexProto.IndexKey key0 = makeKey(tableId, indexId, "k0", ts); + IndexProto.IndexKey key1 = makeKey(tableId, indexId, "k1", ts); + IndexProto.RowLocation loc0 = makeLoc(71L, 0, 0); + IndexProto.RowLocation loc1 = makeLoc(71L, 0, 1); + when(rm.insertRecord(eq("s"), eq("tbl"), ArgumentMatchers.any(), eq(ts), eq(0))) + .thenReturn(makePrimaryEntryBuilder(key0, 61L, loc0), + makePrimaryEntryBuilder(key1, 62L, loc1)); + doThrow(new IndexException("primary write failed")) + .when(localIndex).putPrimaryIndexEntriesOnly(eq(tableId), eq(indexId), + ArgumentMatchers.>any(), any()); + + RetinaServerImpl server = buildServerWithLocalIndex(localIndex, rm); + AtomicReference respHolder = new AtomicReference<>(); + server.updateRecord(makeInsertRequest(tableId, indexId, "s", "tbl", ts, "k0", "k1"), + new StreamObserver() + { + @Override public void onNext(RetinaProto.UpdateRecordResponse v) { respHolder.set(v); } + @Override public void onError(Throwable t) { fail(t.getMessage()); } + @Override public void onCompleted() { } + }); + + assertNotNull(respHolder.get()); + assertEquals(2, respHolder.get().getHeader().getErrorCode()); + verify(rm).deleteRecord(eq(loc0), eq(ts)); + verify(rm).deleteRecord(eq(loc1), eq(ts)); + } + + @Test + public void testUpdateDataUsesStagedUpdateIndexPath() throws Exception + { + LocalIndexService localIndex = mock(LocalIndexService.class); + RetinaResourceManager rm = mock(RetinaResourceManager.class); + + long tableId = 1L; + long indexId = 100L; + long secondaryIndexId = 200L; + long ts = 7L; + IndexProto.IndexKey key = makeKey(tableId, indexId, "k-upd", ts); + IndexProto.IndexKey secondaryKey = makeKey(tableId, secondaryIndexId, "sec-k-upd", ts); + IndexProto.RowLocation prevLoc = makeLoc(7L, 0, 3); + IndexProto.RowLocation newLoc = makeLoc(70L, 0, 4); + + when(localIndex.resolvePrimary(eq(tableId), eq(indexId), + ArgumentMatchers.>any(), any())) + .thenReturn(Collections.singletonList(Optional.of(new ResolvedPrimary(42L, prevLoc)))); + when(rm.insertRecord(eq("s"), eq("tbl"), ArgumentMatchers.any(), eq(ts), eq(0))) + .thenReturn(makePrimaryEntryBuilder(key, 99L, newLoc)); + + RetinaProto.UpdateRecordRequest req = RetinaProto.UpdateRecordRequest.newBuilder() + .setHeader(RetinaProto.RequestHeader.newBuilder().setToken("t")) + .setSchemaName("s") + .addTableUpdateData(RetinaProto.TableUpdateData.newBuilder() + .setTableName("tbl") + .setPrimaryIndexId(indexId) + .setTimestamp(ts) + .addUpdateData(RetinaProto.UpdateData.newBuilder() + .addIndexKeys(key) + .addIndexKeys(secondaryKey) + .addColValues(ByteString.copyFromUtf8("v")))) + .build(); + + RetinaServerImpl server = buildServerWithLocalIndex(localIndex, rm); + AtomicReference respHolder = new AtomicReference<>(); + server.updateRecord(req, new StreamObserver() + { + @Override public void onNext(RetinaProto.UpdateRecordResponse v) { respHolder.set(v); } + @Override public void onError(Throwable t) { fail(t.getMessage()); } + @Override public void onCompleted() { } + }); + + assertNotNull(respHolder.get()); + assertEquals(0, respHolder.get().getHeader().getErrorCode()); + + InOrder inOrder = inOrder(localIndex, rm); + inOrder.verify(localIndex).resolvePrimary(eq(tableId), eq(indexId), + ArgumentMatchers.>any(), any()); + inOrder.verify(rm).insertRecord(eq("s"), eq("tbl"), ArgumentMatchers.any(), + eq(ts), eq(0)); + inOrder.verify(localIndex).putMainIndexEntriesOnly(eq(tableId), + ArgumentMatchers.>any()); + inOrder.verify(localIndex).updatePrimaryIndexEntriesOnly(eq(tableId), eq(indexId), + ArgumentMatchers.>any(), any()); + inOrder.verify(rm).deleteRecord(eq(prevLoc), eq(ts)); + inOrder.verify(localIndex).updateSecondaryIndexEntries(eq(tableId), eq(secondaryIndexId), + ArgumentMatchers.>any(), any()); + + verify(localIndex, never()).deletePrimaryIndexEntriesOnly(anyLong(), anyLong(), + ArgumentMatchers.>any(), any()); + verify(localIndex, never()).putPrimaryIndexEntriesOnly(anyLong(), anyLong(), + ArgumentMatchers.>any(), any()); + verify(localIndex, never()).updatePrimaryIndexEntries(anyLong(), anyLong(), + ArgumentMatchers.>any(), any()); + } + + @Test + public void testStagedUpdatePrimaryFailureMasksInsertedRows() throws Exception + { + LocalIndexService localIndex = mock(LocalIndexService.class); + RetinaResourceManager rm = mock(RetinaResourceManager.class); + + long tableId = 1L; + long indexId = 100L; + long ts = 8L; + IndexProto.IndexKey key = makeKey(tableId, indexId, "k-upd-fail", ts); + IndexProto.RowLocation prevLoc = makeLoc(7L, 0, 3); + IndexProto.RowLocation newLoc = makeLoc(70L, 0, 4); + + when(localIndex.resolvePrimary(eq(tableId), eq(indexId), + ArgumentMatchers.>any(), any())) + .thenReturn(Collections.singletonList(Optional.of(new ResolvedPrimary(42L, prevLoc)))); + when(rm.insertRecord(eq("s"), eq("tbl"), ArgumentMatchers.any(), eq(ts), eq(0))) + .thenReturn(makePrimaryEntryBuilder(key, 99L, newLoc)); + doThrow(new IndexException("primary update failed")) + .when(localIndex).updatePrimaryIndexEntriesOnly(eq(tableId), eq(indexId), + ArgumentMatchers.>any(), any()); + + RetinaProto.UpdateRecordRequest req = RetinaProto.UpdateRecordRequest.newBuilder() + .setHeader(RetinaProto.RequestHeader.newBuilder().setToken("t")) + .setSchemaName("s") + .addTableUpdateData(RetinaProto.TableUpdateData.newBuilder() + .setTableName("tbl") + .setPrimaryIndexId(indexId) + .setTimestamp(ts) + .addUpdateData(RetinaProto.UpdateData.newBuilder() + .addIndexKeys(key) + .addColValues(ByteString.copyFromUtf8("v")))) + .build(); + + RetinaServerImpl server = buildServerWithLocalIndex(localIndex, rm); + AtomicReference respHolder = new AtomicReference<>(); + server.updateRecord(req, new StreamObserver() + { + @Override public void onNext(RetinaProto.UpdateRecordResponse v) { respHolder.set(v); } + @Override public void onError(Throwable t) { fail(t.getMessage()); } + @Override public void onCompleted() { } + }); + + assertNotNull(respHolder.get()); + assertEquals(2, respHolder.get().getHeader().getErrorCode()); + + InOrder inOrder = inOrder(localIndex, rm); + inOrder.verify(localIndex).resolvePrimary(eq(tableId), eq(indexId), + ArgumentMatchers.>any(), any()); + inOrder.verify(rm).insertRecord(eq("s"), eq("tbl"), ArgumentMatchers.any(), + eq(ts), eq(0)); + inOrder.verify(localIndex).putMainIndexEntriesOnly(eq(tableId), + ArgumentMatchers.>any()); + inOrder.verify(localIndex).updatePrimaryIndexEntriesOnly(eq(tableId), eq(indexId), + ArgumentMatchers.>any(), any()); + inOrder.verify(rm).deleteRecord(eq(newLoc), eq(ts)); + verify(rm, never()).deleteRecord(eq(prevLoc), eq(ts)); + verify(localIndex, never()).putPrimaryIndexEntriesOnly(anyLong(), anyLong(), + ArgumentMatchers.>any(), any()); + } + + @Test + public void testStagedUpdateMissingPrimaryFailsBeforeAppend() throws Exception + { + LocalIndexService localIndex = mock(LocalIndexService.class); + RetinaResourceManager rm = mock(RetinaResourceManager.class); + + long tableId = 1L; + long indexId = 100L; + long ts = 9L; + IndexProto.IndexKey key = makeKey(tableId, indexId, "k-upd-missing", ts); + + when(localIndex.resolvePrimary(eq(tableId), eq(indexId), + ArgumentMatchers.>any(), any())) + .thenReturn(Collections.singletonList(Optional.empty())); + + RetinaProto.UpdateRecordRequest req = RetinaProto.UpdateRecordRequest.newBuilder() + .setHeader(RetinaProto.RequestHeader.newBuilder().setToken("t")) + .setSchemaName("s") + .addTableUpdateData(RetinaProto.TableUpdateData.newBuilder() + .setTableName("tbl") + .setPrimaryIndexId(indexId) + .setTimestamp(ts) + .addUpdateData(RetinaProto.UpdateData.newBuilder() + .addIndexKeys(key) + .addColValues(ByteString.copyFromUtf8("v")))) + .build(); + + RetinaServerImpl server = buildServerWithLocalIndex(localIndex, rm); + AtomicReference respHolder = new AtomicReference<>(); + server.updateRecord(req, new StreamObserver() + { + @Override public void onNext(RetinaProto.UpdateRecordResponse v) { respHolder.set(v); } + @Override public void onError(Throwable t) { fail(t.getMessage()); } + @Override public void onCompleted() { } + }); + + assertNotNull(respHolder.get()); + assertEquals(2, respHolder.get().getHeader().getErrorCode()); + verify(rm, never()).insertRecord(ArgumentMatchers.anyString(), ArgumentMatchers.anyString(), + ArgumentMatchers.any(), ArgumentMatchers.anyLong(), ArgumentMatchers.anyInt()); + verify(localIndex, never()).putMainIndexEntriesOnly(anyLong(), + ArgumentMatchers.>any()); + verify(localIndex, never()).updatePrimaryIndexEntriesOnly(anyLong(), anyLong(), + ArgumentMatchers.>any(), any()); + } + + @Test + public void testFailsClosedOnNonLocalIndexService() throws Exception + { + // UpdateRecord uses LocalIndexService-only primary-index operations. + IndexService nonLocal = mock(IndexService.class); + RetinaResourceManager rm = mock(RetinaResourceManager.class); + MetadataService md = mock(MetadataService.class); + try + { + new RetinaServerImpl(md, nonLocal, rm); + fail("RetinaServerImpl must require LocalIndexService"); + } + catch (IllegalStateException e) + { + assertTrue(e.getMessage().contains("LocalIndexService") + || (e.getCause() != null && e.getCause().getMessage() != null + && e.getCause().getMessage().contains("LocalIndexService"))); + } + } } diff --git a/pixels-index/pixels-index-main-sqlite/src/main/java/io/pixelsdb/pixels/index/main/sqlite/SqliteMainIndex.java b/pixels-index/pixels-index-main-sqlite/src/main/java/io/pixelsdb/pixels/index/main/sqlite/SqliteMainIndex.java index 6958692a18..35581dc2be 100644 --- a/pixels-index/pixels-index-main-sqlite/src/main/java/io/pixelsdb/pixels/index/main/sqlite/SqliteMainIndex.java +++ b/pixels-index/pixels-index-main-sqlite/src/main/java/io/pixelsdb/pixels/index/main/sqlite/SqliteMainIndex.java @@ -241,12 +241,9 @@ public IndexProto.RowLocation getLocation(long rowId) throws MainIndexException } if (location == null) { + // Return null when the rowId has no mapping in either the buffer or + // SQLite, leaving the caller to decide how to handle the miss. location = getRowLocationFromSqlite(rowId); - if (location == null) - { - throw new MainIndexException("Failed to get row location for rowId=" + rowId - + " (tableId=" + tableId + ")"); - } } return location; } @@ -260,18 +257,18 @@ public List getLocations(List rowIds) throws MainI { for (long rowId : rowIds) { - IndexProto.RowLocation location; - location = this.indexBuffer.lookup(rowId); + IndexProto.RowLocation location = this.indexBuffer.lookup(rowId); if (location == null) { location = getRowLocationFromSqlite(rowId); - if (location == null) - { - throw new MainIndexException("Failed to get row location for rowId=" + rowId - + " (tableId=" + tableId + ")"); - } } - builder.add(location); + // Skip rowIds that have no mapping in either the buffer or SQLite; + // the returned list contains only the resolvable locations and the + // caller decides how to handle the missing ones. + if (location != null) + { + builder.add(location); + } } } finally diff --git a/pixels-index/pixels-index-main-sqlite/src/test/java/io/pixelsdb/pixels/index/main/sqlite/TestSqliteMainIndex.java b/pixels-index/pixels-index-main-sqlite/src/test/java/io/pixelsdb/pixels/index/main/sqlite/TestSqliteMainIndex.java index 9313977b12..e16b8fdf48 100644 --- a/pixels-index/pixels-index-main-sqlite/src/test/java/io/pixelsdb/pixels/index/main/sqlite/TestSqliteMainIndex.java +++ b/pixels-index/pixels-index-main-sqlite/src/test/java/io/pixelsdb/pixels/index/main/sqlite/TestSqliteMainIndex.java @@ -813,9 +813,11 @@ private void assertLocation(long rowId, long fileId, int rgId, int rgRowOffset) Assertions.assertEquals(rgRowOffset, location.getRgRowOffset()); } - private void assertLocationMissing(long rowId) + private void assertLocationMissing(long rowId) throws MainIndexException { - Assertions.assertThrows(MainIndexException.class, () -> mainIndex.getLocation(rowId)); + // A missing rowId is reported as null so the caller can treat the absence + // as a logical not-found rather than a failure. + Assertions.assertNull(mainIndex.getLocation(rowId)); } private void assertFlushFailsAndBufferSurvives(long fileId, long firstRowId, long secondRowId) throws Exception From c20b625f83a1d6e5710c9a9c7ef2701309477e65 Mon Sep 17 00:00:00 2001 From: Dongyang Geng Date: Tue, 26 May 2026 00:47:56 +0800 Subject: [PATCH 15/17] feat(retina)!: introduce recovery checkpoint subsystem --- .../common/transaction/TransService.java | 17 +- .../pixels/common/utils/EtcdUtil.java | 31 + .../pixels/common/utils/RetinaUtils.java | 8 +- .../src/main/resources/pixels.properties | 10 +- .../reader/TestVisibilityCheckpointCache.java | 4 +- .../daemon/retina/RetinaServerImpl.java | 4 +- .../daemon/transaction/TransServiceImpl.java | 18 +- .../daemon/retina/TestRetinaServer.java | 6 +- pixels-retina/pom.xml | 1 - .../retina/IngestFileMetadataRegistry.java | 228 ----- .../pixels/retina/IngestFilePublisher.java | 11 + .../pixelsdb/pixels/retina/ObjectEntry.java | 13 +- .../pixels/retina/PixelsWriteBuffer.java | 54 +- .../pixels/retina/RecoveryCheckpoint.java | 789 ++++++++++++++++++ .../pixels/retina/RetinaResourceManager.java | 719 +++++++--------- .../retina/StorageGarbageCollector.java | 1 - .../TestIngestFileMetadataRegistry.java | 129 --- .../pixels/retina/TestPixelsWriteBuffer.java | 21 - .../pixels/retina/TestRetinaCheckpoint.java | 422 +--------- .../retina/TestStorageGarbageCollector.java | 112 --- proto/transaction.proto | 12 +- 21 files changed, 1249 insertions(+), 1361 deletions(-) delete mode 100644 pixels-retina/src/main/java/io/pixelsdb/pixels/retina/IngestFileMetadataRegistry.java create mode 100644 pixels-retina/src/main/java/io/pixelsdb/pixels/retina/RecoveryCheckpoint.java delete mode 100644 pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestIngestFileMetadataRegistry.java diff --git a/pixels-common/src/main/java/io/pixelsdb/pixels/common/transaction/TransService.java b/pixels-common/src/main/java/io/pixelsdb/pixels/common/transaction/TransService.java index 847f5de4b8..d0e5be784c 100644 --- a/pixels-common/src/main/java/io/pixelsdb/pixels/common/transaction/TransService.java +++ b/pixels-common/src/main/java/io/pixelsdb/pixels/common/transaction/TransService.java @@ -20,7 +20,6 @@ package io.pixelsdb.pixels.common.transaction; import com.google.common.collect.ImmutableList; -import com.google.protobuf.Empty; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; import io.pixelsdb.pixels.common.error.ErrorCode; @@ -498,12 +497,22 @@ public boolean bindExternalTraceId(long transId, String externalTraceId) throws return true; } - public long getSafeGcTimestamp() throws TransException + /** + * Get the safe upper bound (inclusive) for folding DELETE timestamps into + * the visibility base bitmap. + * + * @param includeRunningQueries whether the returned timestamp must remain safe for live running queries + */ + public long getSafeVisibilityFoldingTimestamp(boolean includeRunningQueries) throws TransException { - TransProto.GetSafeGcTimestampResponse response = this.stub.getSafeGcTimestamp(Empty.getDefaultInstance()); + TransProto.GetSafeVisibilityFoldingTimestampRequest request = + TransProto.GetSafeVisibilityFoldingTimestampRequest.newBuilder() + .setIncludeRunningQueries(includeRunningQueries).build(); + TransProto.GetSafeVisibilityFoldingTimestampResponse response = + this.stub.getSafeVisibilityFoldingTimestamp(request); if (response.getErrorCode() != ErrorCode.SUCCESS) { - throw new TransException("failed to get safe garbage collection timestamp" + throw new TransException("failed to get safe visibility folding timestamp, error code=" + response.getErrorCode()); } return response.getTimestamp(); diff --git a/pixels-common/src/main/java/io/pixelsdb/pixels/common/utils/EtcdUtil.java b/pixels-common/src/main/java/io/pixelsdb/pixels/common/utils/EtcdUtil.java index a0a2eea9fc..ff0df52922 100644 --- a/pixels-common/src/main/java/io/pixelsdb/pixels/common/utils/EtcdUtil.java +++ b/pixels-common/src/main/java/io/pixelsdb/pixels/common/utils/EtcdUtil.java @@ -24,7 +24,11 @@ import io.etcd.jetcd.KeyValue; import io.etcd.jetcd.Watch; import io.etcd.jetcd.kv.PutResponse; +import io.etcd.jetcd.kv.TxnResponse; import io.etcd.jetcd.lease.LeaseGrantResponse; +import io.etcd.jetcd.op.Cmp; +import io.etcd.jetcd.op.CmpTarget; +import io.etcd.jetcd.op.Op; import io.etcd.jetcd.options.DeleteOption; import io.etcd.jetcd.options.GetOption; import io.etcd.jetcd.options.PutOption; @@ -260,6 +264,33 @@ public long putKeyValueWithLeaseId(String key, String value, long leaseId) return 0L; } + /** + * Atomic compare-and-swap put. + * + * @param key + * @param expectedValue + * @param newValue + * @return true if the txn committed; false if CAS failed + */ + public boolean compareAndPut(String key, String expectedValue, String newValue) + throws ExecutionException, InterruptedException + { + ByteSequence keyBs = ByteSequence.from(key, StandardCharsets.UTF_8); + Cmp cmp = (expectedValue == null) + ? new Cmp(keyBs, Cmp.Op.EQUAL, CmpTarget.version(0L)) + : new Cmp(keyBs, Cmp.Op.EQUAL, CmpTarget.value( + ByteSequence.from(expectedValue, StandardCharsets.UTF_8))); + Op putOp = Op.put(keyBs, + ByteSequence.from(newValue, StandardCharsets.UTF_8), + PutOption.DEFAULT); + TxnResponse resp = this.client.getKVClient().txn() + .If(cmp) + .Then(putOp) + .commit() + .get(); + return resp.isSucceeded(); + } + /** * delete key-value by key. * diff --git a/pixels-common/src/main/java/io/pixelsdb/pixels/common/utils/RetinaUtils.java b/pixels-common/src/main/java/io/pixelsdb/pixels/common/utils/RetinaUtils.java index dc17eac21b..d132e253d2 100644 --- a/pixels-common/src/main/java/io/pixelsdb/pixels/common/utils/RetinaUtils.java +++ b/pixels-common/src/main/java/io/pixelsdb/pixels/common/utils/RetinaUtils.java @@ -29,8 +29,8 @@ public class RetinaUtils { - public static final String CHECKPOINT_PREFIX_GC = "vis_gc_"; public static final String CHECKPOINT_PREFIX_OFFLOAD = "vis_offload_"; + public static final String CHECKPOINT_PREFIX_RECOVERY = "recovery_"; public static final String CHECKPOINT_SUFFIX = ".bin"; private static volatile RetinaUtils instance; @@ -132,12 +132,12 @@ public static String getCheckpointPrefix(String typePrefix, String hostname) } /** - * Builds the checkpoint file path from a directory, prefix, hostname and timestamp. + * Builds the checkpoint file path from a directory, type prefix, hostname and identifier timestamp. * * @param checkpointDir directory where checkpoint files reside (may or may not end with '/') - * @param prefix {@link #CHECKPOINT_PREFIX_GC} or {@link #CHECKPOINT_PREFIX_OFFLOAD} + * @param prefix {@link #CHECKPOINT_PREFIX_OFFLOAD} or {@link #CHECKPOINT_PREFIX_RECOVERY} * @param hostname the retina host name - * @param timestamp the GC or offload timestamp + * @param timestamp the checkpoint identifier timestamp (offload ts for offload, applied ts for recovery) */ public static String buildCheckpointPath(String checkpointDir, String prefix, String hostname, long timestamp) { diff --git a/pixels-common/src/main/resources/pixels.properties b/pixels-common/src/main/resources/pixels.properties index 95587fb7c2..a8851b461d 100644 --- a/pixels-common/src/main/resources/pixels.properties +++ b/pixels-common/src/main/resources/pixels.properties @@ -291,8 +291,6 @@ retina.buffer.flush.count=20 retina.buffer.flush.interval=30 # interval in seconds for retina visibility garbage retina.gc.interval=300 -# number of threads for retina checkpoint -retina.checkpoint.threads=4 # retina buffer reader prefetch threads num retina.reader.prefetch.threads=8 # retina service init threads num @@ -305,8 +303,12 @@ retina.upsert-mode.enabled=false pixels.transaction.offload.threshold=1800 # lease duration for retina offload cache in seconds, default 600s retina.offload.cache.lease.duration=600 -# snapshot storage directory -retina.checkpoint.dir=file:///tmp/pixels-checkpoints +# number of threads for offload checkpoint writers +retina.offload.checkpoint.threads=4 +# storage URI for long-running query offload visibility snapshots; cleared on Retina startup +retina.offload.checkpoint.dir=file:///tmp/pixels-offload-checkpoints +# storage URI for recovery checkpoint body objects (one body per node per round) +retina.recovery.checkpoint.dir=file:///tmp/pixels-recovery-checkpoints # set to true to enable storage GC (rewrites high-deletion-ratio files to reclaim space) retina.storage.gc.enabled=false # invalidRatio must be strictly greater than this value for a file to be a GC candidate diff --git a/pixels-core/src/test/java/io/pixelsdb/pixels/core/reader/TestVisibilityCheckpointCache.java b/pixels-core/src/test/java/io/pixelsdb/pixels/core/reader/TestVisibilityCheckpointCache.java index e4ca0e3040..874b23d8db 100644 --- a/pixels-core/src/test/java/io/pixelsdb/pixels/core/reader/TestVisibilityCheckpointCache.java +++ b/pixels-core/src/test/java/io/pixelsdb/pixels/core/reader/TestVisibilityCheckpointCache.java @@ -46,7 +46,7 @@ public class TestVisibilityCheckpointCache @Before public void setUp() throws IOException { - testCheckpointDir = ConfigFactory.Instance().getProperty("retina.checkpoint.dir"); + testCheckpointDir = ConfigFactory.Instance().getProperty("retina.offload.checkpoint.dir"); storage = StorageFactory.Instance().getStorage(testCheckpointDir); if (!storage.exists(testCheckpointDir)) @@ -86,7 +86,7 @@ private void createDummyCheckpoint(String path, int numFiles, int rgsPerFile, lo public void testCacheLoading() throws Exception { long timestamp = 1000L; - String checkpointPath = resolve(testCheckpointDir, "vis_gc_tencent_100.bin"); + String checkpointPath = resolve(testCheckpointDir, "vis_offload_tencent_100.bin"); long[] dummyBitmap = new long[]{0x1L, 0x2L}; createDummyCheckpoint(checkpointPath, 1, 1, dummyBitmap); diff --git a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/retina/RetinaServerImpl.java b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/retina/RetinaServerImpl.java index 9f5434c566..b8c5f62374 100644 --- a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/retina/RetinaServerImpl.java +++ b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/retina/RetinaServerImpl.java @@ -114,7 +114,7 @@ public RetinaServerImpl() private void initializeRetinaResources() throws Exception { logger.info("Pre-loading checkpoints..."); - this.retinaResourceManager.recoverCheckpoints(); + this.retinaResourceManager.recoverOffloadCheckpoints(); List schemas = this.metadataService.getSchemas(); for (Schema schema : schemas) @@ -825,7 +825,7 @@ public void queryVisibility(RetinaProto.QueryVisibilityRequest request, .newBuilder() .setHeader(headerBuilder.build()); - String checkpointPath = this.retinaResourceManager.getCheckpointPath(timestamp); + String checkpointPath = this.retinaResourceManager.getOffloadCheckpointPath(timestamp); if (checkpointPath != null) { responseBuilder.setCheckpointPath(checkpointPath); diff --git a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/transaction/TransServiceImpl.java b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/transaction/TransServiceImpl.java index 94a7d7b958..06d49f464a 100644 --- a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/transaction/TransServiceImpl.java +++ b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/transaction/TransServiceImpl.java @@ -629,14 +629,18 @@ public void dumpTrans(TransProto.DumpTransRequest request, } @Override - public void getSafeGcTimestamp(com.google.protobuf.Empty request, - StreamObserver responseObserver) + public void getSafeVisibilityFoldingTimestamp(TransProto.GetSafeVisibilityFoldingTimestampRequest request, + StreamObserver responseObserver) { - long safeTs = Math.max(0, lowWatermark.get() - 1); - TransProto.GetSafeGcTimestampResponse response = TransProto.GetSafeGcTimestampResponse.newBuilder() - .setErrorCode(ErrorCode.SUCCESS) - .setTimestamp(safeTs) - .build(); + long writerSafeTs = Math.max(0, highWatermark.get() - 1); + long safeTs = request.getIncludeRunningQueries() + ? Math.min(lowWatermark.get(), writerSafeTs) + : writerSafeTs; + TransProto.GetSafeVisibilityFoldingTimestampResponse response = + TransProto.GetSafeVisibilityFoldingTimestampResponse.newBuilder() + .setErrorCode(ErrorCode.SUCCESS) + .setTimestamp(safeTs) + .build(); responseObserver.onNext(response); responseObserver.onCompleted(); } diff --git a/pixels-daemon/src/test/java/io/pixelsdb/pixels/daemon/retina/TestRetinaServer.java b/pixels-daemon/src/test/java/io/pixelsdb/pixels/daemon/retina/TestRetinaServer.java index 2f0ffe8bb7..6e3e360326 100644 --- a/pixels-daemon/src/test/java/io/pixelsdb/pixels/daemon/retina/TestRetinaServer.java +++ b/pixels-daemon/src/test/java/io/pixelsdb/pixels/daemon/retina/TestRetinaServer.java @@ -98,7 +98,7 @@ public void testRetinaServerImplInitializationFailureIsFailClosed() throws Excep assertTrue(e.getMessage().contains("Failed to initialize RetinaServerImpl")); } - verify(resourceManager).recoverCheckpoints(); + verify(resourceManager).recoverOffloadCheckpoints(); verify(resourceManager, never()).startBackgroundGc(); } @@ -137,7 +137,7 @@ public void testRetinaServerImplStartsBackgroundGcAfterSuccessfulInitialization( doAnswer(invocation -> { lifecycleEvents.add("recover"); return null; - }).when(resourceManager).recoverCheckpoints(); + }).when(resourceManager).recoverOffloadCheckpoints(); doAnswer(invocation -> { lifecycleEvents.add("visibility:" + invocation.getArgument(0)); return null; @@ -188,7 +188,7 @@ public void testRetinaServerImplBackgroundGcStartFailureIsFailClosed() throws Ex } InOrder inOrder = inOrder(resourceManager); - inOrder.verify(resourceManager).recoverCheckpoints(); + inOrder.verify(resourceManager).recoverOffloadCheckpoints(); inOrder.verify(resourceManager).startBackgroundGc(); } diff --git a/pixels-retina/pom.xml b/pixels-retina/pom.xml index f17e8b27af..b7a9357da4 100644 --- a/pixels-retina/pom.xml +++ b/pixels-retina/pom.xml @@ -88,7 +88,6 @@ io.etcd jetcd-core - test diff --git a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/IngestFileMetadataRegistry.java b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/IngestFileMetadataRegistry.java deleted file mode 100644 index 96b38fad08..0000000000 --- a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/IngestFileMetadataRegistry.java +++ /dev/null @@ -1,228 +0,0 @@ -/* - * Copyright 2026 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ -package io.pixelsdb.pixels.retina; - -import io.pixelsdb.pixels.common.exception.RetinaException; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -/** - * In-process source of truth for the ingest-path metadata - * {@code (fileId, tableId, virtualNodeId, firstBlockId)} of published REGULAR - * files. - *

    - * Callers must {@link #register} files of a given {@code (tableId, virtualNodeId)} - * stream in strictly increasing {@code firstBlockId} order. That ordering is the - * stream-append ordering enforced by {@link IngestFilePublisher} at publish time, - * and is what allows {@link #listByStream} to return ingest order via plain - * insertion order without an explicit sort. Out-of-order registration is treated - * as a publisher contract violation and fails closed. - *

    - * Commit timestamp bounds are intentionally omitted. A REGULAR file's commit - * timestamp bounds are persisted by PixelsWriter in - * {@code footer.hiddenColumnStats}, and callers that need per-segment - * timestamps for not-yet-published data consult the source memtable directly - * via {@link MemTable#getMinCommitTs()}. - */ -public class IngestFileMetadataRegistry -{ - private final Map entriesByFileId = new HashMap<>(); - private final Map> entriesByStream = new HashMap<>(); - - IngestFileMetadataRegistry() - { - } - - synchronized void register(long fileId, long tableId, int virtualNodeId, - long firstBlockId) throws RetinaException - { - Entry entry = new Entry(fileId, tableId, virtualNodeId, firstBlockId); - Entry existing = entriesByFileId.get(fileId); - if (existing != null) - { - if (existing.equals(entry)) - { - return; - } - throw new RetinaException("Conflicting ingest metadata registration for fileId=" + fileId); - } - - StreamKey streamKey = new StreamKey(tableId, virtualNodeId); - List streamEntries = entriesByStream.get(streamKey); - if (streamEntries != null && !streamEntries.isEmpty()) - { - Entry tail = streamEntries.get(streamEntries.size() - 1); - if (firstBlockId <= tail.getFirstBlockId()) - { - throw new RetinaException("Out-of-order ingest metadata registration for fileId=" + fileId - + ": firstBlockId=" + firstBlockId - + " must be strictly greater than prior tail firstBlockId=" + tail.getFirstBlockId()); - } - } - if (streamEntries == null) - { - streamEntries = new ArrayList<>(); - entriesByStream.put(streamKey, streamEntries); - } - entriesByFileId.put(fileId, entry); - streamEntries.add(entry); - } - - synchronized void unregister(long fileId) - { - Entry removed = entriesByFileId.remove(fileId); - if (removed == null) - { - return; - } - StreamKey streamKey = new StreamKey(removed.getTableId(), removed.getVirtualNodeId()); - List streamEntries = entriesByStream.get(streamKey); - if (streamEntries == null) - { - return; - } - streamEntries.removeIf(entry -> entry.getFileId() == fileId); - if (streamEntries.isEmpty()) - { - entriesByStream.remove(streamKey); - } - } - - synchronized Entry get(long fileId) throws RetinaException - { - Entry entry = entriesByFileId.get(fileId); - if (entry == null) - { - throw new RetinaException("Missing ingest metadata for fileId=" + fileId); - } - return entry; - } - - synchronized boolean contains(long fileId) - { - return entriesByFileId.containsKey(fileId); - } - - synchronized List listByStream(long tableId, int virtualNodeId) - { - List streamEntries = entriesByStream.get(new StreamKey(tableId, virtualNodeId)); - if (streamEntries == null) - { - return Collections.emptyList(); - } - return Collections.unmodifiableList(new ArrayList<>(streamEntries)); - } - - static final class Entry - { - private final long fileId; - private final long tableId; - private final int virtualNodeId; - private final long firstBlockId; - - Entry(long fileId, long tableId, int virtualNodeId, long firstBlockId) - { - this.fileId = fileId; - this.tableId = tableId; - this.virtualNodeId = virtualNodeId; - this.firstBlockId = firstBlockId; - } - - long getFileId() - { - return this.fileId; - } - - long getTableId() - { - return this.tableId; - } - - int getVirtualNodeId() - { - return this.virtualNodeId; - } - - long getFirstBlockId() - { - return this.firstBlockId; - } - - @Override - public boolean equals(Object o) - { - if (this == o) - { - return true; - } - if (!(o instanceof Entry)) - { - return false; - } - Entry entry = (Entry) o; - return fileId == entry.fileId && tableId == entry.tableId && - virtualNodeId == entry.virtualNodeId && firstBlockId == entry.firstBlockId; - } - - @Override - public int hashCode() - { - return Objects.hash(fileId, tableId, virtualNodeId, firstBlockId); - } - } - - private static final class StreamKey - { - private final long tableId; - private final int virtualNodeId; - - private StreamKey(long tableId, int virtualNodeId) - { - this.tableId = tableId; - this.virtualNodeId = virtualNodeId; - } - - @Override - public boolean equals(Object o) - { - if (this == o) - { - return true; - } - if (!(o instanceof StreamKey)) - { - return false; - } - StreamKey streamKey = (StreamKey) o; - return tableId == streamKey.tableId && virtualNodeId == streamKey.virtualNodeId; - } - - @Override - public int hashCode() - { - return Objects.hash(tableId, virtualNodeId); - } - } -} diff --git a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/IngestFilePublisher.java b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/IngestFilePublisher.java index 2fa7ccf503..c9ec95be4e 100644 --- a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/IngestFilePublisher.java +++ b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/IngestFilePublisher.java @@ -53,6 +53,17 @@ interface PublishAction this.nextCommitFirstBlockId = nextCommitFirstBlockId; } + /** + * The {@code firstBlockId} of the next FileWriterManager waiting to be + * published. Since block ids are assigned monotonically and commit + * timestamps are monotonic across blocks, this is the block whose + * minimum ts equals the buffer's earliest not-yet-published commit ts. + */ + synchronized long getNextCommitFirstBlockId() + { + return this.nextCommitFirstBlockId; + } + synchronized List admitReady(FileWriterManager fileWriterManager, PublishAction publishAction) throws RetinaException { diff --git a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/ObjectEntry.java b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/ObjectEntry.java index 1820b258ea..6df4a0fa61 100644 --- a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/ObjectEntry.java +++ b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/ObjectEntry.java @@ -31,13 +31,19 @@ public class ObjectEntry implements Referenceable private final long fileId; private final int startIndex; private final int length; + /** + * Min commit timestamp captured from the source memtable at flush time. + * {@link Long#MAX_VALUE} indicates "no rows captured". + */ + private final long minCommitTs; - public ObjectEntry(long id, long fileId, int startIndex, int length) + public ObjectEntry(long id, long fileId, int startIndex, int length, long minCommitTs) { this.id = id; this.fileId = fileId; this.startIndex = startIndex; this.length = length; + this.minCommitTs = minCommitTs; } public long getId() @@ -60,6 +66,11 @@ public int getLength() return this.length; } + public long getMinCommitTs() + { + return this.minCommitTs; + } + @Override public void ref() { diff --git a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/PixelsWriteBuffer.java b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/PixelsWriteBuffer.java index c80d8f9e22..799e487cbf 100644 --- a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/PixelsWriteBuffer.java +++ b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/PixelsWriteBuffer.java @@ -314,6 +314,9 @@ private void retireActiveMemTableLocked() throws RetinaException private void triggerFlushToObject(MemTable flushMemTable) { + // Capture ts before submitting: the memtable will be unref'd after + // flush, but checkpoint generation still needs its minCommitTs. + long capturedMinCommitTs = flushMemTable.getMinCommitTs(); flushObjectExecutor.submit(() -> { try { @@ -322,7 +325,7 @@ private void triggerFlushToObject(MemTable flushMemTable) this.objectStorageManager.write(this.tableId, virtualNodeId, id, flushMemTable.serialize()); ObjectEntry objectEntry = new ObjectEntry(id, flushMemTable.getFileId(), - flushMemTable.getStartIndex(), flushMemTable.getSize()); + flushMemTable.getStartIndex(), flushMemTable.getSize(), capturedMinCommitTs); objectEntry.ref(); // update watermark @@ -370,6 +373,52 @@ private void triggerFlushToObject(MemTable flushMemTable) }); } + public long getTableId() + { + return this.tableId; + } + + public int getVirtualNodeId() + { + return this.virtualNodeId; + } + + /** + * Earliest not-yet-published commit timestamp seen by this buffer. + */ + public long getEarliestPendingMinTs() + { + long nextBlockId = this.ingestFilePublisher.getNextCommitFirstBlockId(); + SuperVersion sv = getCurrentVersion(); + try + { + for (ObjectEntry oe : sv.getObjectEntries()) + { + if (oe.getId() == nextBlockId) + { + return oe.getMinCommitTs(); + } + } + for (MemTable mt : sv.getImmutableMemTables()) + { + if (mt.getId() == nextBlockId) + { + return mt.getMinCommitTs(); + } + } + MemTable activeMt = sv.getActiveMemTable(); + if (activeMt != null && activeMt.getId() == nextBlockId) + { + return activeMt.getMinCommitTs(); + } + return Long.MAX_VALUE; + } + finally + { + sv.unref(); + } + } + /** * Get the current version. * Caller must call unref(). @@ -451,9 +500,6 @@ private void publishPreparedFile(FileWriterManager fileWriterManager) throws Ret throw new RetinaException("Failed to publish ingest file " + fileWriterManager.getFileId() + " as REGULAR"); } - RetinaResourceManager.Instance().registerIngestFileMetadata( - fileWriterManager.getFileId(), tableId, fileWriterManager.getVirtualNodeId(), - fileWriterManager.getFirstBlockId()); } catch (MetadataException e) { throw new RetinaException("Failed to publish ingest file " diff --git a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/RecoveryCheckpoint.java b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/RecoveryCheckpoint.java new file mode 100644 index 0000000000..7fa1ec605c --- /dev/null +++ b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/RecoveryCheckpoint.java @@ -0,0 +1,789 @@ +/* + * Copyright 2026 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the Affero GNU General Public + * License along with Pixels. If not, see + * . + */ +package io.pixelsdb.pixels.retina; + +import io.etcd.jetcd.KeyValue; +import io.pixelsdb.pixels.common.exception.RetinaException; +import io.pixelsdb.pixels.common.physical.Storage; +import io.pixelsdb.pixels.common.physical.StorageFactory; +import io.pixelsdb.pixels.common.utils.ConfigFactory; +import io.pixelsdb.pixels.common.utils.EtcdUtil; +import io.pixelsdb.pixels.common.utils.NetUtils; +import io.pixelsdb.pixels.common.utils.RetinaUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.zip.CRC32; +import java.util.zip.CheckedOutputStream; + +/** + * Single owner of the recovery-checkpoint contract for a Retina host: + * binary format, value object ({@link Body} with its entry POJOs), and + * the etcd-pointer + Storage IO protocol that publishes and loads + * bodies. Catalog reconciliation, replay-start computation, and orphan + * retirement are not this class's concern; see {@link RecoveryProcedure}. + *

    + * High-level surface: + *

      + *
    • {@link #generate(long, List, List)} — given pre-collected + * {@code rgEntries} and {@code segments} captured by the caller at + * {@code checkpointAppliedTs}, sort canonically, serialise a body, + * write it through {@link Storage}, and publish the per-host etcd + * pointer via CAS. Idempotent across rounds: a no-op when + * {@code checkpointAppliedTs} has not advanced since the last + * successful round. Pure transform + IO; never reads back into RRM.
      • + *
    • {@link #load()} — read the etcd pointer, fetch the body it + * references, and run minimal header-level acceptability checks + * (matching {@code retinaNodeId}, sane {@code checkpointAppliedTs}, + * and a fail-closed {@code virtualNodesPerNode} match). Returns + * {@code null} when the pointer is absent or the body is unusable + * so the caller can fall back to fresh-deployment handling.
      • + *
    • {@link Body#serialize()} / {@link Body#readFrom(byte[])} — the + * on-disk format codec; bytes route through {@link CRC32} and the + * loader rejects bodies whose trailer length or CRC disagrees.
      • + *
    + */ +public final class RecoveryCheckpoint +{ + private static final Logger logger = LogManager.getLogger(RecoveryCheckpoint.class); + + // ============================================================ + // Section 1 — On-disk format constants + // ============================================================ + + private static final int MAGIC = 0x5052434B; + + /** Body length (4) + CRC32 (4). */ + private static final int TRAILER_SIZE = 4 + 4; + + /** Initial buffer capacity hint; ByteArrayOutputStream grows as needed. */ + private static final int INITIAL_BUFFER_HINT = 4 * 1024; + + private static final int WRITE_BUFFER = 4 * 1024 * 1024; + + // ============================================================ + // Section 2 — Configuration / IO state + // ============================================================ + + private final Storage storage; + private final String checkpointDir; + private final EtcdUtil etcd; + private final int virtualNodesPerNode; + private final String retinaNodeId; + private final String pointerKey; + /** Last checkpointAppliedTs that was successfully persisted; -1 before the first round. */ + private long lastFoldingTs = -1L; + + public RecoveryCheckpoint(Storage storage, + String checkpointDir, + EtcdUtil etcd, + int virtualNodesPerNode, + String retinaNodeId) + { + this.storage = storage; + this.checkpointDir = checkpointDir; + this.etcd = etcd; + this.virtualNodesPerNode = virtualNodesPerNode; + this.retinaNodeId = retinaNodeId; + this.pointerKey = "/pixels/retina/recovery/checkpoint/" + retinaNodeId + "/current"; + } + + /** + * Build a recovery checkpoint using the default wiring (service + * singletons, shared {@link EtcdUtil#Instance()}, body storage resolved + * from {@code retina.recovery.checkpoint.dir}). The local hostname is + * used as the per-host retinaNodeId. + */ + public static RecoveryCheckpoint createDefault() throws RetinaException + { + ConfigFactory config = ConfigFactory.Instance(); + String retinaNodeId = NetUtils.getLocalHostName(); + String dir = config.getProperty("retina.recovery.checkpoint.dir"); + String checkpointDir = trimTrailingSlash(dir); + Storage storage; + try { + storage = StorageFactory.Instance().getStorage(checkpointDir); + } catch (IOException e) { + throw new RetinaException("Failed to resolve storage for " + checkpointDir, e); + } + int virtualNodesPerNode = Integer.parseInt(config.getProperty("node.virtual.num")); + + return new RecoveryCheckpoint( + storage, + checkpointDir, + EtcdUtil.Instance(), + virtualNodesPerNode, + retinaNodeId); + } + + public int getVirtualNodesPerNode() + { + return virtualNodesPerNode; + } + + public String getRetinaNodeId() + { + return retinaNodeId; + } + + // ============================================================ + // Section 3 — Entry POJOs serialised inside a body + // ============================================================ + + /** + * Per-scope earliest unsafe-insert commit timestamp captured at + * checkpoint time: the smallest commit ts across the scope's + * pending/open {@link io.pixelsdb.pixels.retina.FileWriterManager}s. + * Already-published REGULAR files are not tracked separately in the + * body; their {@code fileId} appears in {@link VisibilityEntry} and + * that is the only ingest-path identity recovery needs. + */ + public static final class PendingSegmentEntry + { + private final long tableId; + private final int virtualNodeId; + private final long minCommitTs; + + public PendingSegmentEntry(long tableId, int virtualNodeId, long minCommitTs) + { + this.tableId = tableId; + this.virtualNodeId = virtualNodeId; + this.minCommitTs = minCommitTs; + } + + public long getTableId() { return tableId; } + public int getVirtualNodeId() { return virtualNodeId; } + public long getMinCommitTs() { return minCommitTs; } + } + + /** + * One {@code (fileId, rgId, bitmap)} entry captured by the recovery + * checkpoint. The bitmap folds every delete with + * {@code delete_ts <= baseTimestamp} into the base, so the loader can + * rebuild RGVisibility with an empty deletion chain. + */ + public static final class VisibilityEntry + { + private final long fileId; + private final int rgId; + private final int recordNum; + private final long baseTimestamp; + private final long[] bitmap; + + public VisibilityEntry(long fileId, int rgId, int recordNum, + long baseTimestamp, long[] bitmap) + { + this.fileId = fileId; + this.rgId = rgId; + this.recordNum = recordNum; + this.baseTimestamp = baseTimestamp; + this.bitmap = bitmap; + } + + public long getFileId() { return fileId; } + public int getRgId() { return rgId; } + public int getRecordNum() { return recordNum; } + public long getBaseTimestamp() { return baseTimestamp; } + public long[] getBitmap() { return bitmap; } + } + + // ============================================================ + // Section 4 — Body value object + format codec + // ============================================================ + + /** + * Immutable in-memory representation of one checkpoint body. + * Use {@link Body#builder()} to construct, {@link #serialize()} to + * write, and {@link #readFrom(byte[])} to parse; both routes thread + * header+payload through {@link CRC32}. + */ + public static final class Body + { + private final long writeTimeMs; + private final long checkpointSnapshotTs; + private final long checkpointAppliedTs; + /** FNV-1a hash of {@code retinaNodeId = host:port}, used as a defence-in-depth check. */ + private final long retinaNodeIdHash; + /** Value of {@code node.virtual.num} at checkpoint time; mismatch aborts recovery. */ + private final int virtualNodesPerNode; + /** Original retinaNodeId string, stored for diagnostics. */ + private final String retinaNodeId; + + private final List segmentEntries; + private final List rgEntries; + + private Body(Builder builder) + { + this.writeTimeMs = builder.writeTimeMs; + this.checkpointSnapshotTs = builder.checkpointSnapshotTs; + this.checkpointAppliedTs = builder.checkpointAppliedTs; + this.retinaNodeIdHash = fnv1a64(builder.retinaNodeId); + this.virtualNodesPerNode = builder.virtualNodesPerNode; + this.retinaNodeId = builder.retinaNodeId; + this.segmentEntries = Collections.unmodifiableList(new ArrayList<>(emptyIfNull(builder.segmentEntries))); + this.rgEntries = Collections.unmodifiableList(new ArrayList<>(emptyIfNull(builder.rgEntries))); + } + + public long getWriteTimeMs() { return writeTimeMs; } + public long getCheckpointSnapshotTs() { return checkpointSnapshotTs; } + public long getCheckpointAppliedTs() { return checkpointAppliedTs; } + public long getRetinaNodeIdHash() { return retinaNodeIdHash; } + public int getVirtualNodesPerNode() { return virtualNodesPerNode; } + public String getRetinaNodeId() { return retinaNodeId; } + public List getSegmentEntries() { return segmentEntries; } + public List getRgEntries() { return rgEntries; } + + /** + * Serialise this body and append the trailer (bodyLength + CRC32 over + * header+payload bytes). + */ + public byte[] serialize() throws IOException + { + ByteArrayOutputStream baos = new ByteArrayOutputStream(INITIAL_BUFFER_HINT); + CRC32 crc = new CRC32(); + CheckedOutputStream cos = new CheckedOutputStream(baos, crc); + DataOutputStream dos = new DataOutputStream(cos); + writeHeader(dos); + writePayload(dos); + dos.flush(); + + int bodyLen = baos.size(); + long crcValue = crc.getValue(); + DataOutputStream trailerOut = new DataOutputStream(baos); + trailerOut.writeInt(bodyLen); + trailerOut.writeInt((int) (crcValue & 0xFFFFFFFFL)); + trailerOut.flush(); + return baos.toByteArray(); + } + + private void writeHeader(DataOutputStream dos) throws IOException + { + dos.writeInt(MAGIC); + dos.writeLong(retinaNodeIdHash); + dos.writeLong(writeTimeMs); + dos.writeLong(checkpointSnapshotTs); + dos.writeLong(checkpointAppliedTs); + dos.writeInt(virtualNodesPerNode); + dos.writeInt(segmentEntries.size()); + dos.writeInt(rgEntries.size()); + } + + private void writePayload(DataOutputStream dos) throws IOException + { + byte[] nodeIdBytes = retinaNodeId.getBytes(StandardCharsets.UTF_8); + dos.writeInt(nodeIdBytes.length); + dos.write(nodeIdBytes); + + for (PendingSegmentEntry se : segmentEntries) + { + dos.writeLong(se.tableId); + dos.writeInt(se.virtualNodeId); + dos.writeLong(se.minCommitTs); + } + + for (VisibilityEntry ve : rgEntries) + { + dos.writeLong(ve.fileId); + dos.writeInt(ve.rgId); + dos.writeInt(ve.recordNum); + dos.writeLong(ve.baseTimestamp); + long[] bitmap = ve.bitmap; + int bitmapLen = bitmap == null ? 0 : bitmap.length; + dos.writeInt(bitmapLen); + for (int i = 0; i < bitmapLen; i++) + { + dos.writeLong(bitmap[i]); + } + } + } + + /** + * Parse the supplied bytes. Throws {@link RetinaException} on + * magic / version mismatch, truncated trailer, or CRC mismatch. + */ + public static Body readFrom(byte[] bytes) throws RetinaException + { + if (bytes == null || bytes.length < TRAILER_SIZE) + { + throw new RetinaException("body too small: " + (bytes == null ? -1 : bytes.length)); + } + + int trailerOffset = bytes.length - TRAILER_SIZE; + int declaredLen = readIntBE(bytes, trailerOffset); + int declaredCrc = readIntBE(bytes, trailerOffset + 4); + if (declaredLen != trailerOffset) + { + throw new RetinaException("trailer length mismatch: declared=" + declaredLen + + ", actual=" + trailerOffset); + } + CRC32 crc = new CRC32(); + crc.update(bytes, 0, trailerOffset); + long expected = ((long) declaredCrc) & 0xFFFFFFFFL; + if (crc.getValue() != expected) + { + throw new RetinaException("checksum mismatch: expected=" + expected + + ", actual=" + crc.getValue()); + } + + try (DataInputStream dis = new DataInputStream(new ByteArrayInputStream(bytes, 0, trailerOffset))) + { + int magic = dis.readInt(); + if (magic != MAGIC) + { + throw new RetinaException("bad magic: " + Integer.toHexString(magic)); + } + long retinaNodeIdHash = dis.readLong(); + long writeTimeMs = dis.readLong(); + long checkpointSnapshotTs = dis.readLong(); + long checkpointAppliedTs = dis.readLong(); + int virtualNodesPerNode = dis.readInt(); + int segmentEntryCount = dis.readInt(); + int rgEntryCount = dis.readInt(); + if (segmentEntryCount < 0 || rgEntryCount < 0) + { + throw new RetinaException("negative entry counts"); + } + + int nodeIdLen = dis.readInt(); + if (nodeIdLen < 0 || nodeIdLen > dis.available()) + { + throw new RetinaException("invalid retinaNodeId length: " + nodeIdLen); + } + byte[] nodeIdBytes = new byte[nodeIdLen]; + dis.readFully(nodeIdBytes); + String retinaNodeId = new String(nodeIdBytes, StandardCharsets.UTF_8); + long computedHash = fnv1a64(retinaNodeId); + if (computedHash != retinaNodeIdHash) + { + throw new RetinaException("retinaNodeId hash mismatch: header=" + + Long.toHexString(retinaNodeIdHash) + + ", body=" + Long.toHexString(computedHash)); + } + + List segments = new ArrayList<>(); + for (int i = 0; i < segmentEntryCount; i++) + { + long tableId = dis.readLong(); + int virtualNodeId = dis.readInt(); + long minCommitTs = dis.readLong(); + segments.add(new PendingSegmentEntry(tableId, virtualNodeId, minCommitTs)); + } + List rgs = new ArrayList<>(); + for (int i = 0; i < rgEntryCount; i++) + { + long fileId = dis.readLong(); + int rgId = dis.readInt(); + int recordNum = dis.readInt(); + long baseTimestamp = dis.readLong(); + int bitmapLen = dis.readInt(); + if (rgId < 0 || recordNum <= 0 || bitmapLen < 0 || bitmapLen > dis.available() / Long.BYTES) + { + throw new RetinaException("invalid visibility entry for fileId=" + fileId + + ", rgId=" + rgId + ", recordNum=" + recordNum + + ", bitmapLen=" + bitmapLen); + } + long[] bitmap = new long[bitmapLen]; + for (int j = 0; j < bitmapLen; j++) + { + bitmap[j] = dis.readLong(); + } + rgs.add(new VisibilityEntry(fileId, rgId, recordNum, baseTimestamp, bitmap)); + } + if (dis.available() != 0) + { + throw new RetinaException("trailing bytes after checkpoint payload: " + dis.available()); + } + + return Body.builder() + .retinaNodeId(retinaNodeId) + .writeTimeMs(writeTimeMs) + .checkpointSnapshotTs(checkpointSnapshotTs) + .checkpointAppliedTs(checkpointAppliedTs) + .virtualNodesPerNode(virtualNodesPerNode) + .segmentEntries(segments) + .rgEntries(rgs) + .build(); + } + catch (IOException e) + { + throw new RetinaException("failed to parse body", e); + } + } + + public static Builder builder() + { + return new Builder(); + } + + public static final class Builder + { + private long writeTimeMs; + private long checkpointSnapshotTs; + private long checkpointAppliedTs; + private int virtualNodesPerNode; + private String retinaNodeId; + private List segmentEntries = Collections.emptyList(); + private List rgEntries = Collections.emptyList(); + + public Builder writeTimeMs(long writeTimeMs) { this.writeTimeMs = writeTimeMs; return this; } + public Builder checkpointSnapshotTs(long ts) { this.checkpointSnapshotTs = ts; return this; } + public Builder checkpointAppliedTs(long ts) { this.checkpointAppliedTs = ts; return this; } + public Builder virtualNodesPerNode(int n) { this.virtualNodesPerNode = n; return this; } + public Builder retinaNodeId(String id) { this.retinaNodeId = id; return this; } + public Builder segmentEntries(List entries) { this.segmentEntries = entries; return this; } + public Builder rgEntries(List entries) { this.rgEntries = entries; return this; } + + public Body build() + { + if (retinaNodeId == null || retinaNodeId.isEmpty()) + { + throw new IllegalArgumentException("retinaNodeId is required"); + } + return new Body(this); + } + } + } + + // ============================================================ + // Section 5 — Round / load results + // ============================================================ + + /** Result of one successful checkpoint round. */ + public static final class Result + { + private final String bodyObjectName; + private final long checkpointAppliedTs; + private final int segmentEntryCount; + private final int rgEntryCount; + + public Result(String bodyObjectName, long checkpointAppliedTs, + int segmentEntryCount, int rgEntryCount) + { + this.bodyObjectName = bodyObjectName; + this.checkpointAppliedTs = checkpointAppliedTs; + this.segmentEntryCount = segmentEntryCount; + this.rgEntryCount = rgEntryCount; + } + + public String getBodyObjectName() { return bodyObjectName; } + public long getCheckpointAppliedTs() { return checkpointAppliedTs; } + public int getSegmentEntryCount() { return segmentEntryCount; } + public int getRgEntryCount() { return rgEntryCount; } + } + + /** Body loaded from the etcd pointer. */ + public static final class LoadedCheckpoint + { + public final String bodyObjectName; + public final Body body; + + LoadedCheckpoint(String bodyObjectName, Body body) + { + this.bodyObjectName = bodyObjectName; + this.body = body; + } + } + + // ============================================================ + // Section 6 — Write path: generate() + // ============================================================ + + /** + * @param checkpointAppliedTs the safe visibility folding timestamp at which + * the body should be snapshotted; supplied by the caller (typically + * the same value the surrounding GC cycle has just folded against) + * so the body reflects exactly that fold and TransService is not + * re-read here. + * @param rgEntries per-RG visibility entries already snapshotted by the + * caller against {@code checkpointAppliedTs} (typically collected + * in-line during Memory GC's single pass over RGVisibility, so the + * post-fold bitmap is reused without a second native traversal). + * Sorted in-place to the canonical on-disk order. + * @param segments per-scope earliest pending commit timestamps already + * snapshotted by the caller. Sorted in-place. + * @return result of this checkpoint round, or {@code null} when + * {@code checkpointAppliedTs} has not advanced since the last + * successful round (no new committed transactions, nothing to flush). + */ + public Result generate(long checkpointAppliedTs, + List rgEntries, + List segments) throws RetinaException + { + if (checkpointAppliedTs == lastFoldingTs) + { + logger.debug("Recovery checkpoint: checkpointAppliedTs={} unchanged since last round; skipping", + checkpointAppliedTs); + return null; + } + long now = System.currentTimeMillis(); + + rgEntries.sort((a, b) -> { + int byFile = Long.compare(a.getFileId(), b.getFileId()); + if (byFile != 0) return byFile; + return Integer.compare(a.getRgId(), b.getRgId()); + }); + sortSegments(segments); + + Body body = Body.builder() + .retinaNodeId(retinaNodeId) + .writeTimeMs(now) + .checkpointSnapshotTs(now) + .checkpointAppliedTs(checkpointAppliedTs) + .virtualNodesPerNode(virtualNodesPerNode) + .segmentEntries(segments) + .rgEntries(rgEntries) + .build(); + + String bodyObjectName = RetinaUtils.getCheckpointFileName( + RetinaUtils.CHECKPOINT_PREFIX_RECOVERY, retinaNodeId, checkpointAppliedTs); + String bodyPath = checkpointDir + "/" + bodyObjectName; + try + { + byte[] serialised = body.serialize(); + try (DataOutputStream out = storage.create(bodyPath, true, WRITE_BUFFER)) + { + out.write(serialised); + out.flush(); + } + } + catch (IOException e) + { + throw new RetinaException("Failed to write recovery checkpoint body " + bodyObjectName, e); + } + + // Body is durable; publish the pointer atomically. If publish fails the + // body becomes a one-round orphan and is overwritten/cleaned next round. + String displacedOld = publishPointer(bodyObjectName); + if (displacedOld != null && !displacedOld.isEmpty()) + { + String displacedPath = checkpointDir + "/" + displacedOld; + try + { + if (storage.exists(displacedPath)) + { + storage.delete(displacedPath, false); + } + } + catch (IOException e) + { + logger.warn("Failed to delete orphan checkpoint body {} under {}; will retry next round", + displacedOld, checkpointDir, e); + } + } + + logger.info("Recovery checkpoint published: body={}, checkpointAppliedTs={}, segments={}, rgs={}", + bodyObjectName, checkpointAppliedTs, + segments.size(), rgEntries.size()); + lastFoldingTs = checkpointAppliedTs; + return new Result(bodyObjectName, checkpointAppliedTs, + segments.size(), rgEntries.size()); + } + + /** + * Atomically replace the published checkpoint pointer. + * + * @return the displaced old body name (null on first publish). + */ + private String publishPointer(String newBodyName) throws RetinaException + { + String old = readPointer(); + boolean committed; + try + { + committed = etcd.compareAndPut(pointerKey, old, newBodyName); + } + catch (Exception e) + { + throw new RetinaException("etcd CAS failed for recovery checkpoint pointer " + pointerKey, e); + } + if (!committed) + { + throw new RetinaException("concurrent writer or stale snapshot on recovery checkpoint pointer " + + pointerKey); + } + return old; + } + + private static void sortSegments(List segments) + { + segments.sort((a, b) -> { + int byTable = Long.compare(a.getTableId(), b.getTableId()); + if (byTable != 0) return byTable; + return Integer.compare(a.getVirtualNodeId(), b.getVirtualNodeId()); + }); + } + + // ============================================================ + // Section 7 — Read path: load() + // ============================================================ + + /** + * Read the etcd pointer and load the body it references. Returns + * {@code null} when the pointer is absent or the body is unusable + * (caller falls back to fresh-deployment handling). Throws when the + * body's {@code virtualNodesPerNode} disagrees with the local config: + * recovery must fail closed rather than rebuild with a stale vnode + * mapping. + */ + public LoadedCheckpoint load() throws RetinaException + { + String bodyName = readPointer(); + if (bodyName == null || bodyName.isEmpty()) + { + return null; + } + byte[] bytes; + try + { + bytes = readBody(bodyName); + } + catch (IOException e) + { + logger.warn("Recovery loader: pointer references {} but read failed", bodyName, e); + return null; + } + Body body; + try + { + body = Body.readFrom(bytes); + } + catch (RetinaException e) + { + logger.warn("Recovery loader: body {} is corrupted/unreadable", bodyName, e); + return null; + } + // Fail-closed: configuration changed since last checkpoint. Abort + // recovery and let the operator intervene rather than rebuild with + // a stale vnode mapping. + if (body.getVirtualNodesPerNode() != virtualNodesPerNode) + { + throw new RetinaException(String.format( + "Recovery aborted: body %s was written with node.virtual.num=%d, current=%d. " + + "Configuration changed since last checkpoint; refusing to recover with stale vnode mapping.", + bodyName, body.getVirtualNodesPerNode(), virtualNodesPerNode)); + } + if (!isAcceptable(body, bodyName)) + { + return null; + } + return new LoadedCheckpoint(bodyName, body); + } + + private String readPointer() + { + KeyValue kv = etcd.getKeyValue(pointerKey); + if (kv == null) + { + return null; + } + String value = kv.getValue().toString(StandardCharsets.UTF_8); + return value.isEmpty() ? null : value; + } + + private byte[] readBody(String objectName) throws IOException + { + String path = checkpointDir + "/" + objectName; + long length = storage.getStatus(path).getLength(); + if (length <= 0) + { + throw new IOException("empty body file at " + path); + } + if (length > Integer.MAX_VALUE) + { + throw new IOException("body too large to read into memory: " + length + " bytes at " + path); + } + byte[] result = new byte[(int) length]; + try (DataInputStream in = storage.open(path)) + { + in.readFully(result); + } + return result; + } + + private boolean isAcceptable(Body body, String bodyName) + { + if (!retinaNodeId.equals(body.getRetinaNodeId())) + { + logger.warn("Recovery loader: body {} retinaNodeId='{}' does not match expected '{}'", + bodyName, body.getRetinaNodeId(), retinaNodeId); + return false; + } + if (body.getCheckpointAppliedTs() < 0) + { + logger.warn("Recovery loader: body {} has illegal checkpointAppliedTs={}", + bodyName, body.getCheckpointAppliedTs()); + return false; + } + return true; + } + + // ============================================================ + // Section 8 — Misc helpers + // ============================================================ + + /** FNV-1a 64-bit hash, used for {@code retinaNodeId}. */ + static long fnv1a64(String s) + { + long hash = 0xcbf29ce484222325L; + if (s == null) + { + return hash; + } + byte[] bytes = s.getBytes(StandardCharsets.UTF_8); + for (byte b : bytes) + { + hash ^= (b & 0xFFL); + hash *= 0x100000001b3L; + } + return hash; + } + + private static int readIntBE(byte[] arr, int off) + { + return ((arr[off] & 0xFF) << 24) + | ((arr[off + 1] & 0xFF) << 16) + | ((arr[off + 2] & 0xFF) << 8) + | (arr[off + 3] & 0xFF); + } + + private static List emptyIfNull(List values) + { + return values == null ? Collections.emptyList() : values; + } + + private static String trimTrailingSlash(String dir) + { + int len = dir.length(); + while (len > 0 && dir.charAt(len - 1) == '/') + { + len--; + } + return dir.substring(0, len); + } +} diff --git a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/RetinaResourceManager.java b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/RetinaResourceManager.java index f38fc4dd37..e3cd74c3bc 100644 --- a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/RetinaResourceManager.java +++ b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/RetinaResourceManager.java @@ -38,6 +38,8 @@ import io.pixelsdb.pixels.core.TypeDescription; import io.pixelsdb.pixels.core.encoding.EncodingLevel; import io.pixelsdb.pixels.index.IndexProto; +import io.pixelsdb.pixels.retina.RecoveryCheckpoint.PendingSegmentEntry; +import io.pixelsdb.pixels.retina.RecoveryCheckpoint.VisibilityEntry; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -64,23 +66,25 @@ public class RetinaResourceManager private final MetadataService metadataService; private final Map rgVisibilityMap; private final Map> pixelsWriteBufferMap; - private final IngestFileMetadataRegistry ingestFileMetadataRegistry; private String retinaHostName; // GC related fields private final ScheduledExecutorService gcExecutor; private final AtomicBoolean gcScheduled; private final StorageGarbageCollector storageGarbageCollector; + // Initialised by startBackgroundGc(); recovery checkpoint publication + // is part of every GC cycle once the scheduler is running. Null until + // then so unit/integration tests that never start the scheduler are + // unaffected. + private RecoveryCheckpoint recoveryCheckpoint; - // Checkpoint related fields - private final ExecutorService checkpointExecutor; - private final Map offloadedCheckpoints; - private final Map> checkpointFutures; - private final String checkpointDir; private volatile long latestGcTimestamp = -1; private final int totalVirtualNodeNum; - private final Map checkpointRefCounts; + // Offload checkpoint state (see "Offload Checkpoint Section" at the bottom of this file). + private final String offloadCheckpointDir; + private final ExecutorService offloadCheckpointExecutor; + private final Map offloadCheckpoints = new ConcurrentHashMap<>(); // Dual-write: oldFileId → result AND newFileId → result in a single map. // Direction is distinguished by checking fileId == result.newFileId. @@ -127,33 +131,14 @@ static final class RetiredFile } } - private enum CheckpointType - { - GC, - OFFLOAD - } - private RetinaResourceManager() { this.metadataService = MetadataService.Instance(); this.rgVisibilityMap = new ConcurrentHashMap<>(); this.pixelsWriteBufferMap = new ConcurrentHashMap<>(); - this.ingestFileMetadataRegistry = new IngestFileMetadataRegistry(); - this.offloadedCheckpoints = new ConcurrentHashMap<>(); - this.checkpointFutures = new ConcurrentHashMap<>(); ConfigFactory config = ConfigFactory.Instance(); - this.checkpointRefCounts = new ConcurrentHashMap<>(); - this.checkpointDir = config.getProperty("retina.checkpoint.dir"); - - int cpThreads = Integer.parseInt(config.getProperty("retina.checkpoint.threads")); - this.checkpointExecutor = Executors.newFixedThreadPool(cpThreads, r -> { - Thread t = new Thread(r, "retina-checkpoint-thread"); - t.setDaemon(true); - return t; - }); - this.gcExecutor = Executors.newSingleThreadScheduledExecutor(r -> { Thread t = new Thread(r, "retina-gc-thread"); t.setDaemon(true); @@ -163,6 +148,15 @@ private RetinaResourceManager() totalVirtualNodeNum = Integer.parseInt(ConfigFactory.Instance().getProperty("node.virtual.num")); this.retinaHostName = NetUtils.getLocalHostName(); + this.offloadCheckpointDir = config.getProperty("retina.offload.checkpoint.dir"); + this.offloadCheckpointExecutor = Executors.newFixedThreadPool( + Integer.parseInt(config.getProperty("retina.offload.checkpoint.threads")), + r -> { + Thread t = new Thread(r, "retina-checkpoint-thread"); + t.setDaemon(true); + return t; + }); + StorageGarbageCollector gc = null; try { @@ -238,6 +232,12 @@ public void startBackgroundGc() throws RetinaException return; } + // Fail-closed: recovery checkpoint is a durability primitive. If we + // cannot construct it (missing/unreadable config, unreachable etcd + // or storage backend), refuse to start the GC scheduler rather than + // silently run without crash recovery. + this.recoveryCheckpoint = RecoveryCheckpoint.createDefault(); + try { this.gcExecutor.scheduleAtFixedRate( @@ -339,286 +339,6 @@ public long[] queryVisibility(long fileId, int rgId, long timestamp) throws Reti return queryVisibility(fileId, rgId, timestamp, -1); } - /** - * Long-running queries register an "Offload" status and ensure that - * the required visibility checkpoint is correctly created and manages. - * For long-running transactions, newly written data is not required. - * Therefore, even if checkpoints are created under the same timestamp - * and only one copy is retained, this has virtually no impact on queries. - * - * @param timestamp - * @throws RetinaException - */ - public void registerOffload(long timestamp) throws RetinaException - { - AtomicInteger refCount = checkpointRefCounts.computeIfAbsent(timestamp, k -> new AtomicInteger(0)); - CompletableFuture future; - - synchronized (refCount) - { - refCount.incrementAndGet(); - - // If checkpoint already exists and is fully committed, just return - if (offloadedCheckpoints.containsKey(timestamp)) - { - logger.info("Registered offload for Timestamp: {} (already exists)", timestamp); - return; - } - - // Check if there is an existing future - future = checkpointFutures.get(timestamp); - if (future != null && future.isCompletedExceptionally()) - { - // If previous attempt failed, remove it so we can retry - checkpointFutures.remove(timestamp, future); - future = null; - } - - if (future == null) - { - future = checkpointFutures.computeIfAbsent(timestamp, k -> { - try - { - return createCheckpoint(timestamp, CheckpointType.OFFLOAD); - } catch (RetinaException e) - { - throw new CompletionException(e); - } - }); - } - } - - try - { - future.join(); - logger.info("Registered offload for Timestamp: {}", timestamp); - } catch (Exception e) - { - synchronized (refCount) - { - refCount.decrementAndGet(); - // We don't remove from checkpointFutures here anymore, - // because it's handled above in the synchronized block for retries - // or let the next caller handle it. - } - throw new RetinaException("Failed to create checkpoint for timestamp: " + timestamp, e); - } - } - - public void unregisterOffload(long timestamp) - { - AtomicInteger refCount = checkpointRefCounts.get(timestamp); - if (refCount != null) - { - synchronized (refCount) - { - int remaining = refCount.decrementAndGet(); - if (remaining <= 0) - { - offloadedCheckpoints.remove(timestamp); - checkpointFutures.remove(timestamp); - if (refCount.get() > 0) - { - logger.info("Checkpoint resurrection detected, skipping deletion. TS: {}", timestamp); - return; - } - removeCheckpointFile(timestamp, CheckpointType.OFFLOAD); - checkpointRefCounts.remove(timestamp); - logger.info("Offload checkpoint for timestamp {} removed.", timestamp); - } - } - } - } - - private CompletableFuture createCheckpoint(long timestamp, CheckpointType type) throws RetinaException - { - return createCheckpoint(timestamp, type, null); - } - - void registerIngestFileMetadata(long fileId, long tableId, int virtualNodeId, - long firstBlockId) throws RetinaException - { - this.ingestFileMetadataRegistry.register(fileId, tableId, virtualNodeId, firstBlockId); - } - - void unregisterIngestFileMetadata(long fileId) - { - this.ingestFileMetadataRegistry.unregister(fileId); - } - - IngestFileMetadataRegistry.Entry getIngestFileMetadata(long fileId) throws RetinaException - { - return this.ingestFileMetadataRegistry.get(fileId); - } - - List listIngestFileMetadataByStream(long tableId, int virtualNodeId) - { - return this.ingestFileMetadataRegistry.listByStream(tableId, virtualNodeId); - } - - void validateRgVisibilityFileRegistered(long fileId) throws RetinaException - { - if (!this.ingestFileMetadataRegistry.contains(fileId)) - { - throw new RetinaException("RGVisibilityIndex contains fileId=" + fileId - + " but registry has no entry, indicating publisher/retire ordering bug"); - } - } - - private CompletableFuture createCheckpoint( - long timestamp, CheckpointType type, Map precomputedBitmaps) throws RetinaException - { - String prefix = (type == CheckpointType.GC) ? RetinaUtils.CHECKPOINT_PREFIX_GC : RetinaUtils.CHECKPOINT_PREFIX_OFFLOAD; - String filePath = RetinaUtils.buildCheckpointPath(checkpointDir, prefix, retinaHostName, timestamp); - - // 1. Capture current entries to ensure we process a consistent set of RGs - List> entries = new ArrayList<>(this.rgVisibilityMap.entrySet()); - for (Map.Entry entry : entries) - { - long fileId = RetinaUtils.parseFileIdFromRgKey(entry.getKey()); - validateRgVisibilityFileRegistered(fileId); - } - int totalRgs = entries.size(); - logger.info("Starting {} checkpoint for {} RGs at timestamp {}", type, totalRgs, timestamp); - - // 2. Use a BlockingQueue for producer-consumer pattern - BlockingQueue queue = new LinkedBlockingQueue<>(1024); - - // 3. Start producer tasks to fetch bitmaps - for (Map.Entry entry : entries) - { - checkpointExecutor.submit(() -> { - try - { - String key = entry.getKey(); - long fileId = RetinaUtils.parseFileIdFromRgKey(key); - int rgId = RetinaUtils.parseRgIdFromRgKey(key); - RGVisibility rgVisibility = entry.getValue(); - long[] bitmap; - if (precomputedBitmaps != null && precomputedBitmaps.containsKey(key)) - { - bitmap = precomputedBitmaps.get(key); - } else - { - bitmap = rgVisibility.getVisibilityBitmap(timestamp); - } - queue.put(new CheckpointFileIO.CheckpointEntry(fileId, rgId, (int) rgVisibility.getRecordNum(), bitmap)); - } catch (Exception e) - { - logger.error("Failed to fetch visibility bitmap for checkpoint", e); - } - }); - } - - // 4. Async Write: perform IO in background thread (Consumer). - // Use commonPool to avoid deadlocks with checkpointExecutor. - // Concurrency safety: for OFFLOAD type, registerOffload() guarantees at most - // one future per timestamp via synchronized(refCount) + checkpointFutures.computeIfAbsent. - // For GC type, runGC() is single-threaded. No file-level locking is needed here. - return CompletableFuture.runAsync(() -> { - long startWrite = System.currentTimeMillis(); - try - { - CheckpointFileIO.writeCheckpoint(filePath, totalRgs, queue); - long endWrite = System.currentTimeMillis(); - logger.info("Writing {} checkpoint file to {} took {} ms", type, filePath, (endWrite - startWrite)); - - if (type == CheckpointType.OFFLOAD) - { - offloadedCheckpoints.put(timestamp, filePath); - } - } catch (Exception e) - { - logger.error("Failed to commit {} checkpoint file for timestamp: {}", type, timestamp, e); - try - { - StorageFactory.Instance().getStorage(filePath).delete(filePath, false); - } catch (IOException ignored) - { - } - throw new CompletionException(e); - } - }); - } - - /** - * Writes a checkpoint from pre-built {@link CheckpointFileIO.CheckpointEntry} objects, - * bypassing the {@code rgVisibilityMap} traversal and per-entry thread-pool submission - * that the other {@code createCheckpoint} overload performs. - * - *

    This is used by {@link #runGC()} when the entries have already been constructed - * during the Memory GC single-pass, avoiding a redundant second traversal of - * {@code rgVisibilityMap}. - */ - private CompletableFuture createCheckpointDirect( - long timestamp, CheckpointType type, - List preBuiltEntries) throws RetinaException - { - String prefix = (type == CheckpointType.GC) ? RetinaUtils.CHECKPOINT_PREFIX_GC : RetinaUtils.CHECKPOINT_PREFIX_OFFLOAD; - String filePath = RetinaUtils.buildCheckpointPath(checkpointDir, prefix, retinaHostName, timestamp); - - int totalRgs = preBuiltEntries.size(); - logger.info("Starting {} checkpoint (direct) for {} RGs at timestamp {}", type, totalRgs, timestamp); - - BlockingQueue queue = new LinkedBlockingQueue<>(1024); - - // Feed pre-built entries into the queue via the checkpoint executor so that the - // producer-consumer pattern with the writer thread is preserved (the queue has a - // bounded capacity of 1024, so this may block and must not run on the caller thread). - checkpointExecutor.submit(() -> { - try - { - for (CheckpointFileIO.CheckpointEntry entry : preBuiltEntries) - { - queue.put(entry); - } - } - catch (InterruptedException e) - { - Thread.currentThread().interrupt(); - logger.error("Interrupted while feeding pre-built checkpoint entries", e); - } - }); - - return CompletableFuture.runAsync(() -> { - try - { - CheckpointFileIO.writeCheckpoint(filePath, totalRgs, queue); - - if (type == CheckpointType.OFFLOAD) - { - offloadedCheckpoints.put(timestamp, filePath); - } - } - catch (Exception e) - { - logger.error("Failed to commit {} checkpoint file for timestamp: {}", type, timestamp, e); - try - { - StorageFactory.Instance().getStorage(filePath).delete(filePath, false); - } - catch (IOException ignored) - { - } - throw new CompletionException(e); - } - }); - } - - private void removeCheckpointFile(long timestamp, CheckpointType type) - { - String prefix = (type == CheckpointType.GC) ? RetinaUtils.CHECKPOINT_PREFIX_GC : RetinaUtils.CHECKPOINT_PREFIX_OFFLOAD; - String path = RetinaUtils.buildCheckpointPath(checkpointDir, prefix, retinaHostName, timestamp); - - try - { - StorageFactory.Instance().getStorage(path).delete(path, false); - } catch (IOException e) - { - logger.warn("Failed to delete checkpoint file", e); - } - } - public void reclaimVisibility(long fileId, int rgId, long timestamp) throws RetinaException { String retinaKey = RetinaUtils.buildRgKey(fileId, rgId); @@ -685,11 +405,6 @@ public void processRetiredFiles() }); } - public String getCheckpointPath(long timestamp) - { - return offloadedCheckpoints.get(timestamp); - } - public void deleteRecord(long fileId, int rgId, int rgRowOffset, long timestamp) throws RetinaException { deleteRecord(fileId, rgId, rgRowOffset, timestamp, RGVisibility.ReplayMode.NORMAL); @@ -1030,33 +745,29 @@ private PixelsWriteBuffer checkPixelsWriteBuffer(String schema, String table, in } /** - * Run a full GC cycle: Memory GC → checkpoint → Storage GC. + * Run a full GC cycle: Memory GC → Storage GC → Recovery Checkpoint. * *

    Ordering rationale: *

      *
    1. Memory GC first: {@code collectTileGarbage} compacts Deletion Chain blocks - * whose last item ts ≤ lwm into {@code baseBitmap}. After compaction, the remaining - * chain starts at the first block that straddles the lwm boundary, so the subsequent - * {@code getVisibilityBitmap(lwm)} call traverses at most one partial block - * (≤ {@code BLOCK_CAPACITY} items) instead of the entire pre-GC chain. This makes - * checkpoint bitmap serialisation significantly cheaper.
      2. - *
    2. Checkpoint second, unconditional and blocking: written regardless of whether - * Storage GC finds any candidate files. The {@code .join()} ensures the checkpoint - * file is fully on disk before Storage GC begins rewriting any files, so after a - * crash the post-Memory-GC visibility state can be rebuilt from the checkpoint - * independently of any in-progress Storage GC rewrite. {@code gcExecutor} is - * single-threaded, so the blocking join is also the simplest way to guarantee no - * two GC cycles overlap.
      3. - *
    3. Storage GC third: requires an up-to-date {@code baseBitmap} (hence after - * Memory GC) and its own WAL to resume in-progress tasks after a crash. Placing - * it after the checkpoint keeps the two restart paths independent: the GC checkpoint - * rebuilds the post-Memory-GC visibility state, and the GcWal resumes any - * in-progress Storage GC task separately. Once scan completes, bitmaps for - * non-candidate files are immediately released from memory (they are no longer - * needed by subsequent phases).
      4. - *
    4. Advance {@code latestGcTimestamp} last: updated only after the entire cycle - * succeeds (Memory GC + checkpoint + Storage GC). If any step throws, the timestamp - * is not advanced and the next scheduled invocation will retry the full cycle.
      5. + * whose last item ts ≤ the safe folding timestamp into {@code baseBitmap}. After compaction, + * the remaining chain starts at the first block that straddles that boundary, so the subsequent + * {@code getVisibilityBitmap(timestamp)} call traverses at most one partial block + * (≤ {@code BLOCK_CAPACITY} items) instead of the entire pre-GC chain. The same pass + * also captures one {@link VisibilityEntry} per RG by reusing the post-fold bitmap, + * so Recovery Checkpoint does not have to traverse RGVisibility a second time. + *
    5. Storage GC second: requires an up-to-date {@code baseBitmap} (hence after + * Memory GC) and its own WAL to resume in-progress tasks after a crash. Once scan + * completes, bitmaps for non-candidate files are immediately released from memory + * (they are no longer needed by subsequent phases).
      6. + *
    6. Recovery Checkpoint third: receives the {@code rgEntries} collected in + * Step 1 plus per-scope earliest pending commit timestamps, then publishes the + * body + etcd pointer. Unlike Storage GC, a publish failure here aborts the cycle: + * the outer catch skips the {@code latestGcTimestamp} advancement, and the next + * cycle retries the full sequence so crash recovery never silently lags.
      7. + *
    7. Advance {@code latestGcTimestamp} last: updated only after Memory GC and + * Recovery Checkpoint both succeed. Storage GC failures do not block advancement + * because compaction is opportunistic.
      8. *
    */ private void runGC() @@ -1066,10 +777,10 @@ private void runGC() long timestamp = 0; try { - timestamp = TransService.Instance().getSafeGcTimestamp(); + timestamp = TransService.Instance().getSafeVisibilityFoldingTimestamp(true); } catch (TransException e) { - logger.error("Error while getting safe garbage collection timestamp", e); + logger.error("Error while getting safe visibility folding timestamp", e); return; } @@ -1081,24 +792,24 @@ private void runGC() try { // Step 1: Single pass over rgVisibilityMap — Memory GC + file-level stats - // aggregation + CheckpointEntry pre-building. Produces everything needed by - // checkpoint and Storage GC without any additional traversal. + // aggregation + Recovery Checkpoint entries. Produces everything needed by + // Storage GC and Recovery Checkpoint without any additional traversal of + // rgVisibilityMap or extra native-side bitmap reads. Map gcSnapshotBitmaps = new HashMap<>(); Map fileStats = new HashMap<>(); // fileId → {totalRows, totalInvalid} - List checkpointEntries = new ArrayList<>(); + List rgEntries = new ArrayList<>(this.rgVisibilityMap.size()); for (Map.Entry entry : this.rgVisibilityMap.entrySet()) { String rgKey = entry.getKey(); long fileId = RetinaUtils.parseFileIdFromRgKey(rgKey); int rgId = RetinaUtils.parseRgIdFromRgKey(rgKey); + RGVisibility rgVisibility = entry.getValue(); - validateRgVisibilityFileRegistered(fileId); - - long[] bitmap = entry.getValue().garbageCollect(timestamp); + long[] bitmap = rgVisibility.garbageCollect(timestamp); gcSnapshotBitmaps.put(rgKey, bitmap); - long recordNum = entry.getValue().getRecordNum(); + long recordNum = rgVisibility.getRecordNum(); long rgInvalidCount = 0; for (long word : bitmap) { @@ -1116,15 +827,14 @@ private void runGC() return existing; }); - checkpointEntries.add( - new CheckpointFileIO.CheckpointEntry(fileId, rgId, (int) recordNum, bitmap)); + // Reuse the post-fold bitmap as the checkpoint entry's bitmap: it + // already reflects every delete with delete_ts <= timestamp folded + // into base, which is exactly what the loader needs to rebuild + // RGVisibility with an empty deletion chain. + rgEntries.add(new VisibilityEntry(fileId, rgId, (int) recordNum, timestamp, bitmap)); } - // Step 2: Checkpoint — write pre-built entries directly to disk, skipping - // the second rgVisibilityMap traversal and per-entry thread-pool submission. - createCheckpointDirect(timestamp, CheckpointType.GC, checkpointEntries).join(); - - // Step 3: Storage GC — pass file-level stats so that candidate selection + // Step 2: Storage GC — pass file-level stats so that candidate selection // uses O(1) lookups instead of per-RG aggregation loops. if (storageGarbageCollector != null) { @@ -1138,105 +848,282 @@ private void runGC() } } - // Step 4: Advance the timestamp only after the full cycle succeeds. - // latestGcTimestamp is no longer updated inside createCheckpoint's async - // callback for GC type; this is the single authoritative update point. - long oldGcTs = this.latestGcTimestamp; - this.latestGcTimestamp = timestamp; - if (oldGcTs != -1 && oldGcTs != timestamp) + // Step 3: Publish a recovery checkpoint at the same timestamp the + // Memory GC just folded against, reusing the rgEntries already + // collected in Step 1. Unlike Storage GC failures (which we swallow + // because compaction is opportunistic), checkpoint publication + // failures must propagate: the outer catch will skip the + // latestGcTimestamp advancement so the next cycle retries. + if (recoveryCheckpoint != null) { - removeCheckpointFile(oldGcTs, CheckpointType.GC); + // Project per-scope earliest pending commit ts. Buffers with + // ts == Long.MAX_VALUE have no committed pending data and are + // omitted: the scope contributes nothing to recovery replay. + List segments = new ArrayList<>(); + for (Map perTable : this.pixelsWriteBufferMap.values()) + { + for (PixelsWriteBuffer buffer : perTable.values()) + { + long ts = buffer.getEarliestPendingMinTs(); + if (ts != Long.MAX_VALUE) + { + segments.add(new PendingSegmentEntry(buffer.getTableId(), + buffer.getVirtualNodeId(), ts)); + } + } + } + recoveryCheckpoint.generate(timestamp, rgEntries, segments); } + + // Step 4: Advance the timestamp only after the full cycle succeeds. + this.latestGcTimestamp = timestamp; } catch (Exception e) { logger.error("Error while running GC", e); } } - public void recoverCheckpoints() + // ───────────────────────────────────────────────────────────────────── + // Offload Checkpoint Section + // + // Long-running queries register an "offload" status with a logical + // timestamp; this section materialises one visibility checkpoint file per + // registered timestamp and reference-counts concurrent registrations so + // that the file is created exactly once and deleted only after the last + // unregistration. + // + // State lives in three RRM fields declared at the top of the class: + // offloadCheckpointDir, offloadCheckpointExecutor, offloadCheckpoints. + // ───────────────────────────────────────────────────────────────────── + + /** + * Per-timestamp state aggregating reference count, in-flight creation + * future, and the resulting file path. Doubles as the synchronization + * monitor for all transitions on this timestamp's lifecycle. + */ + private static final class OffloadCheckpoint { - try + final AtomicInteger refCount = new AtomicInteger(0); + /** Set once createOffloadCheckpoint successfully commits the file; null otherwise. */ + volatile String filePath; + /** Tracks the in-flight creation task; cleared lazily on retry after failure. */ + volatile CompletableFuture future; + } + + /** + * Long-running queries register an "Offload" status to ensure that the + * required visibility checkpoint is created. Concurrent registrations of + * the same timestamp are reference-counted and share a single checkpoint + * file, which has virtually no impact on queries since long-running + * transactions do not need newly written data. + */ + public void registerOffload(long timestamp) throws RetinaException + { + OffloadCheckpoint cp = offloadCheckpoints.computeIfAbsent(timestamp, k -> new OffloadCheckpoint()); + CompletableFuture future; + + synchronized (cp) { - Storage storage = StorageFactory.Instance().getStorage(checkpointDir); - if (!storage.exists(checkpointDir)) + cp.refCount.incrementAndGet(); + + if (cp.filePath != null) { - storage.mkdirs(checkpointDir); + logger.info("Registered offload for Timestamp: {} (already exists)", timestamp); return; } - List allFiles = storage.listPaths(checkpointDir); - // filter only .bin files - allFiles = allFiles.stream().filter(p -> p.endsWith(".bin")).collect(Collectors.toList()); + future = cp.future; + if (future != null && future.isCompletedExceptionally()) + { + // Previous attempt failed; drop the stale future so this caller retries. + cp.future = null; + future = null; + } - List gcTimestamps = new ArrayList<>(); - String offloadPrefix = RetinaUtils.getCheckpointPrefix(RetinaUtils.CHECKPOINT_PREFIX_OFFLOAD, retinaHostName); - String gcPrefix = RetinaUtils.getCheckpointPrefix(RetinaUtils.CHECKPOINT_PREFIX_GC, retinaHostName); + if (future == null) + { + future = createOffloadCheckpoint(timestamp, cp); + cp.future = future; + } + } - for (String path : allFiles) + try + { + future.join(); + logger.info("Registered offload for Timestamp: {}", timestamp); + } + catch (Exception e) + { + synchronized (cp) { - // use Paths.get().getFileName() to extract filename from path string - String filename = Paths.get(path).getFileName().toString(); - if (filename.startsWith(offloadPrefix)) - { - // delete offload checkpoint files when restarting - try - { - storage.delete(path, false); - } catch (IOException e) - { - logger.error("Failed to delete checkpoint file {}", path, e); - } - } else if (filename.startsWith(gcPrefix)) - { - try - { - gcTimestamps.add(Long.parseLong(filename.replace(gcPrefix, "").replace(".bin", ""))); - } catch (Exception e) - { - logger.error("Failed to parse checkpoint timestamp from file {}", path, e); - } - } + cp.refCount.decrementAndGet(); } + throw new RetinaException("Failed to create checkpoint for timestamp: " + timestamp, e); + } + } - if (gcTimestamps.isEmpty()) + public void unregisterOffload(long timestamp) + { + OffloadCheckpoint cp = offloadCheckpoints.get(timestamp); + if (cp == null) + { + return; + } + synchronized (cp) + { + if (cp.refCount.decrementAndGet() > 0) { return; } + offloadCheckpoints.remove(timestamp); + deleteOffloadCheckpoint(timestamp); + logger.info("Offload checkpoint for timestamp {} removed.", timestamp); + } + } - Collections.sort(gcTimestamps); - long latestTs = gcTimestamps.get(gcTimestamps.size() - 1); - this.latestGcTimestamp = latestTs; - logger.info("Loading system state from GC checkpoint: {}", latestTs); + public String getOffloadCheckpointPath(long timestamp) + { + OffloadCheckpoint cp = offloadCheckpoints.get(timestamp); + return cp == null ? null : cp.filePath; + } - // load to rgVisibilityMap - String latestPath = RetinaUtils.buildCheckpointPath( - checkpointDir, RetinaUtils.CHECKPOINT_PREFIX_GC, retinaHostName, latestTs); + /** + * Cleans up stale offload checkpoint files left over by previous runs of + * this node before the service opens for queries. Long-running queries + * that owned those checkpoints are no longer active after a restart, so + * the files are safe to drop. + * + *

    Cross-restart visibility recovery is the responsibility of the + * recovery checkpoint flow (see {@code recovery.md}); this method does + * not rebuild {@code rgVisibilityMap}. + */ + public void recoverOffloadCheckpoints() + { + try + { + Storage storage = StorageFactory.Instance().getStorage(offloadCheckpointDir); + if (!storage.exists(offloadCheckpointDir)) + { + storage.mkdirs(offloadCheckpointDir); + return; + } - try + String offloadPrefix = RetinaUtils.getCheckpointPrefix( + RetinaUtils.CHECKPOINT_PREFIX_OFFLOAD, retinaHostName); + for (String path : storage.listPaths(offloadCheckpointDir)) { - Storage latestStorage = StorageFactory.Instance().getStorage(latestPath); - if (latestStorage.exists(latestPath)) + if (!path.endsWith(".bin")) { - final long ts = latestTs; - int rgCount = CheckpointFileIO.readCheckpointParallel(latestPath, entry -> { - addVisibility(entry.fileId, entry.rgId, entry.recordNum, ts, entry.bitmap, true); - }, checkpointExecutor); - - logger.info("Recovered {} RG entries from GC checkpoint", rgCount); + continue; + } + String filename = Paths.get(path).getFileName().toString(); + if (!filename.startsWith(offloadPrefix)) + { + continue; + } + try + { + storage.delete(path, false); + } + catch (IOException e) + { + logger.error("Failed to delete stale offload checkpoint file {}", path, e); } - } catch (IOException e) - { - logger.error("Failed to read checkpoint file", e); } + } + catch (IOException e) + { + logger.error("Failed to recover offload checkpoints", e); + } + } + + /** + * Two-phase checkpoint creation: + *

      + *
    1. Fold each RG's deletion chain at {@code timestamp} in parallel. + * A failure in any fold task surfaces through the returned future + * (no swallowed errors, no waiting on the writer's 60s timeout).
      2. + *
    2. Once all bitmaps are ready, drain them into the queue and write + * the file. On any failure the partial file is removed via the + * {@code whenComplete} side effect.
      3. + *
    + * + *

    Concurrency safety: {@link #registerOffload} guarantees at most one + * in-flight future per OffloadCheckpoint via {@code synchronized(cp)} + + * single-writer of {@code cp.future}, so no file-level locking is needed. + */ + private CompletableFuture createOffloadCheckpoint(long timestamp, OffloadCheckpoint cp) + { + String filePath = RetinaUtils.buildCheckpointPath( + offloadCheckpointDir, RetinaUtils.CHECKPOINT_PREFIX_OFFLOAD, retinaHostName, timestamp); + + List> entries = new ArrayList<>(rgVisibilityMap.entrySet()); + int totalRgs = entries.size(); + logger.info("Starting offload checkpoint for {} RGs at timestamp {}", totalRgs, timestamp); + + List> bitmapFutures = new ArrayList<>(totalRgs); + for (Map.Entry entry : entries) + { + bitmapFutures.add(CompletableFuture.supplyAsync(() -> { + String key = entry.getKey(); + long fileId = RetinaUtils.parseFileIdFromRgKey(key); + int rgId = RetinaUtils.parseRgIdFromRgKey(key); + RGVisibility rgVisibility = entry.getValue(); + long[] bitmap = rgVisibility.getVisibilityBitmap(timestamp); + return new CheckpointFileIO.CheckpointEntry( + fileId, rgId, (int) rgVisibility.getRecordNum(), bitmap); + }, offloadCheckpointExecutor)); + } - // delete old GC checkpoint files - for (int i = 0; i < gcTimestamps.size() - 1; i++) + return CompletableFuture + .allOf(bitmapFutures.toArray(new CompletableFuture[0])) + .thenRunAsync(() -> { + long startWrite = System.currentTimeMillis(); + BlockingQueue queue = + new ArrayBlockingQueue<>(Math.max(1, totalRgs)); + try + { + for (CompletableFuture f : bitmapFutures) + { + queue.put(f.join()); + } + CheckpointFileIO.writeCheckpoint(filePath, totalRgs, queue); + long endWrite = System.currentTimeMillis(); + logger.info("Writing offload checkpoint file to {} took {} ms", + filePath, (endWrite - startWrite)); + cp.filePath = filePath; + } + catch (Exception e) + { + throw new CompletionException(e); + } + }, offloadCheckpointExecutor) + .whenComplete((unused, throwable) -> { + if (throwable != null) + { + logger.error("Failed to create offload checkpoint for timestamp: {}", + timestamp, throwable); + deleteOffloadCheckpoint(timestamp); + } + }); + } + + private void deleteOffloadCheckpoint(long timestamp) + { + String path = RetinaUtils.buildCheckpointPath( + offloadCheckpointDir, RetinaUtils.CHECKPOINT_PREFIX_OFFLOAD, retinaHostName, timestamp); + + try + { + Storage storage = StorageFactory.Instance().getStorage(path); + if (storage.exists(path)) { - removeCheckpointFile(gcTimestamps.get(i), CheckpointType.GC); + storage.delete(path, false); } - } catch (IOException e) + } + catch (IOException e) { - logger.error("Failed to recover checkpoints", e); + logger.warn("Failed to delete offload checkpoint file {}", path, e); } } } diff --git a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/StorageGarbageCollector.java b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/StorageGarbageCollector.java index bd973cf7a4..d72ef5aaa9 100644 --- a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/StorageGarbageCollector.java +++ b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/StorageGarbageCollector.java @@ -1203,7 +1203,6 @@ void commitFileGroup(RewriteResult result) throws Exception for (FileCandidate fc : result.group.files) { - resourceManager.unregisterIngestFileMetadata(fc.fileId); resourceManager.scheduleRetiredFile( new RetinaResourceManager.RetiredFile( fc.fileId, fc.rgCount, fc.filePath, retireDeadline, result.oldRowIds)); diff --git a/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestIngestFileMetadataRegistry.java b/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestIngestFileMetadataRegistry.java deleted file mode 100644 index ea3f55a9e3..0000000000 --- a/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestIngestFileMetadataRegistry.java +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Copyright 2026 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ -package io.pixelsdb.pixels.retina; - -import io.pixelsdb.pixels.common.exception.RetinaException; -import org.junit.Test; - -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -public class TestIngestFileMetadataRegistry -{ - @Test - public void tracksMetadataByFileIdAndStream() throws Exception - { - IngestFileMetadataRegistry registry = new IngestFileMetadataRegistry(); - - registry.register(100L, 7L, 3, 0L); - registry.register(100L, 7L, 3, 0L); - registry.register(200L, 7L, 3, 10L); - - IngestFileMetadataRegistry.Entry entry = registry.get(100L); - assertEquals(100L, entry.getFileId()); - assertEquals(7L, entry.getTableId()); - assertEquals(3, entry.getVirtualNodeId()); - assertEquals(0L, entry.getFirstBlockId()); - - List streamEntries = registry.listByStream(7L, 3); - assertEquals(2, streamEntries.size()); - assertEquals(100L, streamEntries.get(0).getFileId()); - assertEquals(200L, streamEntries.get(1).getFileId()); - } - - @Test - public void rejectsConflictsAndUnregisters() throws Exception - { - IngestFileMetadataRegistry registry = new IngestFileMetadataRegistry(); - registry.register(100L, 7L, 3, 0L); - - try - { - registry.register(100L, 7L, 3, 1L); - fail("Expected conflicting registration to fail"); - } catch (RetinaException expected) - { - assertTrue(expected.getMessage().contains("Conflicting")); - } - - registry.unregister(100L); - assertTrue(registry.listByStream(7L, 3).isEmpty()); - assertFalse(registry.contains(100L)); - - try - { - registry.get(100L); - fail("Expected unregistered file metadata lookup to fail"); - } catch (RetinaException expected) - { - assertTrue(expected.getMessage().contains("Missing ingest metadata")); - } - } - - @Test - public void unregisterRemovesOnlyMatchingStreamEntry() throws Exception - { - IngestFileMetadataRegistry registry = new IngestFileMetadataRegistry(); - - registry.register(100L, 7L, 3, 0L); - registry.register(200L, 7L, 3, 10L); - registry.register(300L, 7L, 4, 0L); - - registry.unregister(100L); - registry.unregister(999L); - - List streamEntries = registry.listByStream(7L, 3); - assertEquals(1, streamEntries.size()); - assertEquals(200L, streamEntries.get(0).getFileId()); - assertEquals(1, registry.listByStream(7L, 4).size()); - } - - @Test - public void rejectsOutOfOrderRegistrationWithinStream() throws Exception - { - IngestFileMetadataRegistry registry = new IngestFileMetadataRegistry(); - registry.register(200L, 7L, 3, 10L); - - try - { - registry.register(100L, 7L, 3, 0L); - fail("Expected out-of-order registration to fail"); - } catch (RetinaException expected) - { - assertTrue(expected.getMessage().contains("Out-of-order")); - } - - try - { - registry.register(300L, 7L, 3, 10L); - fail("Expected non-strictly-increasing firstBlockId to fail"); - } catch (RetinaException expected) - { - assertTrue(expected.getMessage().contains("Out-of-order")); - } - - registry.register(300L, 7L, 4, 0L); - assertEquals(1, registry.listByStream(7L, 4).size()); - } -} diff --git a/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestPixelsWriteBuffer.java b/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestPixelsWriteBuffer.java index a7b392df8e..4eb9a0dd08 100644 --- a/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestPixelsWriteBuffer.java +++ b/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestPixelsWriteBuffer.java @@ -19,7 +19,6 @@ */ package io.pixelsdb.pixels.retina; -import io.pixelsdb.pixels.common.exception.RetinaException; import io.pixelsdb.pixels.common.metadata.domain.Path; import io.pixelsdb.pixels.core.TypeDescription; import io.pixelsdb.pixels.core.vector.VectorizedRowBatch; @@ -37,8 +36,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; public class TestPixelsWriteBuffer { @@ -131,24 +128,6 @@ public void testConcurrentWriteOperations() } } - @Test - public void rgVisibilityRegistryValidationFailsClosed() throws Exception - { - RetinaResourceManager resourceManager = RetinaResourceManager.Instance(); - - try - { - resourceManager.validateRgVisibilityFileRegistered(500L); - fail("Expected missing RGVisibility registry entry to fail closed"); - } catch (RetinaException expected) - { - assertTrue(expected.getMessage().contains("RGVisibilityIndex contains fileId=500")); - } - - resourceManager.registerIngestFileMetadata(500L, 7L, 3, 0L); - resourceManager.validateRgVisibilityFileRegistered(500L); - } - @Test public void appendedRowsAreImmediatelyVisibleAndAdvanceCommitTsBounds() throws Exception { diff --git a/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestRetinaCheckpoint.java b/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestRetinaCheckpoint.java index 15ba28ce14..87e6adec15 100644 --- a/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestRetinaCheckpoint.java +++ b/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestRetinaCheckpoint.java @@ -22,33 +22,23 @@ import io.pixelsdb.pixels.common.exception.RetinaException; import io.pixelsdb.pixels.common.physical.Storage; import io.pixelsdb.pixels.common.physical.StorageFactory; -import io.pixelsdb.pixels.common.utils.CheckpointFileIO; import io.pixelsdb.pixels.common.utils.ConfigFactory; import io.pixelsdb.pixels.common.utils.RetinaUtils; import org.junit.Before; import org.junit.Test; import java.io.DataInputStream; -import java.io.DataOutputStream; import java.io.IOException; import java.lang.reflect.Field; -import java.lang.reflect.Method; import java.net.InetAddress; -import java.util.Arrays; -import java.util.HashMap; import java.util.Map; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.ThreadLocalRandom; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; /** @@ -67,7 +57,7 @@ public class TestRetinaCheckpoint @Before public void setUp() throws IOException, RetinaException { - testCheckpointDir = ConfigFactory.Instance().getProperty("retina.checkpoint.dir"); + testCheckpointDir = ConfigFactory.Instance().getProperty("retina.offload.checkpoint.dir"); storage = StorageFactory.Instance().getStorage(testCheckpointDir); if (!storage.exists(testCheckpointDir)) @@ -104,10 +94,6 @@ private String getOffloadFileName(long timestamp) { return RetinaUtils.getCheckpointFileName(RetinaUtils.CHECKPOINT_PREFIX_OFFLOAD, hostName, timestamp); } - private String getGcFileName(long timestamp) { - return RetinaUtils.getCheckpointFileName(RetinaUtils.CHECKPOINT_PREFIX_GC, hostName, timestamp); - } - @Test public void testRegisterOffload() throws RetinaException, IOException { @@ -162,69 +148,6 @@ public void testMultipleOffloads() throws RetinaException, IOException System.out.println("Verified: Checkpoint removed after final unregister. testMultipleOffloads passed."); } - @Test - public void testCheckpointRecovery() throws RetinaException, IOException - { - System.out.println("\n[Test] Starting testCheckpointRecovery..."); - retinaManager.addVisibility(fileId, rgId, numRows, 0L, null, false); - long timestamp = 100L; - - // 1. Delete row 10 - int rowToDelete = 10; - System.out.println("Deleting row " + rowToDelete + " in memory..."); - retinaManager.deleteRecord(fileId, rgId, rowToDelete, timestamp); - - // Verify deleted in memory - long[] memBitmap = retinaManager.queryVisibility(fileId, rgId, timestamp); - assertTrue("Row 10 should be deleted in memory", isBitSet(memBitmap, rowToDelete)); - - // 2. Register Offload to generate checkpoint file - System.out.println("Creating checkpoint on disk..."); - retinaManager.registerOffload(timestamp); - String offloadPath = resolve(testCheckpointDir, getOffloadFileName(timestamp)); - assertTrue("Checkpoint file should exist", storage.exists(offloadPath)); - - // 3. Rename offload file to GC file to simulate checkpoint generated by GC - String gcPath = resolve(testCheckpointDir, getGcFileName(timestamp)); - System.out.println("Simulating GC checkpoint by renaming offload file to: " + gcPath); - // Storage interface doesn't have rename, using copy and delete - try (DataInputStream in = storage.open(offloadPath); - DataOutputStream out = storage.create(gcPath, true, 4096)) - { - byte[] buffer = new byte[4096]; - int bytesRead; - while ((bytesRead = in.read(buffer)) != -1) - { - out.write(buffer, 0, bytesRead); - } - } - storage.delete(offloadPath, false); - - // 4. Reset singleton state (Simulate Crash/Restart) - System.out.println("Simulating system restart (resetting memory state)..."); - resetSingletonState(); - - // 5. Perform recovery - System.out.println("Running recoverCheckpoints()..."); - // At this point rgVisibilityMap is empty, recoverCheckpoints will load data directly into rgVisibilityMap - retinaManager.recoverCheckpoints(); - - // 6. Verify recovered state immediately after recovery - System.out.println("Verifying recovered state immediately after recoverCheckpoints()..."); - long[] recoveredBitmap = retinaManager.queryVisibility(fileId, rgId, timestamp); - assertTrue("Row 10 should be deleted after recovery", isBitSet(recoveredBitmap, rowToDelete)); - assertFalse("Row 11 should not be deleted", isBitSet(recoveredBitmap, rowToDelete + 1)); - - // 7. Re-add Visibility, at this point it should see that it already exists in rgVisibilityMap - System.out.println("Re-adding visibility for file (should skip as it already exists)..."); - retinaManager.addVisibility(fileId, rgId, numRows, 0L, null, false); - - // 8. Verify state still correct - long[] finalBitmap = retinaManager.queryVisibility(fileId, rgId, timestamp); - assertTrue("Row 10 should still be deleted", isBitSet(finalBitmap, rowToDelete)); - System.out.println("Verified: Recovery successful, row state restored directly to map. testCheckpointRecovery passed."); - } - @Test public void testCheckpointRetryAfterFailure() throws RetinaException, IOException { @@ -260,51 +183,6 @@ public void testCheckpointRetryAfterFailure() throws RetinaException, IOExceptio System.out.println("Verified: Retry successful. testCheckpointRetryAfterFailure passed."); } - @Test - public void testMultiRGCheckpoint() throws RetinaException, IOException - { - System.out.println("\n[Test] Starting testMultiRGCheckpoint..."); - int numRgs = 3; - for (int i = 0; i < numRgs; i++) - { - retinaManager.addVisibility(fileId, i, numRows, 0L, null, false); - } - long timestamp = 200L; - - // Delete records in different RGs - retinaManager.deleteRecord(fileId, 0, 10, timestamp); - retinaManager.deleteRecord(fileId, 1, 20, timestamp); - retinaManager.deleteRecord(fileId, 2, 30, timestamp); - - // Create checkpoint - retinaManager.registerOffload(timestamp); - String offloadPath = resolve(testCheckpointDir, getOffloadFileName(timestamp)); - - // Simulating GC checkpoint for recovery - String gcPath = resolve(testCheckpointDir, getGcFileName(timestamp)); - try (DataInputStream in = storage.open(offloadPath); - DataOutputStream out = storage.create(gcPath, true, 4096)) - { - byte[] buffer = new byte[4096]; - int bytesRead; - while ((bytesRead = in.read(buffer)) != -1) - { - out.write(buffer, 0, bytesRead); - } - } - - // Reset and recover - resetSingletonState(); - retinaManager.recoverCheckpoints(); - - // Verify all RGs - assertTrue("RG 0 row 10 should be deleted", isBitSet(retinaManager.queryVisibility(fileId, 0, timestamp), 10)); - assertTrue("RG 1 row 20 should be deleted", isBitSet(retinaManager.queryVisibility(fileId, 1, timestamp), 20)); - assertTrue("RG 2 row 30 should be deleted", isBitSet(retinaManager.queryVisibility(fileId, 2, timestamp), 30)); - - System.out.println("Verified: Multi-RG state correctly restored. testMultiRGCheckpoint passed."); - } - @Test public void testCheckpointDataIntegrity() throws RetinaException, IOException { @@ -393,132 +271,6 @@ else if (j % 3 == 1) assertFalse("Errors occurred during concurrency test", errorOccurred.get()); } - @Test - public void testCheckpointPerformance() throws RetinaException, IOException, InterruptedException - { - // 1. Performance Test Configuration - double targetDeleteRatio = 0.0; // @TARGET_DELETE_RATIO@ - int numFiles = 50000; - int rowsPerRg = 200000; - long totalRows = (long) numFiles * rowsPerRg; - long timestamp = System.currentTimeMillis(); - - System.out.printf("Target Delete Ratio: %.2f%%%n", targetDeleteRatio * 100); - System.out.printf("Total Rows: %,d%n", totalRows); - - // 2. Populate Visibility Data - System.out.println("[Perf] Populating visibility data..."); - for (int i = 0; i < numFiles; i++) - { - retinaManager.addVisibility(i, 0, rowsPerRg, 0L, null, false); - } - - // 3. Delete Records based on Ratio - System.out.println("[Perf] Deleting records..."); - long totalDeleted = 0; - if (targetDeleteRatio > 0) - { - // Delete contiguous block for performance stability - int rowsToDeletePerRg = (int) (rowsPerRg * targetDeleteRatio); - for (int i = 0; i < numFiles; i++) - { - // Delete rows 0 to rowsToDeletePerRg - 1 - for (int j = 0; j < rowsToDeletePerRg; j++) - { - retinaManager.deleteRecord(i, 0, j, timestamp); - } - totalDeleted += rowsToDeletePerRg; - } - } - double actualRatio = (double) totalDeleted / totalRows; - System.out.printf("Actual Ratio: %.2f%%%n", actualRatio * 100); - - // Measure Memory before Offload - System.gc(); - Thread.sleep(1000); - long memBeforeOffload = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(); - - // 4. Register Offload (Checkpoint Creation) - System.out.println("[Perf] Starting Offload..."); - long startOffload = System.nanoTime(); - retinaManager.registerOffload(timestamp); - long endOffload = System.nanoTime(); - double offloadTimeMs = (endOffload - startOffload) / 1_000_000.0; - System.out.printf("Total Offload Time: %.2f ms%n", offloadTimeMs); - - // Measure Peak Memory (Approximation: Current - Before) - long memAfterOffload = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(); - double peakMemMb = Math.max(0, (memAfterOffload - memBeforeOffload) / (1024.0 * 1024.0)); - System.out.printf("Offload Peak Mem Overhead: %.2f MB%n", peakMemMb); - - // File Size - String checkpointPath = resolve(testCheckpointDir, getOffloadFileName(timestamp)); - long fileSizeBytes = storage.getStatus(checkpointPath).getLength(); - double fileSizeMb = fileSizeBytes / (1024.0 * 1024.0); - System.out.printf("Checkpoint File Size: %.2f MB%n", fileSizeMb); - - // Write Throughput - double writeThroughput = fileSizeMb / (offloadTimeMs / 1000.0); - System.out.printf("Write Throughput: %.2f MB/s%n", writeThroughput); - - // 5. Simulate System Restart (Cold Load) - System.out.println("[Perf] Simulating restart..."); - // Rename to GC file to simulate persisted state - String gcPath = resolve(testCheckpointDir, getGcFileName(timestamp)); - // Simple copy since no rename - try (DataInputStream in = storage.open(checkpointPath); - DataOutputStream out = storage.create(gcPath, true, 8 * 1024 * 1024)) - { - byte[] buffer = new byte[64 * 1024]; // 64KB copy buffer - int bytesRead; - while ((bytesRead = in.read(buffer)) != -1) - { - out.write(buffer, 0, bytesRead); - } - } - storage.delete(checkpointPath, false); - - resetSingletonState(); - System.gc(); - Thread.sleep(1000); - long memBeforeLoad = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(); - - // Recover - long startLoad = System.nanoTime(); - retinaManager.recoverCheckpoints(); - long endLoad = System.nanoTime(); - double loadTimeMs = (endLoad - startLoad) / 1_000_000.0; - System.out.printf("First Load Time (Cold): %.2f ms%n", loadTimeMs); - - // Load Memory Overhead - long memAfterLoad = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(); - double loadMemMb = Math.max(0, (memAfterLoad - memBeforeLoad) / (1024.0 * 1024.0)); - System.out.printf("Load Memory Overhead: %.2f MB%n", loadMemMb); - - // Read Throughput - double readThroughput = fileSizeMb / (loadTimeMs / 1000.0); - System.out.printf("Read/Parse Throughput: %.2f MB/s%n", readThroughput); - - // 6. Avg Memory Hit Latency - System.out.println("[Perf] Measuring Memory Hit Latency..."); - long totalLatencyNs = 0; - int latencySamples = 10000; - for (int i = 0; i < latencySamples; i++) - { - // Random file query - long randomFileId = ThreadLocalRandom.current().nextInt(numFiles); - long startQuery = System.nanoTime(); - retinaManager.queryVisibility(randomFileId, 0, timestamp); - long endQuery = System.nanoTime(); - totalLatencyNs += (endQuery - startQuery); - } - double avgLatencyMs = (totalLatencyNs / (double) latencySamples) / 1_000_000.0; - System.out.printf("Avg Memory Hit Latency: %.4f ms%n", avgLatencyMs); - - // Cleanup - storage.delete(gcPath, false); - } - /** * Use reflection to reset internal state of RetinaResourceManager, simulating a restart. */ @@ -534,13 +286,9 @@ private void resetSingletonState() bufferMapField.setAccessible(true); ((Map) bufferMapField.get(retinaManager)).clear(); - Field offloadedField = RetinaResourceManager.class.getDeclaredField("offloadedCheckpoints"); - offloadedField.setAccessible(true); - ((Map) offloadedField.get(retinaManager)).clear(); - - Field refCountsField = RetinaResourceManager.class.getDeclaredField("checkpointRefCounts"); - refCountsField.setAccessible(true); - ((Map) refCountsField.get(retinaManager)).clear(); + Field offloadCheckpointsField = RetinaResourceManager.class.getDeclaredField("offloadCheckpoints"); + offloadCheckpointsField.setAccessible(true); + ((Map) offloadCheckpointsField.get(retinaManager)).clear(); Field gcTimestampField = RetinaResourceManager.class.getDeclaredField("latestGcTimestamp"); gcTimestampField.setAccessible(true); @@ -564,166 +312,4 @@ private boolean isBitSet(long[] bitmap, int rowIndex) return (bitmap[longIndex] & (1L << bitOffset)) != 0; } - // ----------------------------------------------------------------------- - // GC checkpoint: completeness + bitmap correctness - // ----------------------------------------------------------------------- - - /** - * Creates a {@code long[]} GC snapshot bitmap for one RG where exactly {@code deletedRows} - * out of {@code totalRows} rows are marked deleted (rows 0..deletedRows-1 are set). - */ - private static long[] makeBitmap(int totalRows, int deletedRows) - { - int words = (totalRows + 63) / 64; - long[] bitmap = new long[words]; - for (int r = 0; r < deletedRows; r++) - { - bitmap[r / 64] |= (1L << (r % 64)); - } - return bitmap; - } - - /** - * Calls {@code RetinaResourceManager.createCheckpoint(ts, CheckpointType.GC, bitmaps)} - * via reflection and blocks until the write completes. - */ - @SuppressWarnings("unchecked") - private void invokeCreateGCCheckpoint(long ts, Map bitmaps) throws Exception - { - // Locate the private CheckpointType enum class - Class cpTypeClass = Arrays.stream(RetinaResourceManager.class.getDeclaredClasses()) - .filter(c -> c.getSimpleName().equals("CheckpointType")) - .findFirst() - .orElseThrow(() -> new RuntimeException("CheckpointType enum not found")); - - // Get the GC constant - Object gcConstant = Arrays.stream(cpTypeClass.getEnumConstants()) - .filter(e -> e.toString().equals("GC")) - .findFirst() - .orElseThrow(() -> new RuntimeException("CheckpointType.GC not found")); - - // Get the overloaded createCheckpoint(long, CheckpointType, Map) method - Method method = RetinaResourceManager.class.getDeclaredMethod( - "createCheckpoint", long.class, cpTypeClass, Map.class); - method.setAccessible(true); - - CompletableFuture future = (CompletableFuture) method.invoke( - retinaManager, ts, gcConstant, bitmaps); - future.join(); - } - - /** - * Verifies that a GC checkpoint written with a full {@code gcSnapshotBitmaps} map - * contains ALL RG entries — including those that would not be selected as Storage GC - * candidates — because the checkpoint is written before S1 scanning begins. - * - *

    Setup: 3 files in {@code rgVisibilityMap}: - *

      - *
    • File A: 80 % deleted (would be a candidate)
      • - *
    • File B: 60 % deleted (would be a candidate)
      • - *
    • File C: 20 % deleted (non-candidate)
      • - *
    - * - *

    Expected: checkpoint rgCount = 3; all three entries present with correct - * {@code recordNum} and bitmap content. - */ - @Test - public void testGCCheckpoint_containsAllRGs() throws Exception - { - final long fileIdA = 77001L; - final long fileIdB = 77002L; - final long fileIdC = 77003L; - final int rows = 100; - final long safeGcTs = 500L; - - retinaManager.addVisibility(fileIdA, 0, rows, 0L, null, false); - retinaManager.addVisibility(fileIdB, 0, rows, 0L, null, false); - retinaManager.addVisibility(fileIdC, 0, rows, 0L, null, false); - - long[] bitmapA = makeBitmap(rows, 80); - long[] bitmapB = makeBitmap(rows, 60); - long[] bitmapC = makeBitmap(rows, 20); - - Map gcBitmaps = new HashMap<>(); - gcBitmaps.put(fileIdA + "_0", bitmapA); - gcBitmaps.put(fileIdB + "_0", bitmapB); - gcBitmaps.put(fileIdC + "_0", bitmapC); - - invokeCreateGCCheckpoint(safeGcTs, gcBitmaps); - - String cpPath = resolve(testCheckpointDir, getGcFileName(safeGcTs)); - assertTrue("GC checkpoint file must exist", storage.exists(cpPath)); - - Map entries = new HashMap<>(); - int rgCount = CheckpointFileIO.readCheckpointParallel(cpPath, - e -> entries.put(e.fileId + "_" + e.rgId, e)); - - assertEquals("checkpoint must contain all 3 RGs (not just candidates)", 3, rgCount); - assertEquals("entries map size must be 3", 3, entries.size()); - - CheckpointFileIO.CheckpointEntry entA = entries.get(fileIdA + "_0"); - assertNotNull("fileIdA must be present", entA); - assertEquals("fileIdA recordNum", rows, entA.recordNum); - assertArrayEquals("fileIdA bitmap must match", bitmapA, entA.bitmap); - - CheckpointFileIO.CheckpointEntry entB = entries.get(fileIdB + "_0"); - assertNotNull("fileIdB must be present", entB); - assertEquals("fileIdB recordNum", rows, entB.recordNum); - assertArrayEquals("fileIdB bitmap must match", bitmapB, entB.bitmap); - - CheckpointFileIO.CheckpointEntry entC = entries.get(fileIdC + "_0"); - assertNotNull("fileIdC (non-candidate) must be present", entC); - assertEquals("fileIdC recordNum", rows, entC.recordNum); - assertArrayEquals("fileIdC bitmap must match", bitmapC, entC.bitmap); - } - - /** - * Verifies that the GC checkpoint bitmap content faithfully matches the - * {@code gcSnapshotBitmaps} passed to {@code createCheckpoint}: each word of each - * per-RG bitmap must be preserved exactly, with no cross-RG contamination. - * - *

    Uses a 2-RG file with deliberately complementary bitmaps: - *

      - *
    • RG 0: first word all-ones ({@code rows 0-63} deleted), second word zero
      • - *
    • RG 1: first word zero, second word all-ones ({@code rows 64-127} deleted)
      • - *
    - */ - @Test - public void testGCCheckpoint_bitmapContentIsExact() throws Exception - { - final long fileId = 88001L; - final int rows = 128; // 2 words per RG - final long safeGcTs = 600L; - - retinaManager.addVisibility(fileId, 0, rows, 0L, null, false); - retinaManager.addVisibility(fileId, 1, rows, 0L, null, false); - - long[] bitmapRg0 = new long[]{-1L, 0L}; // rows 0-63 deleted - long[] bitmapRg1 = new long[]{0L, -1L}; // rows 64-127 deleted - - Map gcBitmaps = new HashMap<>(); - gcBitmaps.put(fileId + "_0", bitmapRg0); - gcBitmaps.put(fileId + "_1", bitmapRg1); - - invokeCreateGCCheckpoint(safeGcTs, gcBitmaps); - - String cpPath = resolve(testCheckpointDir, getGcFileName(safeGcTs)); - assertTrue("GC checkpoint file must exist", storage.exists(cpPath)); - - Map entries = new HashMap<>(); - int rgCount = CheckpointFileIO.readCheckpointParallel(cpPath, - e -> entries.put(e.fileId + "_" + e.rgId, e)); - - assertEquals("checkpoint must contain 2 RGs", 2, rgCount); - - CheckpointFileIO.CheckpointEntry rg0 = entries.get(fileId + "_0"); - assertNotNull("RG 0 must be present", rg0); - assertEquals("RG 0 word 0 must be all-ones (rows 0-63 deleted)", -1L, rg0.bitmap[0]); - assertEquals("RG 0 word 1 must be zero (rows 64-127 live)", 0L, rg0.bitmap[1]); - - CheckpointFileIO.CheckpointEntry rg1 = entries.get(fileId + "_1"); - assertNotNull("RG 1 must be present", rg1); - assertEquals("RG 1 word 0 must be zero (rows 0-63 live)", 0L, rg1.bitmap[0]); - assertEquals("RG 1 word 1 must be all-ones (rows 64-127 deleted)", -1L, rg1.bitmap[1]); - } } diff --git a/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestStorageGarbageCollector.java b/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestStorageGarbageCollector.java index e56b1f1b45..33f09f25fe 100644 --- a/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestStorageGarbageCollector.java +++ b/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestStorageGarbageCollector.java @@ -50,7 +50,6 @@ import org.junit.Test; import java.lang.reflect.Field; -import java.lang.reflect.Method; import java.nio.file.Files; import java.nio.file.Path; import java.sql.PreparedStatement; @@ -1627,117 +1626,6 @@ public void testRgIdForGlobalRowOffset_manyRgs() } } - // ======================================================================= - // Section 7c: createCheckpointDirect vs createCheckpoint consistency - // ======================================================================= - - /** - * Both checkpoint paths (queued via rgVisibilityMap traversal and direct via - * pre-built entries) must produce byte-identical files when given the same - * visibility state. - */ - @Test - public void testCheckpointDirect_matchesStandardCheckpoint() throws Exception - { - long ts = 500L; - int numFiles = 3; - int rowsPerRg = 64; - - for (int fid = 1; fid <= numFiles; fid++) - { - retinaManager.addVisibility(fid, 0, rowsPerRg, 0L, null, false); - for (int d = 0; d < fid; d++) - { - retinaManager.deleteRecord(fid, 0, d, ts - 100); - } - } - - // Build pre-built entries identical to what runGC() would construct. - List entries = new ArrayList<>(); - Field rgMapField = RetinaResourceManager.class.getDeclaredField("rgVisibilityMap"); - rgMapField.setAccessible(true); - @SuppressWarnings("unchecked") - Map rgMap = - (Map) rgMapField.get(retinaManager); - for (Map.Entry e : rgMap.entrySet()) - { - long fileId = RetinaUtils.parseFileIdFromRgKey(e.getKey()); - int rgId = RetinaUtils.parseRgIdFromRgKey(e.getKey()); - long[] bitmap = e.getValue().getVisibilityBitmap(ts); - entries.add(new CheckpointFileIO.CheckpointEntry( - fileId, rgId, (int) e.getValue().getRecordNum(), bitmap)); - } - - // Obtain the private CheckpointType.GC enum value via reflection. - @SuppressWarnings("unchecked") - Class> checkpointTypeClass = (Class>) - Class.forName("io.pixelsdb.pixels.retina.RetinaResourceManager$CheckpointType"); - Object gcType = null; - for (Object constant : checkpointTypeClass.getEnumConstants()) - { - if (constant.toString().equals("GC")) - { - gcType = constant; - break; - } - } - assertNotNull("CheckpointType.GC must exist", gcType); - - // Call createCheckpoint (standard path) - Method createCheckpointMethod = RetinaResourceManager.class.getDeclaredMethod( - "createCheckpoint", long.class, checkpointTypeClass); - createCheckpointMethod.setAccessible(true); - @SuppressWarnings("unchecked") - CompletableFuture f1 = (CompletableFuture) createCheckpointMethod.invoke( - retinaManager, ts, gcType); - f1.join(); - - // Call createCheckpointDirect (optimized path) with a different timestamp to get a different file name - long ts2 = ts + 1; - Method createCheckpointDirectMethod = RetinaResourceManager.class.getDeclaredMethod( - "createCheckpointDirect", long.class, checkpointTypeClass, List.class); - createCheckpointDirectMethod.setAccessible(true); - @SuppressWarnings("unchecked") - CompletableFuture f2 = (CompletableFuture) createCheckpointDirectMethod.invoke( - retinaManager, ts2, gcType, entries); - f2.join(); - - // Read both checkpoint files and compare entries. - // Files may have entries in different order (due to producer-consumer concurrency), - // so we normalize by sorting entries by (fileId, rgId) before comparing. - Field checkpointDirField = RetinaResourceManager.class.getDeclaredField("checkpointDir"); - checkpointDirField.setAccessible(true); - String checkpointDir = (String) checkpointDirField.get(retinaManager); - - Field hostField = RetinaResourceManager.class.getDeclaredField("retinaHostName"); - hostField.setAccessible(true); - String hostName = (String) hostField.get(retinaManager); - - String path1 = RetinaUtils.buildCheckpointPath( - checkpointDir, RetinaUtils.CHECKPOINT_PREFIX_GC, hostName, ts); - String path2 = RetinaUtils.buildCheckpointPath( - checkpointDir, RetinaUtils.CHECKPOINT_PREFIX_GC, hostName, ts2); - - Map standard = new HashMap<>(); - CheckpointFileIO.readCheckpointParallel(path1, entry -> - standard.put(entry.fileId + "_" + entry.rgId, - Arrays.copyOf(entry.bitmap, entry.bitmap.length))); - - Map direct = new HashMap<>(); - CheckpointFileIO.readCheckpointParallel(path2, entry -> - direct.put(entry.fileId + "_" + entry.rgId, - Arrays.copyOf(entry.bitmap, entry.bitmap.length))); - - assertEquals("entry count must match", standard.size(), direct.size()); - for (Map.Entry e : standard.entrySet()) - { - long[] directBitmap = direct.get(e.getKey()); - assertNotNull("direct checkpoint must contain key=" + e.getKey(), directBitmap); - assertTrue("bitmaps must be identical for key=" + e.getKey(), - Arrays.equals(e.getValue(), directBitmap)); - } - } - // ======================================================================= // Section 7d: concurrent dual-write pressure test // ======================================================================= diff --git a/proto/transaction.proto b/proto/transaction.proto index 631afedbc8..0489422470 100644 --- a/proto/transaction.proto +++ b/proto/transaction.proto @@ -22,8 +22,6 @@ syntax = "proto3"; -import "google/protobuf/empty.proto"; - option java_multiple_files = false; option java_package = "io.pixelsdb.pixels.daemon"; option java_outer_classname = "TransProto"; @@ -45,7 +43,8 @@ service TransService { rpc GetTransConcurrency (GetTransConcurrencyRequest) returns (GetTransConcurrencyResponse); rpc BindExternalTraceId (BindExternalTraceIdRequest) returns (BindExternalTraceIdResponse); rpc DumpTrans (DumpTransRequest) returns (DumpTransResponse); - rpc GetSafeGcTimestamp(google.protobuf.Empty) returns (GetSafeGcTimestampResponse); + rpc GetSafeVisibilityFoldingTimestamp(GetSafeVisibilityFoldingTimestampRequest) + returns (GetSafeVisibilityFoldingTimestampResponse); rpc MarkTransOffloaded (MarkTransOffloadedRequest) returns (MarkTransOffloadedResponse); } @@ -219,7 +218,12 @@ message DumpTransResponse { int32 errorCode = 1; } -message GetSafeGcTimestampResponse { +message GetSafeVisibilityFoldingTimestampRequest { + // True when the returned timestamp must remain safe for live running queries. + bool includeRunningQueries = 1; +} + +message GetSafeVisibilityFoldingTimestampResponse { int32 errorCode = 1; uint64 timestamp = 2; } From 8c86396ab68498a0c87d22712f07d5b8041462d9 Mon Sep 17 00:00:00 2001 From: Dongyang Geng Date: Wed, 27 May 2026 19:48:50 +0800 Subject: [PATCH 16/17] fix: storage gc index updates via localIndexService --- .../pixels/retina/RetinaResourceManager.java | 6 +- .../retina/StorageGarbageCollector.java | 127 ++++++++++-------- .../retina/TestStorageGarbageCollector.java | 17 +-- 3 files changed, 88 insertions(+), 62 deletions(-) diff --git a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/RetinaResourceManager.java b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/RetinaResourceManager.java index e3cd74c3bc..62009093f0 100644 --- a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/RetinaResourceManager.java +++ b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/RetinaResourceManager.java @@ -22,6 +22,8 @@ import com.google.protobuf.ByteString; import io.pixelsdb.pixels.common.exception.RetinaException; import io.pixelsdb.pixels.common.exception.TransException; +import io.pixelsdb.pixels.common.index.service.IndexService; +import io.pixelsdb.pixels.common.index.service.IndexServiceProvider; import io.pixelsdb.pixels.common.metadata.MetadataService; import io.pixelsdb.pixels.common.metadata.domain.Column; import io.pixelsdb.pixels.common.metadata.domain.Layout; @@ -64,6 +66,7 @@ public class RetinaResourceManager private static final Logger logger = LogManager.getLogger(RetinaResourceManager.class); private final MetadataService metadataService; + private final IndexService indexService; private final Map rgVisibilityMap; private final Map> pixelsWriteBufferMap; private String retinaHostName; @@ -134,6 +137,7 @@ static final class RetiredFile private RetinaResourceManager() { this.metadataService = MetadataService.Instance(); + this.indexService = IndexServiceProvider.getService(IndexServiceProvider.ServiceMode.local); this.rgVisibilityMap = new ConcurrentHashMap<>(); this.pixelsWriteBufferMap = new ConcurrentHashMap<>(); @@ -171,7 +175,7 @@ private RetinaResourceManager() EncodingLevel encodingLevel = EncodingLevel.from( Integer.parseInt(config.getProperty("retina.storage.gc.encoding.level"))); long retireDelayMs = (long) (Double.parseDouble(config.getProperty("retina.storage.gc.file.retire.delay.hours")) * 3_600_000L); - gc = new StorageGarbageCollector(this, this.metadataService, + gc = new StorageGarbageCollector(this, this.metadataService, this.indexService, threshold, targetFileSize, maxFilesPerGroup, maxGroups, rowGroupSize, encodingLevel, retireDelayMs); logger.info("Storage GC enabled (threshold={}, targetFileSize={}, maxFilesPerGroup={}, maxGroups={})", diff --git a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/StorageGarbageCollector.java b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/StorageGarbageCollector.java index d72ef5aaa9..fd0d9cb751 100644 --- a/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/StorageGarbageCollector.java +++ b/pixels-retina/src/main/java/io/pixelsdb/pixels/retina/StorageGarbageCollector.java @@ -23,10 +23,9 @@ import io.pixelsdb.pixels.common.exception.MetadataException; import io.pixelsdb.pixels.common.exception.RetinaException; import io.pixelsdb.pixels.common.index.IndexOption; -import io.pixelsdb.pixels.common.index.MainIndex; -import io.pixelsdb.pixels.common.index.MainIndexFactory; -import io.pixelsdb.pixels.common.index.RowIdRange; -import io.pixelsdb.pixels.common.index.SinglePointIndexFactory; +import io.pixelsdb.pixels.common.index.ResolvedPrimary; +import io.pixelsdb.pixels.common.index.RollbackEntry; +import io.pixelsdb.pixels.common.index.service.IndexService; import io.pixelsdb.pixels.common.metadata.MetadataService; import io.pixelsdb.pixels.common.metadata.domain.File; import io.pixelsdb.pixels.common.metadata.domain.KeyColumns; @@ -67,6 +66,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -84,6 +84,7 @@ public class StorageGarbageCollector private final RetinaResourceManager resourceManager; private final MetadataService metadataService; + private final IndexService indexService; private final double gcThreshold; private final long targetFileSize; private final int maxFilesPerGroup; @@ -213,9 +214,9 @@ static final class RewriteResult *
    * Alignment invariant: {@code oldRowIds.size() == pendingIndexEntries.size()}; each * slot corresponds 1:1 to the same-position entry in {@link #pendingIndexEntries}. Slots - * where {@link io.pixelsdb.pixels.common.index.SinglePointIndex#updatePrimaryEntry} returned - * a negative value (i.e. no prior entry to replace) are stored as {@code -1L} placeholders, - * so that rollback can pair each {@code PendingIndexEntry} with its own old rowId. + * where {@link IndexService#resolvePrimary} returned an empty optional (i.e. no prior entry + * to replace) are stored as {@code -1L} placeholders, so that rollback can pair each + * {@code PendingIndexEntry} with its own old rowId. */ List oldRowIds; @@ -243,6 +244,7 @@ static final class RewriteResult StorageGarbageCollector(RetinaResourceManager resourceManager, MetadataService metadataService, + IndexService indexService, double gcThreshold, long targetFileSize, int maxFilesPerGroup, @@ -253,6 +255,7 @@ static final class RewriteResult { this.resourceManager = resourceManager; this.metadataService = metadataService; + this.indexService = indexService; this.gcThreshold = gcThreshold; this.targetFileSize = targetFileSize; this.maxFilesPerGroup = maxFilesPerGroup; @@ -1099,21 +1102,23 @@ void syncIndex(RewriteResult result, long tableId) throws Exception return; } - MainIndex mainIndex = MainIndexFactory.Instance().getMainIndex(tableId); - IndexProto.RowIdBatch rowIdBatch = mainIndex.allocateRowIdBatch(tableId, totalRows); + long primaryIndexId = metadataService.getPrimaryIndex(tableId).getId(); + IndexOption indexOption = IndexOption.builder().vNodeId(result.group.virtualNodeId).build(); + + IndexProto.RowIdBatch rowIdBatch = indexService.allocateRowIdBatch(tableId, totalRows); long newRowIdStart = rowIdBatch.getRowIdStart(); result.newRowIdStart = newRowIdStart; - insertMainIndexEntries(result, mainIndex, newRowIdStart); + insertMainIndexEntries(result, tableId, primaryIndexId, indexOption, newRowIdStart); if (!result.pendingIndexEntries.isEmpty()) { - result.oldRowIds = updateSinglePointIndex(result, tableId, newRowIdStart); + result.oldRowIds = updateSinglePointIndex(result, tableId, primaryIndexId, indexOption, newRowIdStart); } } - private void insertMainIndexEntries(RewriteResult result, MainIndex mainIndex, - long newRowIdStart) throws Exception + private void insertMainIndexEntries(RewriteResult result, long tableId, long primaryIndexId, + IndexOption indexOption, long newRowIdStart) throws Exception { int totalRows = result.newFileRgRowStart[result.newFileRgCount]; List entries = new ArrayList<>(totalRows); @@ -1132,39 +1137,61 @@ private void insertMainIndexEntries(RewriteResult result, MainIndex mainIndex, .setFileId(result.newFileId).setRgId(curRgId).setRgRowOffset(rgOff)) .build()); } - mainIndex.putEntries(entries); - mainIndex.flushCache(result.newFileId); + indexService.putMainIndexEntriesOnly(tableId, entries); + indexService.flushIndexEntriesOfFile(tableId, primaryIndexId, result.newFileId, true, indexOption); } - private List updateSinglePointIndex(RewriteResult result, long tableId, - long newRowIdStart) throws Exception + /** + * Mirrors Retina's write-path "resolve + Only" pattern: one batch resolve to capture + * pre-update rowIds (recorded for rollback), then one batch updatePrimaryIndexEntriesOnly + * to swing the primary pointers onto the freshly allocated rowIds. + * + *

    TODO(concurrency): This pair of calls is not atomic, unlike the previous single-shot + * {@code SinglePointIndex#updatePrimaryEntry} (per-key atomic getAndSet). If a concurrent + * writer mutates the same primary key between {@code resolvePrimary} and + * {@code updatePrimaryIndexEntriesOnly}, the {@code oldRowIds} we record can be stale w.r.t. + * the value actually clobbered by our update. Rollback is still safe — {@code restorePrimaryIndexEntries} + * only writes back when the current pointer still equals our {@code newRowId}, so concurrent + * writes that ran after our update are never overwritten — but a rollback in the narrow + * resolve→update window can restore a stale {@code oldRowId} instead of the concurrent + * writer's value. This matches the rest of Retina's write path and is acceptable here because + * Storage GC by design targets files dominated by deleted rows. Revisit if/when + * {@code IndexService} grows a batch API that returns the rowIds atomically replaced. + */ + private List updateSinglePointIndex(RewriteResult result, long tableId, long primaryIndexId, + IndexOption indexOption, long newRowIdStart) throws Exception { - io.pixelsdb.pixels.common.metadata.domain.SinglePointIndex primaryIndex = - metadataService.getPrimaryIndex(tableId); - IndexOption indexOption = IndexOption.builder().vNodeId(result.group.virtualNodeId).build(); - io.pixelsdb.pixels.common.index.SinglePointIndex spIndex = - SinglePointIndexFactory.Instance().getSinglePointIndex( - tableId, primaryIndex.getId(), indexOption); - - // Keep oldRowIds aligned 1:1 with pendingIndexEntries: slots where - // updatePrimaryEntry returned a negative value are stored as -1L placeholders. - // rollbackSinglePointIndex relies on this alignment to pair each PendingIndexEntry - // with its own old rowId. - List oldRowIds = new ArrayList<>(result.pendingIndexEntries.size()); + int size = result.pendingIndexEntries.size(); + List keys = new ArrayList<>(size); + List entries = new ArrayList<>(size); for (PendingIndexEntry pe : result.pendingIndexEntries) { - long newRowId = newRowIdStart + pe.newGlobalRowOffset; IndexProto.IndexKey key = IndexProto.IndexKey.newBuilder() - .setTableId(tableId).setIndexId(primaryIndex.getId()) + .setTableId(tableId).setIndexId(primaryIndexId) .setKey(pe.pkBytes).setTimestamp(pe.createTs).build(); - long oldRowId = spIndex.updatePrimaryEntry(key, newRowId); + keys.add(key); + entries.add(IndexProto.PrimaryIndexEntry.newBuilder() + .setIndexKey(key) + .setRowId(newRowIdStart + pe.newGlobalRowOffset) + .build()); + } + + List> resolved = + indexService.resolvePrimary(tableId, primaryIndexId, keys, indexOption); + List oldRowIds = new ArrayList<>(size); + for (int i = 0; i < size; i++) + { + long oldRowId = resolved.get(i).map(ResolvedPrimary::getRowId).orElse(-1L); oldRowIds.add(oldRowId); if (oldRowId < 0) { - logger.warn("StorageGC syncIndex: updatePrimaryEntry returned {} for tableId={}, " + - "newGlobalRowOffset={} — index may be inconsistent", oldRowId, tableId, pe.newGlobalRowOffset); + logger.warn("StorageGC syncIndex: no resolvable primary for tableId={}, " + + "newGlobalRowOffset={} — index may be inconsistent", + tableId, result.pendingIndexEntries.get(i).newGlobalRowOffset); } } + + indexService.updatePrimaryIndexEntriesOnly(tableId, primaryIndexId, entries, indexOption); return oldRowIds; } @@ -1237,20 +1264,9 @@ void rollback(RewriteResult result) rollbackSinglePointIndex(result); } - if (result.newRowIdStart > 0) - { - try - { - int totalRows = result.newFileRgRowStart[result.newFileRgCount]; - MainIndex mainIndex = MainIndexFactory.Instance().getMainIndex(result.group.tableId); - mainIndex.deleteRowIdRange(new RowIdRange(result.newRowIdStart, - result.newRowIdStart + totalRows, result.newFileId, 0, 0, totalRows)); - } - catch (Exception ex) - { - logger.warn("Rollback: failed to clean MainIndex for fileId={}", result.newFileId, ex); - } - } + // TODO: MainIndex entries for [newRowIdStart, newRowIdStart + totalRows) on newFileId are not cleaned here. + // Safe under current invariants (rowIds are monotonic and never reused; newFileId is deleted from catalog + // and not reused; no scanner traverses MainIndex globally). Revisit if any of these invariants change. unregisterDualWrite(result); @@ -1308,10 +1324,8 @@ private void rollbackSinglePointIndex(RewriteResult result) { return; } + long primaryIndexId = primaryIndex.getId(); IndexOption indexOption = IndexOption.builder().vNodeId(result.group.virtualNodeId).build(); - io.pixelsdb.pixels.common.index.SinglePointIndex spIndex = - SinglePointIndexFactory.Instance().getSinglePointIndex( - result.group.tableId, primaryIndex.getId(), indexOption); // Alignment invariant: oldRowIds.size() == pendingIndexEntries.size() // (established in updateSinglePointIndex). Walk them in lockstep by @@ -1324,6 +1338,7 @@ private void rollbackSinglePointIndex(RewriteResult result) "rolling back the common prefix only — index may remain inconsistent", result.pendingIndexEntries.size(), result.oldRowIds.size()); } + List rollbackEntries = new ArrayList<>(n); for (int i = 0; i < n; i++) { long oldRowId = result.oldRowIds.get(i); @@ -1333,9 +1348,15 @@ private void rollbackSinglePointIndex(RewriteResult result) } PendingIndexEntry pe = result.pendingIndexEntries.get(i); IndexProto.IndexKey key = IndexProto.IndexKey.newBuilder() - .setTableId(result.group.tableId).setIndexId(primaryIndex.getId()) + .setTableId(result.group.tableId).setIndexId(primaryIndexId) .setKey(pe.pkBytes).setTimestamp(pe.createTs).build(); - spIndex.updatePrimaryEntry(key, oldRowId); + long newRowId = result.newRowIdStart + pe.newGlobalRowOffset; + rollbackEntries.add(new RollbackEntry(key, oldRowId, newRowId)); + } + if (!rollbackEntries.isEmpty()) + { + indexService.restorePrimaryIndexEntries( + result.group.tableId, primaryIndexId, rollbackEntries, indexOption); } } catch (Exception e) diff --git a/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestStorageGarbageCollector.java b/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestStorageGarbageCollector.java index 33f09f25fe..138cb834c8 100644 --- a/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestStorageGarbageCollector.java +++ b/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestStorageGarbageCollector.java @@ -19,6 +19,7 @@ */ package io.pixelsdb.pixels.retina; +import io.pixelsdb.pixels.common.index.service.LocalIndexService; import io.pixelsdb.pixels.common.metadata.MetadataService; import io.pixelsdb.pixels.common.utils.CheckpointFileIO; import io.pixelsdb.pixels.common.utils.MetaDBUtil; @@ -178,8 +179,8 @@ public void setUp() retinaManager = RetinaResourceManager.Instance(); resetManagerState(); cleanupOrderedDir(); - gc = new StorageGarbageCollector(retinaManager, metadataService, 0.5, 134_217_728L, Integer.MAX_VALUE, 10, - 1048576, EncodingLevel.EL2, 86_400_000L); + gc = new StorageGarbageCollector(retinaManager, metadataService, LocalIndexService.Instance(), + 0.5, 134_217_728L, Integer.MAX_VALUE, 10, 1048576, EncodingLevel.EL2, 86_400_000L); } @After @@ -1664,7 +1665,7 @@ public void testDualWrite_concurrentPressure() throws Exception // batch (any encoded pixel exceeds 1 byte), preserving the 1:1 old-RG-to-new-RG // mapping so each thread targets a distinct new RGVisibility object. StorageGarbageCollector localGc = new StorageGarbageCollector( - retinaManager, metadataService, 0.5, 134_217_728L, + retinaManager, metadataService, LocalIndexService.Instance(), 0.5, 134_217_728L, Integer.MAX_VALUE, 10, 1, EncodingLevel.EL2, 86_400_000L); StorageGarbageCollector.RewriteResult result = @@ -3322,7 +3323,7 @@ private static StorageGarbageCollector newGcForGrouping( long targetFileSize, int maxFilesPerGroup, int maxGroups) { return new StorageGarbageCollector( - null, null, 0.5, targetFileSize, maxFilesPerGroup, maxGroups, + null, null, null, 0.5, targetFileSize, maxFilesPerGroup, maxGroups, 1048576, EncodingLevel.EL2, 86_400_000L); } @@ -3994,7 +3995,7 @@ static class DirectScanStorageGC extends StorageGarbageCollector DirectScanStorageGC(RetinaResourceManager rm, double threshold, int maxGroups, List fakeEntries) { - super(rm, null, threshold, 134_217_728L, Integer.MAX_VALUE, maxGroups, + super(rm, null, null, threshold, 134_217_728L, Integer.MAX_VALUE, maxGroups, 1048576, EncodingLevel.EL2, 86_400_000L); this.fakeEntries = fakeEntries; } @@ -4051,7 +4052,7 @@ static class TrackingRunStorageGC extends StorageGarbageCollector TrackingRunStorageGC(List groupsToReturn) { - super(null, null, 0.5, 0L, Integer.MAX_VALUE, 10, + super(null, null, null, 0.5, 0L, Integer.MAX_VALUE, 10, 1048576, EncodingLevel.EL2, 86_400_000L); this.groupsToReturn = groupsToReturn; } @@ -4093,7 +4094,7 @@ static class FailFirstGroupGC extends StorageGarbageCollector FailFirstGroupGC() { - super(null, null, 0.5, 0L, Integer.MAX_VALUE, 10, + super(null, null, null, 0.5, 0L, Integer.MAX_VALUE, 10, 1048576, EncodingLevel.EL2, 86_400_000L); } @@ -4131,7 +4132,7 @@ static class NoIndexSyncGC extends StorageGarbageCollector int maxGroups, int rowGroupSize, EncodingLevel encodingLevel, long retireDelayMs) { - super(rm, ms, threshold, targetFileSize, maxFilesPerGroup, maxGroups, + super(rm, ms, null, threshold, targetFileSize, maxFilesPerGroup, maxGroups, rowGroupSize, encodingLevel, retireDelayMs); } From f3c571d229c50249affd43e468e6062737d88673 Mon Sep 17 00:00:00 2001 From: Dongyang Geng Date: Wed, 27 May 2026 20:30:33 +0800 Subject: [PATCH 17/17] feat: use retina heartbeat for recovery readiness gate --- .../pixels/common/error/ErrorCode.java | 7 + .../daemon/heartbeat/HeartbeatWorker.java | 27 +++- .../daemon/transaction/TransServer.java | 153 +++++++++++++++++- 3 files changed, 179 insertions(+), 8 deletions(-) diff --git a/pixels-common/src/main/java/io/pixelsdb/pixels/common/error/ErrorCode.java b/pixels-common/src/main/java/io/pixelsdb/pixels/common/error/ErrorCode.java index 2bcd676994..8334b2a3f4 100644 --- a/pixels-common/src/main/java/io/pixelsdb/pixels/common/error/ErrorCode.java +++ b/pixels-common/src/main/java/io/pixelsdb/pixels/common/error/ErrorCode.java @@ -163,4 +163,11 @@ public class ErrorCode public static final int NODE_RETINA_INFO_FAIL = ERROR_NODE_SERVER + 1; public static final int NODE_NO_AVAILABLE = ERROR_NODE_SERVER + 2; public static final int NODE_INVALID_BUCKET = ERROR_NODE_SERVER + 3; + + // error code for retina lifecycle/recovery + private static final int ERROR_RETINA_SERVER = ERROR_BASE + 800; + public static final int RETINA_NOT_READY = ERROR_RETINA_SERVER + 1; + public static final int RETINA_MARK_READY_FAILED = ERROR_RETINA_SERVER + 2; + public static final int RETINA_UPDATE_FAILED = ERROR_RETINA_SERVER + 3; + public static final int RETINA_VISIBILITY_FAILED = ERROR_RETINA_SERVER + 4; } diff --git a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/heartbeat/HeartbeatWorker.java b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/heartbeat/HeartbeatWorker.java index 78201b6260..ac358c05f1 100644 --- a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/heartbeat/HeartbeatWorker.java +++ b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/heartbeat/HeartbeatWorker.java @@ -42,11 +42,12 @@ public class HeartbeatWorker implements Server { private static final Logger logger = LogManager.getLogger(HeartbeatWorker.class); - private static final AtomicInteger currentStatus = new AtomicInteger(NodeStatus.READY.StatusCode); + private static final AtomicInteger currentStatus = new AtomicInteger(NodeStatus.INIT.StatusCode); private final HeartbeatConfig heartbeatConfig = new HeartbeatConfig(); private final ScheduledExecutorService scheduledExecutor = Executors.newSingleThreadScheduledExecutor(); private final NodeProto.NodeRole role; private String hostName; + private String workerKey; private WorkerRegister workerRegister; private boolean initializeSuccess = false; private CountDownLatch runningLatch; @@ -59,6 +60,15 @@ public HeartbeatWorker(NodeProto.NodeRole role) initialize(); } + public static void setCurrentStatus(NodeStatus status) + { + if (status == null) + { + throw new IllegalArgumentException("status is null"); + } + currentStatus.set(status.StatusCode); + } + /** * Initialize heartbeat worker: *

    @@ -92,13 +102,16 @@ private void initialize() default: throw new IllegalStateException("Unknown heartbeat role: " + role); } + this.workerKey = key; + currentStatus.set(role == NodeProto.NodeRole.RETINA + ? NodeStatus.INIT.StatusCode + : NodeStatus.READY.StatusCode); EtcdUtil.Instance().putKeyValueWithLeaseId(key, String.valueOf(currentStatus.get()), leaseId); // start a scheduled thread to update node status periodically this.workerRegister = new WorkerRegister(key, leaseClient, leaseId); scheduledExecutor.scheduleAtFixedRate(workerRegister, 0, heartbeatConfig.getNodeHeartbeatPeriod(), TimeUnit.SECONDS); initializeSuccess = true; - currentStatus.set(NodeStatus.READY.StatusCode); logger.info("Heartbeat worker on {} is initialized", hostName); } catch (Exception e) { @@ -126,10 +139,16 @@ public void shutdown() switch (role) { case WORKER: - EtcdUtil.Instance().deleteByPrefix(Constants.HEARTBEAT_WORKER_LITERAL); + if (workerKey != null) + { + EtcdUtil.Instance().delete(workerKey); + } break; case RETINA: - EtcdUtil.Instance().deleteByPrefix(Constants.HEARTBEAT_RETINA_LITERAL); + if (workerKey != null) + { + EtcdUtil.Instance().delete(workerKey); + } break; default: throw new IllegalStateException("Unknown heartbeat role: " + role); diff --git a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/transaction/TransServer.java b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/transaction/TransServer.java index 2e3be1a464..05dd64192a 100644 --- a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/transaction/TransServer.java +++ b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/transaction/TransServer.java @@ -19,13 +19,28 @@ */ package io.pixelsdb.pixels.daemon.transaction; -import io.grpc.ServerBuilder; -import io.pixelsdb.pixels.common.server.Server; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import java.io.IOException; -import java.util.concurrent.TimeUnit; +import io.etcd.jetcd.KeyValue; +import io.grpc.ServerBuilder; +import io.pixelsdb.pixels.common.server.Server; +import io.pixelsdb.pixels.common.utils.ConfigFactory; +import io.pixelsdb.pixels.common.utils.Constants; +import io.pixelsdb.pixels.common.utils.EtcdUtil; +import io.pixelsdb.pixels.daemon.heartbeat.NodeStatus; /** * @author hank @@ -35,6 +50,13 @@ public class TransServer implements Server { private static final Logger log = LogManager.getLogger(TransServer.class); + /** + * Default time to wait for all expected Retina nodes to reach READY before giving up + * and aborting the trans server boot. Overridable by {@code trans.server.retina.readiness.timeout.ms}. + */ + private static final long DEFAULT_RETINA_READINESS_TIMEOUT_MS = 10 * 60 * 1000L; + private static final long RETINA_READINESS_POLL_INTERVAL_MS = 1_000L; + private boolean running = false; private final io.grpc.Server rpcServer; @@ -69,6 +91,7 @@ public void run() { try { + awaitRetinaReady(); this.rpcServer.start(); this.running = true; this.rpcServer.awaitTermination(); @@ -83,4 +106,126 @@ public void run() this.shutdown(); } } + + /** + * Boot-time gate. When {@code retina.enable=true}, blocks until every node listed in + * {@code $PIXELS_HOME/etc/retina} reports {@code NodeStatus.READY} via heartbeat. When + * {@code retina.enable=false}, returns immediately. On timeout, throws so that + * {@link #run()} aborts and the supervisor can restart the process. + * + *

    This is intentionally a one-shot check executed before the gRPC server starts. + * Once the trans server is serving, it does not re-check Retina lifecycle state. + */ + private void awaitRetinaReady() + { + ConfigFactory config = ConfigFactory.Instance(); + if (!Boolean.parseBoolean(config.getProperty("retina.enable"))) + { + return; + } + + // Load expected Retina nodes from $PIXELS_HOME/etc/retina. + Path retinaFile = Paths.get(config.getProperty("pixels.home"), "etc", "retina"); + if (!Files.isRegularFile(retinaFile)) + { + throw new IllegalStateException(retinaFile + " is missing"); + } + Set expected = new LinkedHashSet<>(); + try + { + for (String raw : Files.readAllLines(retinaFile, StandardCharsets.UTF_8)) + { + String line = raw.trim(); + if (line.isEmpty() || line.startsWith("#")) + { + continue; + } + String host = line.split("\\s+", 2)[0]; + expected.add(host); + } + } catch (IOException e) + { + throw new IllegalStateException("Failed to load expected Retina nodes from " + + "$PIXELS_HOME/etc/retina", e); + } + if (expected.isEmpty()) + { + throw new IllegalStateException( + "retina.enable=true but $PIXELS_HOME/etc/retina has no nodes"); + } + + long deadline = System.currentTimeMillis() + DEFAULT_RETINA_READINESS_TIMEOUT_MS; + EtcdUtil etcd = EtcdUtil.Instance(); + String prefix = Constants.HEARTBEAT_RETINA_LITERAL; + int prefixLen = prefix.length(); + log.info("Waiting for {} Retina node(s) to report READY (timeout {} ms)", + expected.size(), DEFAULT_RETINA_READINESS_TIMEOUT_MS); + while (true) + { + String reason = null; + // Poll all Retina heartbeat keys once and check whether every expected node is READY. + Map observed; + try + { + List all = etcd.getKeyValuesByPrefix(prefix); + observed = new HashMap<>(all.size() * 2); + for (KeyValue kv : all) + { + String key = kv.getKey().toString(StandardCharsets.UTF_8); + if (key.length() > prefixLen) + { + observed.put(key.substring(prefixLen), kv); + } + } + } catch (RuntimeException e) + { + observed = null; + reason = "etcd heartbeat read failed: " + e.getMessage(); + } + if (reason == null) + { + for (String host : expected) + { + KeyValue kv = observed.get(host); + if (kv == null) + { + reason = "Retina node " + host + " has no heartbeat status"; + break; + } + if (kv.getLease() <= 0) + { + reason = "Retina node " + host + " has heartbeat status without lease"; + break; + } + String status = kv.getValue().toString(StandardCharsets.UTF_8).trim(); + if (!String.valueOf(NodeStatus.READY.StatusCode).equals(status)) + { + reason = "Retina node " + host + " heartbeat status is " + status; + break; + } + } + } + if (reason == null) + { + log.info("All Retina nodes are READY, starting trans server"); + return; + } + if (System.currentTimeMillis() >= deadline) + { + throw new IllegalStateException( + "Timed out waiting for Retina readiness after " + + DEFAULT_RETINA_READINESS_TIMEOUT_MS + + " ms; last reason: " + reason); + } + try + { + Thread.sleep(RETINA_READINESS_POLL_INTERVAL_MS); + } catch (InterruptedException e) + { + Thread.currentThread().interrupt(); + throw new IllegalStateException( + "Interrupted while waiting for Retina readiness", e); + } + } + } }