From cda4aa00d585ac67d8c2df5b347941670e9a5a0c Mon Sep 17 00:00:00 2001 From: Alex Gaetano Padula Date: Thu, 10 Sep 2026 06:43:49 -0400 Subject: [PATCH] major release of java binding extending to tidesdb v10.0.0 also now housing in-repo documentation --- README.md | 20 +- doc/java.md | 1155 +++++ doc/manual.json | 19 + pom.xml | 4 +- src/main/c/com_tidesdb_TidesDB.c | 2950 ++++++----- src/main/java/com/tidesdb/CfStats.java | 418 ++ src/main/java/com/tidesdb/ColumnFamily.java | 299 +- .../java/com/tidesdb/ColumnFamilyConfig.java | 813 ++-- .../com/tidesdb/CompressionAlgorithm.java | 45 +- src/main/java/com/tidesdb/Config.java | 652 ++- src/main/java/com/tidesdb/DbStats.java | 617 ++- src/main/java/com/tidesdb/EncodingStats.java | 125 + src/main/java/com/tidesdb/IoClass.java | 96 + src/main/java/com/tidesdb/IoStat.java | 107 + src/main/java/com/tidesdb/IoStats.java | 99 + src/main/java/com/tidesdb/LogLevel.java | 38 +- .../java/com/tidesdb/ObjectStoreConfig.java | 227 - .../java/com/tidesdb/PreparedTransaction.java | 71 + src/main/java/com/tidesdb/RangeStats.java | 86 + src/main/java/com/tidesdb/S3Config.java | 169 - src/main/java/com/tidesdb/Snapshot.java | 112 + src/main/java/com/tidesdb/StallReason.java | 109 + src/main/java/com/tidesdb/StallStat.java | 81 + src/main/java/com/tidesdb/StallStats.java | 95 + src/main/java/com/tidesdb/Stats.java | 470 -- src/main/java/com/tidesdb/TidesDB.java | 721 +-- .../java/com/tidesdb/TidesDBException.java | 93 +- .../java/com/tidesdb/TidesDBIterator.java | 6 + src/main/java/com/tidesdb/Transaction.java | 648 ++- .../java/com/tidesdb/TransactionState.java | 80 + .../java/com/tidesdb/PojoAndEnumTest.java | 1308 +++-- src/test/java/com/tidesdb/TidesDBTest.java | 4317 +++++------------ 32 files changed, 8537 insertions(+), 7513 deletions(-) create mode 100644 doc/java.md create mode 100644 doc/manual.json create mode 100644 src/main/java/com/tidesdb/CfStats.java create mode 100644 src/main/java/com/tidesdb/EncodingStats.java create mode 100644 src/main/java/com/tidesdb/IoClass.java create mode 100644 src/main/java/com/tidesdb/IoStat.java create mode 100644 src/main/java/com/tidesdb/IoStats.java delete mode 100644 src/main/java/com/tidesdb/ObjectStoreConfig.java create mode 100644 src/main/java/com/tidesdb/PreparedTransaction.java create mode 100644 src/main/java/com/tidesdb/RangeStats.java delete mode 100644 src/main/java/com/tidesdb/S3Config.java create mode 100644 src/main/java/com/tidesdb/Snapshot.java create mode 100644 src/main/java/com/tidesdb/StallReason.java create mode 100644 src/main/java/com/tidesdb/StallStat.java create mode 100644 src/main/java/com/tidesdb/StallStats.java delete mode 100644 src/main/java/com/tidesdb/Stats.java create mode 100644 src/main/java/com/tidesdb/TransactionState.java diff --git a/README.md b/README.md index 86dc78e..588a849 100644 --- a/README.md +++ b/README.md @@ -2,21 +2,25 @@ tidesdb-java is the official Java binding for TidesDB. -TidesDB is a fast and efficient key-value storage engine library written in C. The underlying data structure is based on a log-structured merge-tree (LSM-tree). This Java binding provides a safe, idiomatic Java interface to TidesDB with full support for all features. +TidesDB is a fast and efficient key-value storage engine library. The underlying data structure is based on a log-structured merge-tree (LSM-tree). This Java binding provides a safe, idiomatic Java interface to TidesDB with full support for all features. ## Features - MVCC with five isolation levels from READ UNCOMMITTED to SERIALIZABLE +- Snapshots and point-in-time reads at a named sequence +- Two-phase commit with recovery of transactions left in doubt - Column families (isolated key-value stores with independent configuration) -- Bidirectional iterators with forward/backward traversal and seek support +- Bidirectional iterators with forward/backward traversal, seek, and range-scoped scans +- Range and prefix deletes that cost one entry however many keys they cover - TTL (time to live) support with automatic key expiration -- LZ4, LZ4 Fast, ZSTD, Snappy, or no compression -- Bloom filters with configurable false positive rates -- Global block CLOCK cache for hot blocks +- Stacked encoding pipelines: LZ4, LZ4 Fast, ZSTD, Snappy, or none +- Key/value separation with a shared value log and background reclamation +- Partition range filters with configurable false positive rates +- Global block cache for hot blocks - Savepoints for partial transaction rollback -- Six built-in comparators plus custom registration - -For Java usage you can go to the TidesDB Java Reference [here](https://tidesdb.com/reference/java/). +- Transaction timeouts and cross-thread aborts +- Commit hooks for change data capture +- Statistics for column families, the database, the cache, write stalls, device I/O, encoding chains, and key ranges ## License diff --git a/doc/java.md b/doc/java.md new file mode 100644 index 0000000..f8b005f --- /dev/null +++ b/doc/java.md @@ -0,0 +1,1155 @@ +--- +title: TidesDB Java API Reference +description: Complete Java API reference for TidesDB +--- + +
+ +If you want to download the source of this document, you can find it [here](https://github.com/tidesdb/tidesdb.github.io/blob/master/src/content/docs/reference/java.md). + +
+ +
+ +## Getting Started + +### Prerequisites + +You **must** have the TidesDB shared C library installed on your system. You can find the installation instructions [here](/reference/building/#_top). + +This binding targets **TidesDB 10.x**. It does not work against a 9.x library: the two have different public APIs and different on-disk formats. + +## Requirements + +- Java 11 or higher +- Maven 3.9.6+ +- TidesDB 10.x native library installed on the system + +### Building the JNI Library + +```bash +cd src/main/c +cmake -S . -B build +cmake --build build +sudo cmake --install build +``` + +### Adding to Your Project + +**Maven** + +```xml + + com.tidesdb + tidesdb-java + 1.0.0 + +``` + +## Usage + +### Opening and Closing a Database + +`TidesDB` implements `Closeable`, so use it with try-with-resources. Close every +`Transaction`, `TidesDBIterator`, and `Snapshot` derived from it first. + +```java +import com.tidesdb.*; + +Config config = Config.builder("/path/to/db") + .logLevel(LogLevel.INFO) + .memtableSyncMode(SyncMode.SYNC_FULL) + .build(); + +try (TidesDB db = TidesDB.open(config)) { + // ... use the database +} +``` + +Every field left at zero is resolved to the engine's own default, so a minimal +configuration is just a path: + +```java +try (TidesDB db = TidesDB.open(Config.builder("/path/to/db").build())) { + // ... +} +``` + +To see what the engine will actually pick, ask for the native defaults: + +```java +Config defaults = Config.defaultConfig("/path/to/db"); +System.out.println(defaults.getMemtableWriteBufferSize()); // e.g. 67108864 +System.out.println(defaults.getValueSeparationThreshold()); // e.g. 1024 +``` + +`defaultConfig` returns an immutable value; use `toBuilder()` to adjust it. + +```java +Config config = Config.defaultConfig("/path/to/db") + .toBuilder() + .logToFile(true) + .memtableL0QueueStallThreshold(16) + .build(); +``` + +A second handle on the same directory — from this process or any other — is +refused with `ERR_LOCKED`. + +Closing reports no status: an I/O error while writing the last of the data is +logged rather than thrown. If you need your data on the device, ask for that +first with `syncWal()`, `checkpoint()`, or by running under `SYNC_FULL`. + +### Raising the Open-File Limit + +The engine reads the process open-file ceiling at open time and lowers +`maxOpenSSTables` to fit it, so an over-large setting costs descriptors it never +gets rather than failing opens. Raising the ceiling is an explicit, opt-in +action, and must happen **before** `open`. + +```java +long ceiling = TidesDB.raiseOpenFileLimit(8192); +System.out.println("open-file ceiling is now " + ceiling); + +try (TidesDB db = TidesDB.open(Config.builder("/path/to/db") + .maxOpenSSTables(4096) + .build())) { + // ... +} +``` + +Passing a value of zero or less just reports the current ceiling. + +### Creating and Dropping Column Families + +A column family is an isolated key-value store with its own configuration. The +memtable, write-ahead log, block cache, and value log are database-level and +shared across all of them. + +```java +ColumnFamilyConfig cfConfig = ColumnFamilyConfig.builder() + .compression(CompressionAlgorithm.LZ4) + .enableBloomFilter(true) + .bloomFpr(0.01) + .build(); + +db.createColumnFamily("users", cfConfig); + +ColumnFamily users = db.getColumnFamily("users"); + +for (String name : db.listColumnFamilies()) { + System.out.println(name); +} + +db.dropColumnFamily("users"); +``` + +`ColumnFamilyConfig.builder()` starts from the native defaults, so you only set +what you want to change. The `name` field on the config is ignored by +`createColumnFamily` — the name argument is authoritative. + +Dropping destroys data and cannot be undone, and any `ColumnFamily` handle held +for that family is invalid afterwards. + +#### Renaming and Cloning + +```java +db.renameColumnFamily("users", "accounts"); + +// a point-in-time copy; later writes to the source do not appear in it +db.cloneColumnFamily("accounts", "accounts_snapshot"); +``` + +Both flush the whole database memtable first — it is shared, so this is not just +that family's data — and claim the family against the compaction scheduler. Both +can report `ERR_LOCKED` if the family stays under compaction for the whole +quiesce window. + +### Working with Transactions + +Every read and write goes through a transaction. + +#### Writing Data + +```java +try (Transaction txn = db.beginTransaction()) { + txn.put(cf, "key".getBytes(), "value".getBytes()); + txn.commit(); +} +``` + +An empty value is allowed and stores the key present carrying nothing. That is a +distinct state from an absence: a read returns it with a zero length rather than +reporting the key missing. + +#### Writing with TTL + +The TTL is **how long the entry lives, in seconds from now**. Zero or negative +never expires. The engine converts it to an absolute deadline once, at the +boundary, so a long recovery cannot extend an entry's life. + +```java +try (Transaction txn = db.beginTransaction()) { + txn.put(cf, "session".getBytes(), "token".getBytes(), 3600); // one hour + txn.commit(); +} +``` + +> **Changed in 1.0.0.** In 0.8.x the TTL argument was an absolute Unix timestamp +> and `-1` meant no expiry. It is now a relative duration in seconds. + +#### Reading Data + +```java +try (Transaction txn = db.beginTransaction()) { + byte[] value = txn.get(cf, "key".getBytes()); + if (value != null) { + System.out.println(new String(value)); + } + txn.rollback(); +} +``` + +An absent key returns `null` rather than throwing. + +For an existence probe that should not pollute the conflict footprint — a +primary-key uniqueness check, say — use the non-tracking forms: + +```java +byte[] value = txn.getNoTrack(cf, key); // reads without recording the read +boolean present = txn.contains(cf, key); // existence only, no value read +``` + +#### Deleting Data + +```java +try (Transaction txn = db.beginTransaction()) { + txn.delete(cf, "key".getBytes()); + txn.commit(); +} +``` + +#### Single-Delete + +A tombstone the caller promises supersedes at most one put, so the two can be +reaped together at compaction. + +**Caller contract:** between any two single-deletes on the same key, and from the +start of the key's history to its first single-delete, the key has been put at +most once. The engine cannot verify this; violating it can leave older puts +visible after the single-delete. When in doubt, prefer `delete`. + +```java +txn.singleDelete(cf, "write-once-key".getBytes()); +``` + +#### Range and Prefix Deletes + +Both cost one entry however many keys they cover, and both delete keys written +*before* them as well as keys written after — so neither is the same as deleting +the keys that happen to be there when you call it. A write to a covered key +survives when it is newer: buffered after it in the same transaction, or +committed at a later sequence. + +```java +try (Transaction txn = db.beginTransaction()) { + // [lo, hi) — lower bound inclusive, upper bound exclusive + txn.deleteRange(cf, "k05".getBytes(), "k10".getBytes()); + + // null upper bound runs to the end of the family + txn.deleteRange(cf, "archive:".getBytes(), null); + + // every key under a prefix + txn.deletePrefix(cf, "session:".getBytes()); + + txn.commit(); +} +``` + +Each bound is at most `Transaction.MAX_RANGE_BOUND_SIZE` (256) bytes. An +interval delete holds its bounds in a fixed slot for the length of its commit, +which is what stops a concurrent write landing inside the range; a range too wide +to name in one call is expressed as several. + +The delete is O(1) to write but not to reclaim: the keys it covers stay on disk +until a compaction rewrites the range, and reads pay a bounded interval lookup +until then. + +At `SNAPSHOT` isolation and above the commit is refused with `ERR_CONFLICT` when +any key in the range was written after the transaction drew its snapshot. + +#### Transaction Rollback + +```java +try (Transaction txn = db.beginTransaction()) { + txn.put(cf, "key".getBytes(), "value".getBytes()); + txn.rollback(); // nothing is written +} +``` + +#### Multi-Operation and Multi-Column-Family Transactions + +A transaction spans column families, and the whole batch commits atomically. + +```java +try (Transaction txn = db.beginTransaction()) { + txn.put(users, "u1".getBytes(), "alice".getBytes()); + txn.put(orders, "o1".getBytes(), "u1:widget".getBytes()); + txn.delete(sessions, "s1".getBytes()); + txn.commit(); +} +``` + +### Transaction Isolation Levels + +| Level | Behaviour | +|---|---| +| `READ_UNCOMMITTED` | Every version is visible, including sequences still in progress. | +| `READ_COMMITTED` | The newest committed version, with the ceiling re-read on every operation. | +| `REPEATABLE_READ` | The ceiling is frozen at begin; a commit validates that every key it read still holds the version it read. | +| `SNAPSHOT` | Frozen ceiling, and a commit reserves each key it writes on a first-committer-wins basis. | +| `SERIALIZABLE` | Both of the above checks, plus the one that catches write skew. | + +```java +try (Transaction txn = db.beginTransaction(IsolationLevel.SERIALIZABLE)) { + // ... + txn.commit(); +} + +// or take the column family's configured default +try (Transaction txn = db.beginTransaction(cf)) { + // ... + txn.commit(); +} +``` + +You can ask which committed versions a transaction can see: + +```java +long ceiling = txn.getReadSnapshot(); +``` + +The engine's sequence is unsigned 64-bit, so a ceiling of `UINT64_MAX` — what +read-uncommitted filters at — arrives in Java as `-1`. Compare sequences with +`Long.compareUnsigned`. + +### Savepoints + +```java +try (Transaction txn = db.beginTransaction()) { + txn.put(cf, "a".getBytes(), "1".getBytes()); + + txn.savepoint("checkpoint"); + txn.put(cf, "b".getBytes(), "2".getBytes()); + txn.rollbackToSavepoint("checkpoint"); // "b" is discarded, "a" is kept + + txn.releaseSavepoint("checkpoint"); // or release without rolling back + txn.commit(); +} +``` + +Rolling back to or releasing a name that was never marked fails with +`ERR_NOT_FOUND`. + +### Transaction Reset + +Reuse a resolved transaction rather than freeing and reallocating one in a hot +loop. + +```java +try (Transaction txn = db.beginTransaction()) { + for (int batch = 0; batch < 1000; batch++) { + txn.put(cf, key(batch), value(batch)); + txn.commit(); + txn.reset(IsolationLevel.READ_COMMITTED); + } +} +``` + +### Timeouts and Aborts + +An abandoned transaction holds its snapshot and its write reservations, which +keeps the reclamation floor down and stops compaction dropping old versions. +Bound one that may be left unresolved. + +```java +Config config = Config.builder("/path/to/db") + .txnTimeoutSeconds(300) // database-wide default + .build(); + +try (Transaction txn = db.beginTransaction()) { + txn.setTimeout(30); // override for this transaction + // ... + txn.setTimeout(0); // clear it again + txn.commit(); +} +``` + +The transaction is not aborted in the background when the deadline passes: the +next operation on it notices, aborts it, and fails with `ERR_TXN_EXPIRED`. The +engine ages transactions against a clock refreshed once a second. + +`requestAbort()` is the one method that may be called on a transaction owned by +a **different thread**. It exists for a caller that is itself the authority on +whether a transaction may proceed — a replication plugin whose cluster has +certified against it, say. + +```java +txn.requestAbort(); // stores a flag and returns; nothing else changes here +``` + +The thread running the transaction observes the flag when it next enters an +operation and fails with `ERR_TXN_ABORTED` — a code of its own, so a caller can +tell an outside ruling from the engine's own `ERR_CONFLICT`. A transaction that +has already prepared is unaffected: it has voted, and only the coordinator's +decision may resolve it. + +### Two-Phase Commit + +`prepare` runs the same conflict checks as `commit` and durably logs the write +batch under a transaction id, but leaves the writes invisible and unapplied so a +coordinator can gather votes before deciding. + +```java +try (Transaction txn = db.beginTransaction()) { + txn.put(cf, "key".getBytes(), "value".getBytes()); + + txn.prepare("xid-42".getBytes()); + assert txn.state() == TransactionState.PREPARED; + + // ... the coordinator gathers votes from every participant ... + + txn.commitPrepared(); // or txn.rollbackPrepared() +} +``` + +`TransactionState` reports `ACTIVE`, `PREPARED`, `COMMITTED`, or `ABORTED`. A +transient I/O failure in phase two leaves the transaction prepared so the +coordinator can retry. + +After a restart, transactions that were prepared and never decided are listed +for the coordinator to finish: + +```java +for (PreparedTransaction prepared : db.recoverPrepared()) { + byte[] xid = prepared.getXid(); + try (Transaction txn = prepared.getTransaction()) { + if (coordinatorSaysCommit(xid)) { + txn.commitPrepared(); + } else { + txn.rollbackPrepared(); + } + } +} +``` + +One that was decided in the log is settled during open and never appears here. + +### Snapshots and Point-in-Time Reads + +A snapshot names the database as it stands now so it can be read again later. It +holds the reclamation floor at its own sequence for as long as it lives, which +is what keeps the versions it resolves to from being compacted away — and is +also its cost. Release it as soon as the point in time is no longer wanted. + +```java +try (Snapshot snapshot = db.createSnapshot()) { + System.out.println("reading as of " + snapshot.getSeq()); + + // writes continue on the live database + try (Transaction w = db.beginTransaction()) { + w.put(cf, "key".getBytes(), "new".getBytes()); + w.commit(); + } + + // and this still sees the old value + try (Transaction r = db.beginTransactionAtSnapshot(snapshot)) { + byte[] asOfSnapshot = r.get(cf, "key".getBytes()); + r.rollback(); + } +} +``` + +Every transaction opened against a snapshot must be freed before the snapshot is +released, since they read versions only it keeps alive. + +You can also read at an explicit sequence — one read back from +`getReadSnapshot()`, or recorded elsewhere: + +```java +try (Transaction r = db.beginTransactionAtSeq(seq)) { + // ... + r.rollback(); +} +``` + +This refuses rather than approximates. A sequence is readable only while +something holds the reclamation floor under it: an open transaction, or a +snapshot taken in advance. Once a collection has run past that sequence, the call +fails with `ERR_TOO_OLD` and reads nothing. + +```java +long floor = db.getOldestReadableSeq(); // the oldest sequence still accepted +``` + +### Iterating Over Data + +Iterators are created from a transaction and read at its snapshot. Close them +before freeing the transaction. + +`isValid()` is the single way to ask whether the cursor is on an entry — every +positioning call leaves the iterator invalid rather than throwing when the merged +stream has nothing where it was asked to stand. + +#### Forward Iteration + +```java +try (Transaction txn = db.beginTransaction(); + TidesDBIterator it = txn.newIterator(cf)) { + it.seekToFirst(); + while (it.isValid()) { + System.out.println(new String(it.key()) + " = " + new String(it.value())); + it.next(); + } + txn.rollback(); +} +``` + +Keys are ordered byte-wise (`memcmp`). A caller who wants a different order +encodes keys to be memcomparable — big-endian integers, sign-flipped signed +integers, inverted bytes for descending — since the engine carries no pluggable +comparator. + +#### Backward Iteration + +```java +it.seekToLast(); +while (it.isValid()) { + process(it.key(), it.value()); + it.prev(); +} +``` + +#### Combined Key-Value Retrieval + +One JNI crossing instead of two: + +```java +KeyValue kv = it.keyValue(); +byte[] key = kv.getKey(); +byte[] value = kv.getValue(); +``` + +#### Seeking + +```java +it.seek("k100".getBytes()); // first key >= the target +it.seekForPrev("k100".getBytes()); // last key <= the target +``` + +#### Prefix Scans + +```java +byte[] prefix = "user:".getBytes(); +it.seek(prefix); +while (it.isValid() && startsWith(it.key(), prefix)) { + process(it.key(), it.value()); + it.next(); +} +``` + +#### Range Iterators + +An iterator holds one open cursor per SSTable that could answer it, descends +each of them on every seek and compares each on every step. Telling it the range +up front lets it leave out the SSTables whose own key range cannot meet it, so a +scan of a narrow band costs what that band costs rather than what the whole +column family costs. On a family holding hundreds of SSTables that is the +difference between a scan that competes for the whole store and one that does +not. + +```java +try (Transaction txn = db.beginTransaction(); + TidesDBIterator it = txn.newRangeIterator(cf, "k020".getBytes(), "k030".getBytes())) { + it.seek("k020".getBytes()); + while (it.isValid() && compare(it.key(), "k030".getBytes()) < 0) { + process(it.key(), it.value()); + it.next(); + } + txn.rollback(); +} +``` + +The range is a promise about what will be read, **not a fence the iterator +enforces**. Results are defined only inside it, because the SSTables that could +answer outside it were never opened: seeking or stepping past either end may +report absent a key that exists. Use `newIterator` for a scan whose extent is +not known in advance. + +### Encoding Pipelines + +A column family applies an ordered chain of encodings to its btree key-log +nodes, undone in reverse on read. The ids are recorded in the SSTable footer, so +a reader rebuilds the same chain from the file. + +```java +// the common case: a single codec +ColumnFamilyConfig cfg = ColumnFamilyConfig.builder() + .compression(CompressionAlgorithm.ZSTD) + .build(); + +// or a chain, applied in order +ColumnFamilyConfig chained = ColumnFamilyConfig.builder() + .encodingPipeline(CompressionAlgorithm.LZ4, CompressionAlgorithm.ZSTD) + .build(); + +// or raw ids, for an encoding the enum does not name +ColumnFamilyConfig raw = ColumnFamilyConfig.builder() + .encodingPipelineIds(2, 3) + .build(); +``` + +`compression(CompressionAlgorithm.NONE)` clears the pipeline so data is stored +verbatim. At most `ColumnFamilyConfig.MAX_ENCODING_PIPELINE` (8) entries are +allowed. + +Every algorithm is always named by the enum, but a backend is only linked in +when its build option was set. Ask before choosing one, rather than discovering +it when a node fails to decode: + +```java +if (TidesDB.isCompressionAvailable(CompressionAlgorithm.ZSTD)) { + builder.compression(CompressionAlgorithm.ZSTD); +} +``` + +`CompressionAlgorithm.NONE` is always available. + +### Commit Hook (Change Data Capture) + +The hook fires synchronously after every transaction commit on that family, +receiving the full batch atomically. Keep the callback fast to avoid stalling +writers. + +```java +cf.setCommitHook((ops, commitSeq) -> { + for (CommitOp op : ops) { + if (op.isDelete()) { + replicateDelete(op.getKey()); + } else { + replicatePut(op.getKey(), op.getValue(), op.getTtl()); + } + } + return 0; // non-zero is logged as a warning +}); + +// ... later +cf.clearCommitHook(); +``` + +`CommitOp.getTtl()` is the **absolute expiry** the engine stored, not the +lifetime in seconds that `put` was given, so a hook forwarding the write +elsewhere reproduces the same expiry instant rather than restarting the clock. It +is `-1` when the entry never expires. + +The hook fires after the WAL write, memtable apply, and commit-status marking +complete. A hook failure is logged but does not roll back the commit — the data +is already durable. An exception thrown out of the callback is caught and treated +as a failure. + +Hooks are runtime-only and not persisted. After a restart, re-register them. +Setting a hook while one is installed replaces it; the old one is retired only +once every callback already inside it has returned. + +### Maintenance + +The memtable and write-ahead log are shared across every column family, so the +operations that act on them live on `TidesDB`. Compaction is per-family and lives +on `ColumnFamily`. + +```java +db.flushMemtable(); // rotate and flush, waiting a bounded time for the queue to drain +db.isFlushing(); // whether an immutable is queued or flushing +db.syncWal(); // force an fsync of the write-ahead log +db.checkpoint(); // durability barrier: flush, then force vlog, WAL and manifest to disk + +cf.compact(); // one forced pass over the family +cf.compactRange(startKey, endKey); // every SSTable overlapping [start, end) +cf.isCompacting(); // whether a merge is in progress +``` + +A `null` or empty endpoint on `compactRange` is unbounded on that side; both +unbounded is rejected in favour of `compact()`. + +Every maintenance call can report `ERR_LOCKED`, and it always means the same +thing: the work was not done because something else held what it needed, and +asking again later is the remedy. It is never data loss and never a corrupt +database. + +### Backup + +```java +db.backup("/path/to/backup"); +``` + +Flushes the memtable, then copies the manifest, the shared value log, and every +SSTable it references at a single manifest snapshot while compaction is held off, +so the copy references no file a merge could delete mid-copy. The result is +directly openable: + +```java +try (TidesDB restored = TidesDB.open(Config.builder("/path/to/backup").build())) { + // ... +} +``` + +### Updating Runtime Configuration + +Every field of a column family configuration may change at runtime, since +byte-wise key ordering keeps all SSTables mergeable. The family name and id are +preserved; a rename is separate. + +```java +ColumnFamilyConfig updated = ColumnFamilyConfig.builder() + .compression(CompressionAlgorithm.ZSTD) + .enableBloomFilter(true) + .bloomFpr(0.05) + .l1FileCountTrigger(8) + .build(); + +cf.updateRuntimeConfig(updated, true); // true persists it in the manifest +``` + +Pass `false` to apply the change in memory only. + +## Statistics + +### Column Family Statistics + +```java +CfStats stats = cf.getStats(); + +System.out.println("levels: " + stats.getNumLevels()); +System.out.println("keys on disk: " + stats.getTotalKeys()); +System.out.println("keys still in memory: " + stats.getUnflushedKeyCount()); +System.out.println("on-disk size: " + stats.getTotalDataSize()); +System.out.println("read amplification: " + stats.getReadAmp()); +System.out.println("tombstone ratio: " + stats.getTombstoneRatio()); +System.out.println("filter memory: " + stats.getFilterResidentBytes()); + +// per-level arrays, indexed by level - 1, valid for the first getNumLevels() entries +long[] sizes = stats.getLevelSizes(); +int[] tables = stats.getLevelNumSstables(); +long[] keys = stats.getLevelKeyCounts(); +long[] tombstones = stats.getLevelTombstoneCounts(); + +// the configuration as the engine holds it, including the persisted name +ColumnFamilyConfig live = stats.getConfig(); +``` + +`getTotalKeys()` plus `getUnflushedKeyCount()` is the live logical key count +including what is still in memory. `getTotalDataSize()` is the family's own +on-disk size, the sum of its key logs; what spilled to the shared value log is +database-level and reported as `DbStats.getVlogFileSize()`. + +`getWalBytesWritten()` here is an attribution rather than a measurement — the +batch header and block framing belong to no single family, so these do not sum to +what the log wrote. The database-level figure is the measured one. + +### Cardinality Estimate + +```java +long distinct = cf.estimateCardinality(); +``` + +### Database Statistics + +```java +DbStats stats = db.getDbStats(); + +System.out.println("column families: " + stats.getNumColumnFamilies()); +System.out.println("L0 queue depth: " + stats.getImmutableMemtableCount()); +System.out.println("MVCC clock: " + stats.getGlobalSeq()); +System.out.println("gc floor: " + stats.getMinSnapshotSeq()); +System.out.println("live transactions: " + stats.getActiveTxnCount()); +System.out.println("memtable bytes: " + stats.getMemtableBytes()); + +// write amplification terms +System.out.println("user bytes: " + stats.getUserBytesWritten()); +System.out.println("wal bytes: " + stats.getWalBytesWritten()); +System.out.println("flush bytes: " + stats.getFlushBytesWritten()); +System.out.println("compaction bytes: " + stats.getCompactionBytesWritten()); +System.out.println("vlog bytes: " + stats.getVlogBytesWritten()); + +// value log space +System.out.println("vlog live: " + stats.getVlogLiveBytes()); +System.out.println("vlog dead: " + stats.getVlogDeadBytes()); +System.out.println("drainable segments: " + stats.getVlogSegmentsDrainable()); + +// write admission +System.out.println("throttled: " + stats.getWritesThrottled()); +System.out.println("blocked: " + stats.getWritesBlocked()); +System.out.println("stall time (us): " + stats.getWriteStallUs()); +System.out.println("ceiling hits: " + stats.getWriteStallCeilingHits()); +``` + +`getMinSnapshotSeq()` is unsigned: when nothing holds the floor it sits at +`UINT64_MAX` and reads as `-1` in Java. + +`getVlogLiveBytes()` is the figure space amplification is against — a store can +hold many gigabytes of values no tree can reach, and only this tells them apart +from the ones still worth keeping. A `getVlogSegmentsDrainable()` that stays high +is reclamation falling behind. + +Any `getWriteStallCeilingHits()` at all means flush did not keep up with +ingestion. + +### Cache Statistics + +```java +CacheStats cache = db.getCacheStats(); +System.out.println(cache.getHitRate()); +System.out.println(cache.getTotalBytes() + " over " + cache.getNumPartitions() + " shards"); +``` + +### Stall Statistics + +Where writers have been made to wait. A write latency tail is answerable from +this alone: compare each reason's maximum against the tail you measured, and its +total against the others, rather than attaching a debugger to a stalled commit. + +```java +StallStats stalls = db.getStallStats(); + +for (StallReason reason : StallReason.values()) { + StallStat stat = stalls.get(reason); + System.out.printf("%-16s count=%d total=%dus max=%dus%n", + reason.getNativeName(), stat.getCount(), stat.getTotalUs(), stat.getMaxUs()); +} +System.out.println("all waiting: " + stalls.getTotalUs() + "us"); +``` + +| Reason | Meaning | +|---|---| +| `WAL_APPEND` | Waiting on the write-ahead log, for staging-ring space or for the record to reach the file. | +| `ROTATE_LOCK` | Waiting to take the rotation lock; another committer was rotating. | +| `ROTATE_WORK` | Performing the rotation, which this thread pays on everyone's behalf. | +| `ADMISSION` | Held by write admission because the unflushed backlog was too deep. | +| `MANIFEST_COMMIT` | Inside a manifest commit, which every flush install, compaction install and DDL serialises through. | + +### I/O Statistics + +The other half of the stall statistics: those say writers waited on the log, these +say whether the device was the reason. Bytes over total time is the throughput a +class actually achieved — compare it against what the storage can sustain, +because a saturated device and a stalled engine look identical from the +application. + +```java +IoStats io = db.getIoStats(); + +for (IoClass cls : IoClass.values()) { + IoStat stat = io.get(cls); + System.out.printf("%-8s ops=%d bytes=%d %.1f MB/s max=%dus%n", + cls.getNativeName(), stat.getOps(), stat.getBytes(), + stat.getBytesPerSecond() / 1e6, stat.getMaxUs()); +} +``` + +Only handles the engine opens through its descriptor manager are counted, which +is every key log and every write-ahead log. The value log and the manifest are +not, so this measures the two classes that compete for the device under load +rather than every byte the database writes. + +### Encoding Statistics + +What each encoding chain achieved, reported per chain rather than per column +family — a family can change its codec, and compaction rewrites data under +whichever pipeline is merging it, so a single figure per family would average +across settings that no longer apply. + +```java +for (EncodingStats e : db.getKlogEncodingStats()) { + System.out.printf("ids=%s %d -> %d bytes (%.2fx) over %d sstables%n", + Arrays.toString(e.getIds()), e.getLogicalBytes(), e.getStoredBytes(), + e.getRatio(), e.getItemCount()); +} + +for (EncodingStats e : db.getVlogEncodingStats()) { + // same shape; itemCount counts values rather than sstables +} +``` + +At most `EncodingStats.MAX_CHAINS` (16) chains are reported. + +### Range Statistics + +What a query planner needs to know about a key range, both figures taken from one +layout snapshot so they describe the same instant. + +```java +RangeStats range = cf.rangeStats("k000".getBytes(), "k500".getBytes()); + +System.out.println("sorted runs a scan would merge: " + range.getSstablesOverlapping()); +System.out.println("live keys: " + range.getEstimatedKeys()); +System.out.println("counted exactly: " + range.isKeysExact()); +``` + +The count is memtable-aware, so a range whose data has not been flushed yet +reports a real cardinality rather than an SSTable overlap count. A range small +enough to walk is counted exactly and says so; a wider one is estimated from +SSTable metadata without walking, so the call stays cheap enough for plan time +whatever the range covers. + +## Configuration Options + +### Database Configuration + +A field left at zero is resolved to the engine's default for the thread counts, +`blockCacheSize`, `maxOpenSSTables`, `vlogSegmentSize`, +`memtableWriteBufferSize`, `memtableSkipListMaxLevel` and +`memtableSkipListProbability`. The rest are used as given, `memtableSyncMode` +included, where zero is the meaningful value `SYNC_NONE`. + +| Option | Type | Description | +|---|---|---| +| `dbPath` | `String` | Path to the database directory. | +| `numFlushThreads` | `int` | Flush worker threads. | +| `numCompactionThreads` | `int` | Compaction worker threads. | +| `logLevel` | `LogLevel` | Minimum severity to emit. | +| `blockCacheSize` | `long` | Bytes of database-level block cache for hot SSTable blocks. | +| `maxOpenSSTables` | `long` | Concurrently open SSTable handles; lowered at open to fit the process ceiling. | +| `logToFile` | `boolean` | Write the log to `LOG` inside the database directory rather than stderr. The sink is process-wide. | +| `logTruncationAt` | `long` | Bytes past which the log file is truncated and reopened; 0 for never. | +| `memtableWriteBufferSize` | `long` | Memory the active memtable may occupy before rotation. A budget, not a promise about flush size. | +| `memtableSkipListMaxLevel` | `int` | Skip list max level for the memtable. | +| `memtableSkipListProbability` | `float` | Skip list level probability, in [0.0, 1.0]. | +| `memtableSyncMode` | `SyncMode` | Durability mode for the write-ahead log. | +| `memtableSyncIntervalUs` | `long` | Fsync interval for `SYNC_INTERVAL`, in microseconds. | +| `valueSeparationThreshold` | `long` | Values at or above this size go to the shared value log. | +| `vlogSegmentSize` | `long` | Size at which the value log seals a segment. | +| `memtableL0QueueStallThreshold` | `int` | Immutable-queue depth at which writes stall; 0 never stalls. | +| `memtableIdleFlushSeconds` | `int` | How long the memtable may sit unwritten before the engine rotates it; 0 never does. | +| `txnTimeoutSeconds` | `long` | How long a transaction may stay active; 0 for no timeout. | + +`memtableWriteBufferSize` is a memory budget rather than a promise about the size +of what a rotation flushes: an entry costs its key and value plus about a hundred +bytes of skip list node, pointer arrays and version struct. That overhead is +fixed per entry, so it is most of an entry holding a short value and almost none +of one holding a large one. + +Keep `valueSeparationThreshold` at or under a quarter of a family's +`btreeKlogBlockSize` — an inlined value approaching the node size leaves a node +holding one entry and spends the btree fan-out that makes a lookup cheap. The +pairing is advisory, and a config that breaks it only logs a warning. + +Left at 0, `memtableL0QueueStallThreshold` leaves the queue unbounded and a +writer outrunning the flush threads is never paced, so it is a value to set +deliberately rather than leave. + +### Column Family Configuration + +| Option | Type | Description | +|---|---|---| +| `levelSizeRatio` | `long` | Target size ratio between successive levels. | +| `minLevels` | `int` | Floor on the level count. | +| `dividingLevelOffset` | `int` | How far above the largest level the dividing level sits. | +| `keepValuesInline` | `boolean` | Hold every value in the key log whatever its size, ignoring the database threshold. | +| `btreeKlogBlockSize` | `long` | Target size of a btree key-log node; 0 leaves the choice to the btree. | +| `encodingPipeline` | `int[]` | Encoding ids applied in order, at most 8. | +| `enableBloomFilter` | `boolean` | Build a partition-range filter for point-get pruning. | +| `bloomFpr` | `double` | Target false-positive rate when the filter is enabled. | +| `defaultIsolationLevel` | `IsolationLevel` | Isolation for a transaction opened without an explicit level. | +| `l1FileCountTrigger` | `int` | L1 SSTable count that triggers compaction. | +| `tombstoneDensityTrigger` | `double` | Density above which an SSTable escalates compaction; 0 disables. | +| `tombstoneDensityMinEntries` | `long` | Minimum entry count to be judged by density; 0 imposes no minimum. | + +A separated value costs a scan one value-log read per row, so a family scanned +far more than it is merged can be worth `keepValuesInline` even though its values +are large. The cost is the one the threshold exists to avoid: compaction rewrites +those bytes on every merge. + +### Compression Algorithms + +| Constant | Encoding id | +|---|---| +| `CompressionAlgorithm.NONE` | 0 | +| `CompressionAlgorithm.SNAPPY` | 1 | +| `CompressionAlgorithm.LZ4` | 2 | +| `CompressionAlgorithm.ZSTD` | 3 | +| `CompressionAlgorithm.LZ4_FAST` | 4 | + +### Sync Modes + +| Constant | Behaviour | +|---|---| +| `SyncMode.SYNC_NONE` | Nothing at commit. Fastest; a crash of either kind may lose commits. | +| `SyncMode.SYNC_FULL` | Fsync every commit. Slowest; no commit loss on crash. | +| `SyncMode.SYNC_INTERVAL` | Fsync on a background interval. Bounded loss window. | + +A clean close loses nothing in any mode. + +### Log Levels + +| Constant | Value | +|---|---| +| `LogLevel.NONE` | 0 | +| `LogLevel.TRACE` | 1 | +| `LogLevel.INFO` | 2 | +| `LogLevel.WARN` | 3 | +| `LogLevel.ERROR` | 4 | + +A larger value is more severe, so a higher threshold emits fewer lines. + +## Error Handling + +`TidesDBException` carries the native result code. `getErrorCode()` returns one +of the constants below and `getErrorMessage()` describes it without a native +call; `TidesDB.strerror(code)` asks the library for its own description. + +| Constant | Value | Meaning | +|---|---|---| +| `ERR_SUCCESS` | 0 | Success. | +| `ERR_MEMORY` | -1 | Memory allocation failed. | +| `ERR_INVALID_ARGS` | -2 | Invalid arguments. | +| `ERR_NOT_FOUND` | -3 | Not found. | +| `ERR_IO` | -4 | I/O error. | +| `ERR_CORRUPTION` | -5 | Data corruption. | +| `ERR_EXISTS` | -6 | Already exists. | +| `ERR_CONFLICT` | -7 | The engine's first-committer-wins verdict. | +| `ERR_TOO_LARGE` | -8 | A value does not fit the space that must hold it. | +| `ERR_MEMORY_LIMIT` | -9 | Memory limit exceeded. | +| `ERR_INVALID_DB` | -10 | Invalid handle, or the database is closing. | +| `ERR_UNKNOWN` | -11 | Unknown error. | +| `ERR_LOCKED` | -12 | Transient contention. Retry. | +| `ERR_READONLY` | -13 | The database is read-only. | +| `ERR_TXN_EXPIRED` | -14 | The transaction's timeout passed. | +| `ERR_NO_SPACE` | -15 | The device or filesystem is out of space. | +| `ERR_TXN_ABORTED` | -16 | Aborted from outside through `requestAbort()`. | +| `ERR_TOO_OLD` | -17 | The point in time asked for is no longer reconstructable. | + +### Retrying on Contention + +`ERR_LOCKED` is transient contention, never a failed operation and never data +loss: something else held what the call needed, nothing was written, and the +remedy is always to try again. It is not confined to operations that take a +family exclusively — a read has to open SSTables and walk sources a compaction +may be moving, so any get, existence check or iterator step can report it too. +Treat it as retry, never as absence and never as an error to surface. + +```java +for (int attempt = 0; ; attempt++) { + try (Transaction txn = db.beginTransaction()) { + byte[] value = txn.get(cf, key); + txn.rollback(); + return value; + } catch (TidesDBException e) { + if (!e.isRetryable() || attempt >= 10) { + throw e; + } + Thread.sleep(1L << attempt); + } +} +``` + +`ERR_NO_SPACE` is distinct from `ERR_IO` because it says what to do about it: +the data is intact and the operation succeeds once space is freed, where an I/O +error carries no such promise. + +## Migrating from 0.8.x + +Version 1.0.0 targets TidesDB 10.x and is a breaking change throughout. The +on-disk format is also new: **a 9.x database is not readable by 10.x**, and moves +across by dumping and reloading through the API. + +### Removed + +| Removed | Why / what to use instead | +|---|---| +| Object store mode, S3 connector, `ObjectStoreConfig`, `S3Config` | Not part of the 10.x public API. | +| Replica mode, `promoteToPrimary()` | Removed with object store mode. | +| `registerComparator()` | Keys are ordered byte-wise; encode keys to be memcomparable instead. | +| `purge()`, `ColumnFamily.purge()` | Use `flushMemtable()` and `compact()`. | +| `cancelBackgroundWork()` | Removed; `close()` handles shutdown. | +| `deleteColumnFamily(ColumnFamily)` | Use `dropColumnFamily(String)`. | +| `ColumnFamilyConfig` INI save/load | Removed; configure in code. | +| `rangeCost()` | Replaced by `rangeStats()`, which reports real figures rather than an opaque score. | +| `Stats` | Replaced by `CfStats`. | + +### Moved + +The memtable and write-ahead log are now database-level, so the operations on +them moved from `ColumnFamily` to `TidesDB`: + +| 0.8.x | 1.0.0 | +|---|---| +| `cf.flushMemtable()` | `db.flushMemtable()` | +| `cf.isFlushing()` | `db.isFlushing()` | +| `cf.syncWal()` | `db.syncWal()` | +| `cf.rangeCost(a, b)` | `cf.rangeStats(a, b)` | +| `cf.getStats()` returning `Stats` | `cf.getStats()` returning `CfStats` | +| `db.checkpoint(dir)` | `db.checkpoint()` — a durability barrier in the live database, not a directory copy | + +Per-column-family memtable, WAL, sync and skip-list settings moved from +`ColumnFamilyConfig` to `Config`, and the `unifiedMemtable*` options are gone +because the memtable is always shared now. + +### Changed + +- **TTL is now relative.** `txn.put(cf, key, value, ttl)` takes seconds from now; + zero or negative never expires. It was an absolute Unix timestamp with `-1` + for no expiry. +- **Compression constants renamed** to match the C enum: `NO_COMPRESSION` → + `NONE`, `SNAPPY_COMPRESSION` → `SNAPPY`, `LZ4_COMPRESSION` → `LZ4`, + `ZSTD_COMPRESSION` → `ZSTD`, `LZ4_FAST_COMPRESSION` → `LZ4_FAST`. They are set + through `compression(...)` or `encodingPipeline(...)` rather than a single + `compressionAlgorithm` field. +- **`LogLevel` values changed** to `NONE(0)`, `TRACE(1)`, `INFO(2)`, `WARN(3)`, + `ERROR(4)`. `DEBUG` and `FATAL` are gone. +- **Iterator positioning no longer throws at the end of a range.** `seek`, + `seekForPrev`, `seekToFirst` and `seekToLast` leave the iterator invalid + instead, matching `next` and `prev`. Check `isValid()`. +- **New error codes** `ERR_READONLY`, `ERR_TXN_EXPIRED`, `ERR_NO_SPACE`, + `ERR_TXN_ABORTED` and `ERR_TOO_OLD`. + +### Added + +Snapshots and point-in-time reads, two-phase commit and prepared-transaction +recovery, range and prefix deletes, non-tracking reads and existence checks, +transaction timeouts and cross-thread aborts, range iterators, encoding +pipelines, cardinality estimates, and the stall, I/O, encoding and range +statistics families. + +## Testing + +```bash +# Run all tests +./mvnw test + +# Run a single test class +./mvnw test -Dtest=TidesDBTest + +# Point the JVM at a locally built JNI library +./mvnw test -Dtest.jvm.args="-Djava.library.path=/usr/local/lib" +``` + +## Building from Source + +```bash +# Clone the repository +git clone https://github.com/tidesdb/tidesdb-java.git +cd tidesdb-java + +# Build and install the JNI library +cd src/main/c +cmake -S . -B build +cmake --build build +sudo cmake --install build +cd ../../.. + +# Build the Java package +./mvnw package + +# Install to the local Maven repository +./mvnw install +``` diff --git a/doc/manual.json b/doc/manual.json new file mode 100644 index 0000000..406f00d --- /dev/null +++ b/doc/manual.json @@ -0,0 +1,19 @@ +{ + "$comment": "Table of contents for the TidesDB Java binding docs, read by the TidesDB website (rendered under /docs//bindings//). The binding version comes from this repo's release and is pinned per TidesDB major on the site — do not repeat it here. This currently points at the single reference page. To expand it like the core/TideSQL manuals, split the page into numbered part folders (e.g. 01-getting-started/, 02-guide/, 03-reference/) and list each part and chapter below; give each markdown file only 'title' and 'description' frontmatter — the website injects the slug.", + "title": "TidesDB Java", + "tidesdb": "10.0.0", + "parts": [ + { + "id": "reference", + "title": "API Reference", + "dir": "", + "chapters": [ + { + "file": "java.md", + "slug": "reference", + "title": "Java API Reference" + } + ] + } + ] +} diff --git a/pom.xml b/pom.xml index d8c6fdf..3d4c55d 100644 --- a/pom.xml +++ b/pom.xml @@ -6,11 +6,11 @@ com.tidesdb tidesdb-java - 0.8.3 + 1.0.0 jar TidesDB Java - Java bindings for TidesDB - A high-performance embedded key-value storage engine + Java bindings for TidesDB 10.x - A high-performance embedded key-value storage engine https://github.com/tidesdb/tidesdb-java diff --git a/src/main/c/com_tidesdb_TidesDB.c b/src/main/c/com_tidesdb_TidesDB.c index fec33e6..c2f910a 100644 --- a/src/main/c/com_tidesdb_TidesDB.c +++ b/src/main/c/com_tidesdb_TidesDB.c @@ -20,39 +20,24 @@ #include #include #include -#include #include -#ifndef _WIN32 -#include -#endif +/* the largest Java array length. jsize is a signed 32-bit int, so this is INT32_MAX -- written out + * rather than derived by shifting ~0, which sign-extends and yields -1 instead. */ #ifndef JSIZE_MAX -#define JSIZE_MAX (((jsize)~0) >> 1) +#define JSIZE_MAX ((jsize)0x7fffffff) #endif -/* Forward declaration */ -static jobject buildCfConfigObject(JNIEnv *env, const tidesdb_column_family_config_t *cfg); - -/* ABI-compatible overlay for tidesdb_objstore_t. - * The FFI header (db.h) defines this type as opaque. The real layout is published - * in tidesdb/objstore.h and is part of the stable library ABI. This overlay lets - * us call destroy(ctx) + free() on the connector failure path without pulling in - * the full objstore.h header (which conflicts with db.h). */ -struct _tidesdb_objstore_overlay -{ - int backend; - void *put; - void *get; - void *range_get; - void *delete_object; - void *exists; - void *list; - void *put_if; - void *head; - void (*destroy)(void *ctx); - void *ctx; -}; +/* the most encoding chains tidesdb_get_{klog,vlog}_encoding_stats will report */ +#define JNI_MAX_ENCODING_CHAINS 16 + +/* ===== error reporting ===== */ +/** + * Throws com.tidesdb.TidesDBException carrying the native result code. The + * message comes from tidesdb_strerror, which is a static literal that is never + * NULL, so every code describes itself without a local table to keep in sync. + */ static void throwTidesDBException(JNIEnv *env, int errorCode, const char *message) { jclass exClass = (*env)->FindClass(env, "com/tidesdb/TidesDBException"); @@ -65,13 +50,24 @@ static void throwTidesDBException(JNIEnv *env, int errorCode, const char *messag if (constructor == NULL) { (*env)->ThrowNew(env, exClass, message); + (*env)->DeleteLocalRef(env, exClass); return; } jstring jMessage = (*env)->NewStringUTF(env, message); jthrowable exception = (jthrowable)(*env)->NewObject(env, exClass, constructor, jMessage, errorCode); - (*env)->Throw(env, exception); + if (exception != NULL) + { + (*env)->Throw(env, exception); + } + (*env)->DeleteLocalRef(env, exClass); +} + +/** Throws for a non-success result code, using the library's own description. */ +static void throwResult(JNIEnv *env, int result) +{ + throwTidesDBException(env, result, tidesdb_strerror(result)); } static int jvm_exception_pending(JNIEnv *env) @@ -79,1388 +75,1431 @@ static int jvm_exception_pending(JNIEnv *env) return (*env)->ExceptionCheck(env) == JNI_TRUE; } -static const char *getErrorMessage(int code) -{ - switch (code) - { - case TDB_ERR_MEMORY: - return "memory allocation failed"; - case TDB_ERR_INVALID_ARGS: - return "invalid arguments"; - case TDB_ERR_NOT_FOUND: - return "not found"; - case TDB_ERR_IO: - return "I/O error"; - case TDB_ERR_CORRUPTION: - return "data corruption"; - case TDB_ERR_EXISTS: - return "already exists"; - case TDB_ERR_CONFLICT: - return "transaction conflict"; - case TDB_ERR_TOO_LARGE: - return "key or value too large"; - case TDB_ERR_MEMORY_LIMIT: - return "memory limit exceeded"; - case TDB_ERR_INVALID_DB: - return "invalid database handle"; - case TDB_ERR_LOCKED: - return "database is locked"; - case TDB_ERR_READONLY: - return "database is read-only"; - case TDB_ERR_BUSY: - return "resource is busy"; - default: - return "unknown error"; - } -} +/* ===== argument marshalling ===== */ -JNIEXPORT jlong JNICALL Java_com_tidesdb_TidesDB_nativeOpen( - JNIEnv *env, jclass cls, jstring dbPath, jint numFlushThreads, jint numCompactionThreads, - jint logLevel, jlong blockCacheSize, jlong maxOpenSSTables, jboolean logToFile, - jlong logTruncationAt, jlong maxMemoryUsage, jboolean unifiedMemtable, - jlong unifiedMemtableWriteBufferSize, jint unifiedMemtableSkipListMaxLevel, - jfloat unifiedMemtableSkipListProbability, jint unifiedMemtableSyncMode, - jlong unifiedMemtableSyncIntervalUs, jstring objectStoreFsPath, jstring oscLocalCachePath, - jlong oscLocalCacheMaxBytes, jboolean oscCacheOnRead, jboolean oscCacheOnWrite, - jint oscMaxConcurrentUploads, jint oscMaxConcurrentDownloads, jlong oscMultipartThreshold, - jlong oscMultipartPartSize, jboolean oscSyncManifestToObject, jboolean oscReplicateWal, - jboolean oscWalUploadSync, jlong oscWalSyncThresholdBytes, jboolean oscWalSyncOnCommit, - jboolean oscReplicaMode, jlong oscReplicaSyncIntervalUs, jboolean oscReplicaReplayWal, - jint maxConcurrentFlushes, jboolean finishCompactionsOnClose, jlong objStoreHandle) +/** + * A borrowed view of a jbyteArray. A NULL array yields a NULL pointer and a zero + * length, which is what every optional bound in the API wants. + */ +typedef struct { - const char *path = (*env)->GetStringUTFChars(env, dbPath, NULL); - if (path == NULL) + jbyteArray array; + jbyte *data; + jsize length; +} jni_bytes_t; + +/** + * Acquires the elements of a byte array. Returns 0 on success, -1 if the JVM + * could not hand over the buffer (in which case an OutOfMemoryError is already + * pending and the caller must return without throwing its own). + */ +static int acquireBytes(JNIEnv *env, jbyteArray array, jni_bytes_t *out) +{ + out->array = array; + out->data = NULL; + out->length = 0; + + if (array == NULL) { - throwTidesDBException(env, TDB_ERR_MEMORY, "Failed to get database path"); return 0; } - /* object store connector: a prebuilt connector handle (e.g. S3, created via - * nativeObjstoreS3Create) takes precedence; otherwise fall back to the filesystem - * connector built from objectStoreFsPath. */ - tidesdb_objstore_t *obj_store = NULL; - const char *fs_path = NULL; - if (objStoreHandle != 0) - { - obj_store = (tidesdb_objstore_t *)(uintptr_t)objStoreHandle; - } - else if (objectStoreFsPath != NULL) + out->length = (*env)->GetArrayLength(env, array); + out->data = (*env)->GetByteArrayElements(env, array, NULL); + if (out->data == NULL) { - fs_path = (*env)->GetStringUTFChars(env, objectStoreFsPath, NULL); - if (objectStoreFsPath != NULL && fs_path == NULL) - { - (*env)->ReleaseStringUTFChars(env, dbPath, path); - if (!jvm_exception_pending(env)) - throwTidesDBException(env, TDB_ERR_MEMORY, - "Failed to get object store filesystem path"); - return 0; - } - if (fs_path != NULL) + if (!jvm_exception_pending(env)) { - /* Validate that the path is an existing directory before creating the - * connector. tidesdb_objstore_fs_create silently succeeds on a regular - * file path, which would cause the database to open with a broken - * object store backend. */ - struct stat st; - if (stat(fs_path, &st) != 0 || !S_ISDIR(st.st_mode)) - { - (*env)->ReleaseStringUTFChars(env, objectStoreFsPath, fs_path); - (*env)->ReleaseStringUTFChars(env, dbPath, path); - if (!jvm_exception_pending(env)) - throwTidesDBException(env, TDB_ERR_IO, - "Failed to create filesystem object store connector"); - return 0; - } - obj_store = tidesdb_objstore_fs_create(fs_path); - if (obj_store == NULL) - { - (*env)->ReleaseStringUTFChars(env, objectStoreFsPath, fs_path); - (*env)->ReleaseStringUTFChars(env, dbPath, path); - if (!jvm_exception_pending(env)) - throwTidesDBException(env, TDB_ERR_IO, - "Failed to create filesystem object store connector"); - return 0; - } + throwTidesDBException(env, TDB_ERR_MEMORY, "Failed to acquire byte array"); } + return -1; } + return 0; +} - /* object store behavior config */ - const char *cache_path = NULL; - if (oscLocalCachePath != NULL) +/** Releases a view acquired by acquireBytes without copying anything back. */ +static void releaseBytes(JNIEnv *env, jni_bytes_t *bytes) +{ + if (bytes->data != NULL) { - cache_path = (*env)->GetStringUTFChars(env, oscLocalCachePath, NULL); - if (oscLocalCachePath != NULL && cache_path == NULL) - { - if (fs_path != NULL) (*env)->ReleaseStringUTFChars(env, objectStoreFsPath, fs_path); - (*env)->ReleaseStringUTFChars(env, dbPath, path); - if (!jvm_exception_pending(env)) - throwTidesDBException(env, TDB_ERR_MEMORY, "Failed to get local cache path"); - return 0; - } + (*env)->ReleaseByteArrayElements(env, bytes->array, bytes->data, JNI_ABORT); + bytes->data = NULL; } +} - tidesdb_objstore_config_t os_cfg = { - .local_cache_path = cache_path, - .local_cache_max_bytes = (size_t)oscLocalCacheMaxBytes, - .cache_on_read = oscCacheOnRead ? 1 : 0, - .cache_on_write = oscCacheOnWrite ? 1 : 0, - .max_concurrent_uploads = oscMaxConcurrentUploads, - .max_concurrent_downloads = oscMaxConcurrentDownloads, - .multipart_threshold = (size_t)oscMultipartThreshold, - .multipart_part_size = (size_t)oscMultipartPartSize, - .sync_manifest_to_object = oscSyncManifestToObject ? 1 : 0, - .replicate_wal = oscReplicateWal ? 1 : 0, - .wal_upload_sync = oscWalUploadSync ? 1 : 0, - .wal_sync_threshold_bytes = (size_t)oscWalSyncThresholdBytes, - .wal_sync_on_commit = oscWalSyncOnCommit ? 1 : 0, - .replica_mode = oscReplicaMode ? 1 : 0, - .replica_sync_interval_us = (uint64_t)oscReplicaSyncIntervalUs, - .replica_replay_wal = oscReplicaReplayWal ? 1 : 0}; - - tidesdb_config_t config = { - .db_path = (char *)path, - .num_flush_threads = numFlushThreads, - .num_compaction_threads = numCompactionThreads, - .log_level = (tidesdb_log_level_t)logLevel, - .block_cache_size = (size_t)blockCacheSize, - .max_open_sstables = (size_t)maxOpenSSTables, - .max_memory_usage = (size_t)maxMemoryUsage, - .log_to_file = logToFile ? 1 : 0, - .log_truncation_at = (size_t)logTruncationAt, - .unified_memtable = unifiedMemtable ? 1 : 0, - .unified_memtable_write_buffer_size = (size_t)unifiedMemtableWriteBufferSize, - .unified_memtable_skip_list_max_level = unifiedMemtableSkipListMaxLevel, - .unified_memtable_skip_list_probability = unifiedMemtableSkipListProbability, - .unified_memtable_sync_mode = unifiedMemtableSyncMode, - .unified_memtable_sync_interval_us = (uint64_t)unifiedMemtableSyncIntervalUs, - .object_store = obj_store, - .object_store_config = obj_store != NULL ? &os_cfg : NULL, - .max_concurrent_flushes = maxConcurrentFlushes, - .finish_compactions_on_close = finishCompactionsOnClose ? 1 : 0}; - - tidesdb_t *db = NULL; - int result = tidesdb_open(&config, &db); - - (*env)->ReleaseStringUTFChars(env, dbPath, path); - if (fs_path != NULL) +/** + * Copies a library-allocated buffer into a new Java byte array and frees the + * original with tidesdb_free. Returns NULL with an exception pending on failure. + */ +static jbyteArray toByteArrayAndFree(JNIEnv *env, uint8_t *buffer, size_t size) +{ + if (size > (size_t)JSIZE_MAX) { - (*env)->ReleaseStringUTFChars(env, objectStoreFsPath, fs_path); + tidesdb_free(buffer); + throwTidesDBException(env, TDB_ERR_TOO_LARGE, "Value exceeds the maximum Java array length"); + return NULL; } - if (cache_path != NULL) + + jbyteArray result = (*env)->NewByteArray(env, (jsize)size); + if (result == NULL) { - (*env)->ReleaseStringUTFChars(env, oscLocalCachePath, cache_path); + tidesdb_free(buffer); + return NULL; } - - if (result != TDB_SUCCESS) + if (size > 0) { - /* Destroy any connector that was not transferred to tidesdb_open. - * Externally-provided connectors (e.g. S3) and locally-created - * filesystem connectors are both owned by this JNI call until - * tidesdb_open succeeds. On failure we must release them. */ - if (obj_store != NULL) - { - struct _tidesdb_objstore_overlay *overlay = - (struct _tidesdb_objstore_overlay *)obj_store; - if (overlay->destroy) - { - overlay->destroy(overlay->ctx); - } - free(obj_store); - obj_store = NULL; - } - - throwTidesDBException(env, result, getErrorMessage(result)); - return 0; + (*env)->SetByteArrayRegion(env, result, 0, (jsize)size, (const jbyte *)buffer); } - - return (jlong)(uintptr_t)db; + tidesdb_free(buffer); + return result; } -/* S3 object store support is an optional build feature of the core library - * (TIDESDB_WITH_S3=ON). Resolve the factory at runtime via dlsym so this JNI library links and - * loads against a core build that lacks S3 -- callers get a clear exception instead of a - * load-time failure. */ -typedef tidesdb_objstore_t *(*tdb_s3_create_config_fn)(const tidesdb_objstore_s3_config_t *); - -static tdb_s3_create_config_fn resolve_s3_create_config(void) +/** Builds a long[] from a native array. Returns NULL with an exception pending on failure. */ +static jlongArray newLongArray(JNIEnv *env, const uint64_t *values, jsize count) { -#ifdef _WIN32 - return NULL; /* S3 connector is not exposed on Windows builds */ -#else - return (tdb_s3_create_config_fn)dlsym(RTLD_DEFAULT, "tidesdb_objstore_s3_create_config"); -#endif + jlongArray array = (*env)->NewLongArray(env, count); + if (array == NULL) + { + return NULL; + } + jlong stack[TDB_MAX_LEVELS]; + for (jsize i = 0; i < count; i++) + { + stack[i] = (jlong)values[i]; + } + (*env)->SetLongArrayRegion(env, array, 0, count, stack); + return array; } -JNIEXPORT jboolean JNICALL Java_com_tidesdb_TidesDB_nativeS3Available(JNIEnv *env, jclass cls) +/** Builds a long[] from a native size_t array. */ +static jlongArray newLongArrayFromSizes(JNIEnv *env, const size_t *values, jsize count) { - (void)env; - (void)cls; - return resolve_s3_create_config() != NULL ? JNI_TRUE : JNI_FALSE; + jlongArray array = (*env)->NewLongArray(env, count); + if (array == NULL) + { + return NULL; + } + jlong stack[TDB_MAX_LEVELS]; + for (jsize i = 0; i < count; i++) + { + stack[i] = (jlong)values[i]; + } + (*env)->SetLongArrayRegion(env, array, 0, count, stack); + return array; } -JNIEXPORT jlong JNICALL Java_com_tidesdb_TidesDB_nativeObjstoreS3Create( - JNIEnv *env, jclass cls, jstring endpoint, jstring bucket, jstring prefix, jstring accessKey, - jstring secretKey, jstring region, jboolean useSsl, jboolean usePathStyle, jstring tlsCaPath, - jboolean tlsInsecureSkipVerify, jlong multipartThreshold, jlong multipartPartSize) +/** Builds an int[] from a native int array. */ +static jintArray newIntArray(JNIEnv *env, const int *values, jsize count) { - (void)cls; - - tdb_s3_create_config_fn create_fn = resolve_s3_create_config(); - if (create_fn == NULL) + jintArray array = (*env)->NewIntArray(env, count); + if (array == NULL) { - throwTidesDBException(env, TDB_ERR_INVALID_ARGS, - "TidesDB was built without S3 support (rebuild the core library with " - "TIDESDB_WITH_S3=ON)"); - return 0; + return NULL; } - - /* required strings with NULL checks */ - const char *c_endpoint = endpoint ? (*env)->GetStringUTFChars(env, endpoint, NULL) : NULL; - if (endpoint != NULL && c_endpoint == NULL) return 0; - - const char *c_bucket = bucket ? (*env)->GetStringUTFChars(env, bucket, NULL) : NULL; - if (bucket != NULL && c_bucket == NULL) + jint stack[TDB_MAX_LEVELS]; + for (jsize i = 0; i < count; i++) { - if (c_endpoint) (*env)->ReleaseStringUTFChars(env, endpoint, c_endpoint); - return 0; + stack[i] = (jint)values[i]; } + (*env)->SetIntArrayRegion(env, array, 0, count, stack); + return array; +} - const char *c_access = accessKey ? (*env)->GetStringUTFChars(env, accessKey, NULL) : NULL; - if (accessKey != NULL && c_access == NULL) +/** Builds an int[] from a native uint8_t array, used for encoding pipelines. */ +static jintArray newIntArrayFromBytes(JNIEnv *env, const uint8_t *values, jsize count) +{ + jintArray array = (*env)->NewIntArray(env, count); + if (array == NULL) { - if (c_endpoint) (*env)->ReleaseStringUTFChars(env, endpoint, c_endpoint); - if (c_bucket) (*env)->ReleaseStringUTFChars(env, bucket, c_bucket); - return 0; + return NULL; } - - const char *c_secret = secretKey ? (*env)->GetStringUTFChars(env, secretKey, NULL) : NULL; - if (secretKey != NULL && c_secret == NULL) + jint stack[TDB_ENCODING_PIPELINE_MAX]; + for (jsize i = 0; i < count; i++) { - if (c_endpoint) (*env)->ReleaseStringUTFChars(env, endpoint, c_endpoint); - if (c_bucket) (*env)->ReleaseStringUTFChars(env, bucket, c_bucket); - if (c_access) (*env)->ReleaseStringUTFChars(env, accessKey, c_access); - return 0; + stack[i] = (jint)values[i]; } + (*env)->SetIntArrayRegion(env, array, 0, count, stack); + return array; +} - /* optional strings */ - const char *c_prefix = prefix ? (*env)->GetStringUTFChars(env, prefix, NULL) : NULL; - if (prefix != NULL && c_prefix == NULL) +/** + * Copies a Java int[] of encoding ids into a config's fixed pipeline slot. + * Returns 0 on success, -1 with an exception pending otherwise. The Java builder + * already bounds the length and each id, so a violation here is a programming + * error rather than user input. + */ +static int fillEncodingPipeline(JNIEnv *env, jintArray ids, uint8_t *pipeline, uint8_t *count) +{ + *count = 0; + if (ids == NULL) { - if (c_endpoint) (*env)->ReleaseStringUTFChars(env, endpoint, c_endpoint); - if (c_bucket) (*env)->ReleaseStringUTFChars(env, bucket, c_bucket); - if (c_access) (*env)->ReleaseStringUTFChars(env, accessKey, c_access); - if (c_secret) (*env)->ReleaseStringUTFChars(env, secretKey, c_secret); return 0; } - const char *c_region = region ? (*env)->GetStringUTFChars(env, region, NULL) : NULL; - if (region != NULL && c_region == NULL) + jsize length = (*env)->GetArrayLength(env, ids); + if (length > TDB_ENCODING_PIPELINE_MAX) { - if (c_endpoint) (*env)->ReleaseStringUTFChars(env, endpoint, c_endpoint); - if (c_bucket) (*env)->ReleaseStringUTFChars(env, bucket, c_bucket); - if (c_access) (*env)->ReleaseStringUTFChars(env, accessKey, c_access); - if (c_secret) (*env)->ReleaseStringUTFChars(env, secretKey, c_secret); - if (c_prefix) (*env)->ReleaseStringUTFChars(env, prefix, c_prefix); - return 0; + throwTidesDBException(env, TDB_ERR_INVALID_ARGS, "Encoding pipeline is too long"); + return -1; } - - const char *c_ca = tlsCaPath ? (*env)->GetStringUTFChars(env, tlsCaPath, NULL) : NULL; - if (tlsCaPath != NULL && c_ca == NULL) + if (length == 0) { - if (c_endpoint) (*env)->ReleaseStringUTFChars(env, endpoint, c_endpoint); - if (c_bucket) (*env)->ReleaseStringUTFChars(env, bucket, c_bucket); - if (c_access) (*env)->ReleaseStringUTFChars(env, accessKey, c_access); - if (c_secret) (*env)->ReleaseStringUTFChars(env, secretKey, c_secret); - if (c_prefix) (*env)->ReleaseStringUTFChars(env, prefix, c_prefix); - if (c_region) (*env)->ReleaseStringUTFChars(env, region, c_region); return 0; } - tidesdb_objstore_s3_config_t cfg = {.endpoint = c_endpoint, - .bucket = c_bucket, - .prefix = c_prefix, - .access_key = c_access, - .secret_key = c_secret, - .region = c_region, - .use_ssl = useSsl ? 1 : 0, - .use_path_style = usePathStyle ? 1 : 0, - .tls_ca_path = c_ca, - .tls_insecure_skip_verify = tlsInsecureSkipVerify ? 1 : 0, - .multipart_threshold = (size_t)multipartThreshold, - .multipart_part_size = (size_t)multipartPartSize}; - - tidesdb_objstore_t *connector = create_fn(&cfg); - - if (endpoint) (*env)->ReleaseStringUTFChars(env, endpoint, c_endpoint); - if (bucket) (*env)->ReleaseStringUTFChars(env, bucket, c_bucket); - if (accessKey) (*env)->ReleaseStringUTFChars(env, accessKey, c_access); - if (secretKey) (*env)->ReleaseStringUTFChars(env, secretKey, c_secret); - if (prefix) (*env)->ReleaseStringUTFChars(env, prefix, c_prefix); - if (region) (*env)->ReleaseStringUTFChars(env, region, c_region); - if (tlsCaPath) (*env)->ReleaseStringUTFChars(env, tlsCaPath, c_ca); - - if (connector == NULL) - { - throwTidesDBException(env, TDB_ERR_IO, - "Failed to create S3 object store connector (check endpoint, " - "credentials, and bucket)"); - return 0; + jint stack[TDB_ENCODING_PIPELINE_MAX]; + (*env)->GetIntArrayRegion(env, ids, 0, length, stack); + if (jvm_exception_pending(env)) + { + return -1; } - return (jlong)(uintptr_t)connector; -} - -JNIEXPORT void JNICALL Java_com_tidesdb_TidesDB_nativeClose(JNIEnv *env, jclass cls, jlong handle) -{ - tidesdb_t *db = (tidesdb_t *)(uintptr_t)handle; - if (db != NULL) + for (jsize i = 0; i < length; i++) { - tidesdb_close(db); + if (stack[i] < 0 || stack[i] > 255) + { + throwTidesDBException(env, TDB_ERR_INVALID_ARGS, "Encoding id is outside [0, 255]"); + return -1; + } + pipeline[i] = (uint8_t)stack[i]; } + *count = (uint8_t)length; + return 0; } -JNIEXPORT void JNICALL Java_com_tidesdb_TidesDB_nativeCreateColumnFamily( - JNIEnv *env, jclass cls, jlong handle, jstring name, jlong writeBufferSize, - jlong levelSizeRatio, jint minLevels, jint dividingLevelOffset, jlong klogValueThreshold, - jint compressionAlgorithm, jboolean enableBloomFilter, jdouble bloomFPR, - jboolean enableBlockIndexes, jint indexSampleRatio, jint blockIndexPrefixLen, jint syncMode, - jlong syncIntervalUs, jstring comparatorName, jint skipListMaxLevel, jfloat skipListProbability, - jint defaultIsolationLevel, jlong minDiskSpace, jint l1FileCountTrigger, - jint l0QueueStallThreshold, jdouble tombstoneDensityTrigger, jlong tombstoneDensityMinEntries, - jboolean useBtree, jboolean objectLazyCompaction, jboolean objectPrefetchCompaction) +/** + * Populates a column family config from the flat field list the Java side sends. + * The struct is zeroed first, so the commit hook fields stay NULL and the name + * stays empty -- the create and update calls take the name from elsewhere. + */ +static int fillCfConfig(JNIEnv *env, tidesdb_column_family_config_t *cfg, jlong levelSizeRatio, + jint minLevels, jint dividingLevelOffset, jboolean keepValuesInline, + jlong btreeKlogBlockSize, jintArray encodingPipeline, + jboolean enableBloomFilter, jdouble bloomFpr, jint defaultIsolationLevel, + jint l1FileCountTrigger, jdouble tombstoneDensityTrigger, + jlong tombstoneDensityMinEntries) +{ + memset(cfg, 0, sizeof(*cfg)); + + cfg->level_size_ratio = (size_t)levelSizeRatio; + cfg->min_levels = (int)minLevels; + cfg->dividing_level_offset = (int)dividingLevelOffset; + cfg->keep_values_inline = keepValuesInline ? 1 : 0; + cfg->btree_klog_block_size = (size_t)btreeKlogBlockSize; + cfg->enable_bloom_filter = enableBloomFilter ? 1 : 0; + cfg->bloom_fpr = (double)bloomFpr; + cfg->default_isolation_level = (tidesdb_isolation_level_t)defaultIsolationLevel; + cfg->l1_file_count_trigger = (int)l1FileCountTrigger; + cfg->tombstone_density_trigger = (double)tombstoneDensityTrigger; + cfg->tombstone_density_min_entries = (uint64_t)tombstoneDensityMinEntries; + + return fillEncodingPipeline(env, encodingPipeline, cfg->encoding_pipeline, + &cfg->encoding_count); +} + +/* ===== object builders ===== */ + +/** Builds a com.tidesdb.ColumnFamilyConfig mirroring a native config. */ +static jobject buildCfConfigObject(JNIEnv *env, const tidesdb_column_family_config_t *cfg) { - tidesdb_t *db = (tidesdb_t *)(uintptr_t)handle; - const char *cfName = (*env)->GetStringUTFChars(env, name, NULL); - if (cfName == NULL) - { - throwTidesDBException(env, TDB_ERR_MEMORY, "Failed to get column family name"); - return; - } + jclass cls = (*env)->FindClass(env, "com/tidesdb/ColumnFamilyConfig"); + if (cls == NULL) return NULL; - const char *compName = NULL; - if (comparatorName != NULL) + jmethodID factory = (*env)->GetStaticMethodID( + env, cls, "fromNative", + "(Ljava/lang/String;JIIZJ[IZDIIDJ)Lcom/tidesdb/ColumnFamilyConfig;"); + if (factory == NULL) { - compName = (*env)->GetStringUTFChars(env, comparatorName, NULL); + (*env)->DeleteLocalRef(env, cls); + return NULL; } - tidesdb_column_family_config_t config = { - .write_buffer_size = (size_t)writeBufferSize, - .level_size_ratio = (size_t)levelSizeRatio, - .min_levels = minLevels, - .dividing_level_offset = dividingLevelOffset, - .klog_value_threshold = (size_t)klogValueThreshold, - .compression_algorithm = (compression_algorithm)compressionAlgorithm, - .enable_bloom_filter = enableBloomFilter ? 1 : 0, - .bloom_fpr = bloomFPR, - .enable_block_indexes = enableBlockIndexes ? 1 : 0, - .index_sample_ratio = indexSampleRatio, - .block_index_prefix_len = blockIndexPrefixLen, - .sync_mode = syncMode, - .sync_interval_us = (uint64_t)syncIntervalUs, - .skip_list_max_level = skipListMaxLevel, - .skip_list_probability = skipListProbability, - .default_isolation_level = (tidesdb_isolation_level_t)defaultIsolationLevel, - .min_disk_space = (uint64_t)minDiskSpace, - .l1_file_count_trigger = l1FileCountTrigger, - .l0_queue_stall_threshold = l0QueueStallThreshold, - .tombstone_density_trigger = tombstoneDensityTrigger, - .tombstone_density_min_entries = (uint64_t)tombstoneDensityMinEntries, - .use_btree = useBtree ? 1 : 0, - .object_lazy_compaction = objectLazyCompaction ? 1 : 0, - .object_prefetch_compaction = objectPrefetchCompaction ? 1 : 0}; + /* the name is a fixed char array; bound the read in case it is not terminated */ + char nameBuf[TDB_MAX_CF_NAME_LEN + 1]; + memcpy(nameBuf, cfg->name, TDB_MAX_CF_NAME_LEN); + nameBuf[TDB_MAX_CF_NAME_LEN] = '\0'; - memset(config.comparator_name, 0, TDB_MAX_COMPARATOR_NAME); - if (compName != NULL && strlen(compName) > 0) + jstring name = (*env)->NewStringUTF(env, nameBuf); + if (name == NULL) { - strncpy(config.comparator_name, compName, TDB_MAX_COMPARATOR_NAME - 1); + (*env)->DeleteLocalRef(env, cls); + return NULL; } - memset(config.comparator_ctx_str, 0, TDB_MAX_COMPARATOR_CTX); - config.comparator_fn_cached = NULL; - config.comparator_ctx_cached = NULL; - - int result = tidesdb_create_column_family(db, cfName, &config); - - (*env)->ReleaseStringUTFChars(env, name, cfName); - if (compName != NULL) + uint8_t pipelineCount = cfg->encoding_count; + if (pipelineCount > TDB_ENCODING_PIPELINE_MAX) pipelineCount = TDB_ENCODING_PIPELINE_MAX; + jintArray pipeline = newIntArrayFromBytes(env, cfg->encoding_pipeline, (jsize)pipelineCount); + if (pipeline == NULL) { - (*env)->ReleaseStringUTFChars(env, comparatorName, compName); + (*env)->DeleteLocalRef(env, name); + (*env)->DeleteLocalRef(env, cls); + return NULL; } - if (result != TDB_SUCCESS) - { - throwTidesDBException(env, result, getErrorMessage(result)); - } + jobject result = (*env)->CallStaticObjectMethod( + env, cls, factory, name, (jlong)cfg->level_size_ratio, (jint)cfg->min_levels, + (jint)cfg->dividing_level_offset, cfg->keep_values_inline ? JNI_TRUE : JNI_FALSE, + (jlong)cfg->btree_klog_block_size, pipeline, + cfg->enable_bloom_filter ? JNI_TRUE : JNI_FALSE, (jdouble)cfg->bloom_fpr, + (jint)cfg->default_isolation_level, (jint)cfg->l1_file_count_trigger, + (jdouble)cfg->tombstone_density_trigger, (jlong)cfg->tombstone_density_min_entries); + + (*env)->DeleteLocalRef(env, pipeline); + (*env)->DeleteLocalRef(env, name); + (*env)->DeleteLocalRef(env, cls); + return result; } -JNIEXPORT void JNICALL Java_com_tidesdb_TidesDB_nativeDropColumnFamily(JNIEnv *env, jclass cls, - jlong handle, jstring name) +/** Builds a com.tidesdb.Config mirroring a native config. */ +static jobject buildConfigObject(JNIEnv *env, const tidesdb_config_t *cfg) { - tidesdb_t *db = (tidesdb_t *)(uintptr_t)handle; - const char *cfName = (*env)->GetStringUTFChars(env, name, NULL); - if (cfName == NULL) + jclass cls = (*env)->FindClass(env, "com/tidesdb/Config"); + if (cls == NULL) return NULL; + + jmethodID factory = + (*env)->GetStaticMethodID(env, cls, "fromNative", "(IIIJJZJJIFIJJJIIJ)Lcom/tidesdb/Config;"); + if (factory == NULL) { - throwTidesDBException(env, TDB_ERR_MEMORY, "Failed to get column family name"); - return; + (*env)->DeleteLocalRef(env, cls); + return NULL; } - int result = tidesdb_drop_column_family(db, cfName); - - (*env)->ReleaseStringUTFChars(env, name, cfName); + jobject result = (*env)->CallStaticObjectMethod( + env, cls, factory, (jint)cfg->num_flush_threads, (jint)cfg->num_compaction_threads, + (jint)cfg->log_level, (jlong)cfg->block_cache_size, (jlong)cfg->max_open_sstables, + cfg->log_to_file ? JNI_TRUE : JNI_FALSE, (jlong)cfg->log_truncation_at, + (jlong)cfg->memtable_write_buffer_size, (jint)cfg->memtable_skip_list_max_level, + (jfloat)cfg->memtable_skip_list_probability, (jint)cfg->memtable_sync_mode, + (jlong)cfg->memtable_sync_interval_us, (jlong)cfg->value_separation_threshold, + (jlong)cfg->vlog_segment_size, (jint)cfg->memtable_l0_queue_stall_threshold, + (jint)cfg->memtable_idle_flush_seconds, (jlong)cfg->txn_timeout_seconds); - if (result != TDB_SUCCESS) - { - throwTidesDBException(env, result, getErrorMessage(result)); - } + (*env)->DeleteLocalRef(env, cls); + return result; } -JNIEXPORT jlong JNICALL Java_com_tidesdb_TidesDB_nativeGetColumnFamily(JNIEnv *env, jclass cls, - jlong handle, jstring name) +/** Builds a com.tidesdb.CfStats mirroring native per-column-family statistics. */ +static jobject buildCfStatsObject(JNIEnv *env, const tidesdb_cf_stats_t *stats) { - tidesdb_t *db = (tidesdb_t *)(uintptr_t)handle; - const char *cfName = (*env)->GetStringUTFChars(env, name, NULL); - if (cfName == NULL) + jclass cls = (*env)->FindClass(env, "com/tidesdb/CfStats"); + if (cls == NULL) return NULL; + + jmethodID ctor = (*env)->GetMethodID( + env, cls, "", + "(ILcom/tidesdb/ColumnFamilyConfig;[J[I[J[JJJDDDJJDJDDIJJJJJJJJ)V"); + if (ctor == NULL) { - throwTidesDBException(env, TDB_ERR_MEMORY, "Failed to get column family name"); - return 0; + (*env)->DeleteLocalRef(env, cls); + return NULL; } - tidesdb_column_family_t *cf = tidesdb_get_column_family(db, cfName); - - (*env)->ReleaseStringUTFChars(env, name, cfName); + jobject config = buildCfConfigObject(env, &stats->config); + jlongArray levelSizes = newLongArrayFromSizes(env, stats->level_sizes, TDB_MAX_LEVELS); + jintArray levelSstables = newIntArray(env, stats->level_num_sstables, TDB_MAX_LEVELS); + jlongArray levelKeys = newLongArray(env, stats->level_key_counts, TDB_MAX_LEVELS); + jlongArray levelTombstones = newLongArray(env, stats->level_tombstone_counts, TDB_MAX_LEVELS); - if (cf == NULL) + jobject result = NULL; + if (config != NULL && levelSizes != NULL && levelSstables != NULL && levelKeys != NULL && + levelTombstones != NULL) { - throwTidesDBException(env, TDB_ERR_NOT_FOUND, "Column family not found"); - return 0; + result = (*env)->NewObject( + env, cls, ctor, (jint)stats->num_levels, config, levelSizes, levelSstables, levelKeys, + levelTombstones, (jlong)stats->total_keys, (jlong)stats->total_data_size, + (jdouble)stats->avg_key_size, (jdouble)stats->avg_value_size, (jdouble)stats->read_amp, + (jlong)stats->btree_total_nodes, (jlong)stats->btree_max_height, + (jdouble)stats->btree_avg_height, (jlong)stats->total_tombstones, + (jdouble)stats->tombstone_ratio, (jdouble)stats->max_sst_density, + (jint)stats->max_sst_density_level, (jlong)stats->wal_bytes_written, + (jlong)stats->flush_bytes_written, (jlong)stats->compaction_bytes_written, + (jlong)stats->compaction_bytes_read, (jlong)stats->user_bytes_written, + (jlong)stats->compaction_count, (jlong)stats->unflushed_key_count, + (jlong)stats->filter_resident_bytes); } - return (jlong)(uintptr_t)cf; + if (levelTombstones != NULL) (*env)->DeleteLocalRef(env, levelTombstones); + if (levelKeys != NULL) (*env)->DeleteLocalRef(env, levelKeys); + if (levelSstables != NULL) (*env)->DeleteLocalRef(env, levelSstables); + if (levelSizes != NULL) (*env)->DeleteLocalRef(env, levelSizes); + if (config != NULL) (*env)->DeleteLocalRef(env, config); + (*env)->DeleteLocalRef(env, cls); + return result; } -JNIEXPORT jobjectArray JNICALL Java_com_tidesdb_TidesDB_nativeListColumnFamilies(JNIEnv *env, - jclass cls, - jlong handle) +/** Builds a com.tidesdb.DbStats mirroring native database statistics. */ +static jobject buildDbStatsObject(JNIEnv *env, const tidesdb_db_stats_t *stats) { - tidesdb_t *db = (tidesdb_t *)(uintptr_t)handle; - char **names = NULL; - int count = 0; + jclass cls = (*env)->FindClass(env, "com/tidesdb/DbStats"); + if (cls == NULL) return NULL; - int result = tidesdb_list_column_families(db, &names, &count); - if (result != TDB_SUCCESS) + jmethodID ctor = (*env)->GetMethodID( + env, cls, "", "(IIIIJIJJIJJZJJJJJJJJJJJJJJJJJJJJJJJJJ)V"); + if (ctor == NULL) { - throwTidesDBException(env, result, getErrorMessage(result)); + (*env)->DeleteLocalRef(env, cls); return NULL; } - jclass stringClass = (*env)->FindClass(env, "java/lang/String"); - if (stringClass == NULL) - { - for (int i = 0; i < count; i++) free(names[i]); - free(names); - return NULL; - } + jobject result = (*env)->NewObject( + env, cls, ctor, (jint)stats->num_column_families, (jint)stats->immutable_memtable_count, + (jint)stats->compaction_pending_count, (jint)stats->total_sstable_count, + (jlong)stats->total_data_size_bytes, (jint)stats->num_open_sstables, + (jlong)stats->global_seq, (jlong)stats->min_snapshot_seq, (jint)stats->active_txn_count, + (jlong)stats->txn_memory_bytes, (jlong)stats->memtable_bytes, + stats->is_flushing ? JNI_TRUE : JNI_FALSE, (jlong)stats->next_cf_index, + (jlong)stats->wal_generation, (jlong)stats->flush_count, (jlong)stats->compaction_count, + (jlong)stats->flush_bytes_written, (jlong)stats->compaction_bytes_written, + (jlong)stats->compaction_bytes_read, (jlong)stats->wal_bytes_written, + (jlong)stats->user_bytes_written, (jlong)stats->vlog_file_size, + (jlong)stats->vlog_value_count, (jlong)stats->vlog_used_bytes, + (jlong)stats->vlog_stored_bytes, (jlong)stats->vlog_live_bytes, + (jlong)stats->vlog_segment_count, (jlong)stats->vlog_bytes_written, + (jlong)stats->vlog_dead_bytes, (jlong)stats->vlog_reclaim_calls, + (jlong)stats->vlog_reclaim_passes, (jlong)stats->vlog_segments_retired, + (jlong)stats->vlog_segments_drainable, (jlong)stats->writes_throttled, + (jlong)stats->writes_blocked, (jlong)stats->write_stall_us, + (jlong)stats->write_stall_ceiling_hits); - jobjectArray array = (*env)->NewObjectArray(env, count, stringClass, NULL); - if (array == NULL) - { - for (int i = 0; i < count; i++) free(names[i]); - free(names); - return NULL; - } + (*env)->DeleteLocalRef(env, cls); + return result; +} - for (int i = 0; i < count; i++) - { - jstring str = (*env)->NewStringUTF(env, names[i]); - if (str == NULL) - { - /* OOM: pending JVM exception. Free remaining names and return. */ - for (int j = i; j < count; j++) free(names[j]); - free(names); - return NULL; - } - (*env)->SetObjectArrayElement(env, array, i, str); - (*env)->DeleteLocalRef(env, str); - free(names[i]); - } - free(names); +/* ===== com.tidesdb.Config ===== */ - return array; +JNIEXPORT jobject JNICALL Java_com_tidesdb_Config_nativeDefaultConfig(JNIEnv *env, jclass cls) +{ + (void)cls; + tidesdb_config_t config = tidesdb_default_config(); + return buildConfigObject(env, &config); } -JNIEXPORT jlong JNICALL Java_com_tidesdb_TidesDB_nativeBeginTransaction(JNIEnv *env, jclass cls, - jlong handle) +/* ===== com.tidesdb.ColumnFamilyConfig ===== */ + +JNIEXPORT jobject JNICALL Java_com_tidesdb_ColumnFamilyConfig_nativeDefaultConfig(JNIEnv *env, + jclass cls) { - tidesdb_t *db = (tidesdb_t *)(uintptr_t)handle; - tidesdb_txn_t *txn = NULL; + (void)cls; + tidesdb_column_family_config_t config = tidesdb_default_column_family_config(); + return buildCfConfigObject(env, &config); +} - int result = tidesdb_txn_begin(db, &txn); - if (result != TDB_SUCCESS) +/* ===== com.tidesdb.TidesDB : lifecycle ===== */ + +JNIEXPORT jlong JNICALL Java_com_tidesdb_TidesDB_nativeOpen( + JNIEnv *env, jclass cls, jstring dbPath, jint numFlushThreads, jint numCompactionThreads, + jint logLevel, jlong blockCacheSize, jlong maxOpenSSTables, jboolean logToFile, + jlong logTruncationAt, jlong memtableWriteBufferSize, jint memtableSkipListMaxLevel, + jfloat memtableSkipListProbability, jint memtableSyncMode, jlong memtableSyncIntervalUs, + jlong valueSeparationThreshold, jlong vlogSegmentSize, jint memtableL0QueueStallThreshold, + jint memtableIdleFlushSeconds, jlong txnTimeoutSeconds) +{ + (void)cls; + + const char *path = (*env)->GetStringUTFChars(env, dbPath, NULL); + if (path == NULL) { - throwTidesDBException(env, result, getErrorMessage(result)); + if (!jvm_exception_pending(env)) + { + throwTidesDBException(env, TDB_ERR_MEMORY, "Failed to read the database path"); + } return 0; } - return (jlong)(uintptr_t)txn; -} + tidesdb_config_t config; + memset(&config, 0, sizeof(config)); + config.db_path = (char *)path; + config.num_flush_threads = (int)numFlushThreads; + config.num_compaction_threads = (int)numCompactionThreads; + config.log_level = (tidesdb_log_level_t)logLevel; + config.block_cache_size = (size_t)blockCacheSize; + config.max_open_sstables = (size_t)maxOpenSSTables; + config.log_to_file = logToFile ? 1 : 0; + config.log_truncation_at = (size_t)logTruncationAt; + config.memtable_write_buffer_size = (size_t)memtableWriteBufferSize; + config.memtable_skip_list_max_level = (int)memtableSkipListMaxLevel; + config.memtable_skip_list_probability = (float)memtableSkipListProbability; + config.memtable_sync_mode = (int)memtableSyncMode; + config.memtable_sync_interval_us = (uint64_t)memtableSyncIntervalUs; + config.value_separation_threshold = (size_t)valueSeparationThreshold; + config.vlog_segment_size = (size_t)vlogSegmentSize; + config.memtable_l0_queue_stall_threshold = (int)memtableL0QueueStallThreshold; + config.memtable_idle_flush_seconds = (int)memtableIdleFlushSeconds; + config.txn_timeout_seconds = (int64_t)txnTimeoutSeconds; -JNIEXPORT jlong JNICALL Java_com_tidesdb_TidesDB_nativeBeginTransactionWithIsolation( - JNIEnv *env, jclass cls, jlong handle, jint isolationLevel) -{ - tidesdb_t *db = (tidesdb_t *)(uintptr_t)handle; - tidesdb_txn_t *txn = NULL; + tidesdb_t *db = NULL; + int result = tidesdb_open(&config, &db); + + (*env)->ReleaseStringUTFChars(env, dbPath, path); - int result = - tidesdb_txn_begin_with_isolation(db, (tidesdb_isolation_level_t)isolationLevel, &txn); if (result != TDB_SUCCESS) { - throwTidesDBException(env, result, getErrorMessage(result)); + throwResult(env, result); return 0; } - return (jlong)(uintptr_t)txn; + return (jlong)(uintptr_t)db; } -JNIEXPORT jobject JNICALL Java_com_tidesdb_TidesDB_nativeGetCacheStats(JNIEnv *env, jclass cls, - jlong handle) +JNIEXPORT void JNICALL Java_com_tidesdb_TidesDB_nativeClose(JNIEnv *env, jclass cls, jlong handle) { - tidesdb_t *db = (tidesdb_t *)(uintptr_t)handle; - tidesdb_cache_stats_t stats; - - int result = tidesdb_get_cache_stats(db, &stats); - if (result != TDB_SUCCESS) - { - throwTidesDBException(env, result, getErrorMessage(result)); - return NULL; - } + (void)env; + (void)cls; + tidesdb_close((tidesdb_t *)(uintptr_t)handle); +} - jclass cacheStatsClass = (*env)->FindClass(env, "com/tidesdb/CacheStats"); - if (cacheStatsClass == NULL) return NULL; +JNIEXPORT jboolean JNICALL Java_com_tidesdb_TidesDB_nativeCompressionAvailable(JNIEnv *env, + jclass cls, + jint algorithm) +{ + (void)env; + (void)cls; + return tidesdb_compression_available((tidesdb_compression_algorithm_t)algorithm) ? JNI_TRUE + : JNI_FALSE; +} - jmethodID constructor = (*env)->GetMethodID(env, cacheStatsClass, "", "(ZJJJJDJ)V"); - if (constructor == NULL) return NULL; +JNIEXPORT jstring JNICALL Java_com_tidesdb_TidesDB_nativeStrerror(JNIEnv *env, jclass cls, jint code) +{ + (void)cls; + return (*env)->NewStringUTF(env, tidesdb_strerror((int)code)); +} - return (*env)->NewObject(env, cacheStatsClass, constructor, stats.enabled != 0, - (jlong)stats.total_entries, (jlong)stats.total_bytes, - (jlong)stats.hits, (jlong)stats.misses, stats.hit_rate, - (jlong)stats.num_partitions); +JNIEXPORT jlong JNICALL Java_com_tidesdb_TidesDB_nativeRaiseOpenFileLimit(JNIEnv *env, jclass cls, + jlong desired) +{ + (void)env; + (void)cls; + return (jlong)tidesdb_raise_open_file_limit((long)desired); } -JNIEXPORT void JNICALL Java_com_tidesdb_TidesDB_nativeRegisterComparator(JNIEnv *env, jclass cls, - jlong handle, jstring name, - jstring context) +/* ===== com.tidesdb.TidesDB : column families ===== */ + +JNIEXPORT void JNICALL Java_com_tidesdb_TidesDB_nativeCreateColumnFamily( + JNIEnv *env, jclass cls, jlong handle, jstring name, jlong levelSizeRatio, jint minLevels, + jint dividingLevelOffset, jboolean keepValuesInline, jlong btreeKlogBlockSize, + jintArray encodingPipeline, jboolean enableBloomFilter, jdouble bloomFpr, + jint defaultIsolationLevel, jint l1FileCountTrigger, jdouble tombstoneDensityTrigger, + jlong tombstoneDensityMinEntries) { - tidesdb_t *db = (tidesdb_t *)(uintptr_t)handle; - const char *compName = (*env)->GetStringUTFChars(env, name, NULL); - if (compName == NULL) + (void)cls; + + tidesdb_column_family_config_t config; + if (fillCfConfig(env, &config, levelSizeRatio, minLevels, dividingLevelOffset, + keepValuesInline, btreeKlogBlockSize, encodingPipeline, enableBloomFilter, + bloomFpr, defaultIsolationLevel, l1FileCountTrigger, tombstoneDensityTrigger, + tombstoneDensityMinEntries) != 0) { - throwTidesDBException(env, TDB_ERR_MEMORY, "Failed to get comparator name"); return; } - const char *ctx = NULL; - if (context != NULL) + const char *cfName = (*env)->GetStringUTFChars(env, name, NULL); + if (cfName == NULL) { - ctx = (*env)->GetStringUTFChars(env, context, NULL); + if (!jvm_exception_pending(env)) + { + throwTidesDBException(env, TDB_ERR_MEMORY, "Failed to read the column family name"); + } + return; } - int result = tidesdb_register_comparator(db, compName, NULL, ctx, NULL); - - (*env)->ReleaseStringUTFChars(env, name, compName); - if (ctx != NULL) - { - (*env)->ReleaseStringUTFChars(env, context, ctx); - } + int result = tidesdb_create_column_family((tidesdb_t *)(uintptr_t)handle, cfName, &config); + (*env)->ReleaseStringUTFChars(env, name, cfName); if (result != TDB_SUCCESS) { - throwTidesDBException(env, result, getErrorMessage(result)); + throwResult(env, result); } } -JNIEXPORT void JNICALL Java_com_tidesdb_TidesDB_nativeBackup(JNIEnv *env, jclass cls, jlong handle, - jstring dir) +JNIEXPORT void JNICALL Java_com_tidesdb_TidesDB_nativeDropColumnFamily(JNIEnv *env, jclass cls, + jlong handle, jstring name) { - tidesdb_t *db = (tidesdb_t *)(uintptr_t)handle; - const char *backupDir = (*env)->GetStringUTFChars(env, dir, NULL); - if (backupDir == NULL) - { - throwTidesDBException(env, TDB_ERR_MEMORY, "Failed to get backup directory"); - return; - } - - int result = tidesdb_backup(db, (char *)backupDir); - - (*env)->ReleaseStringUTFChars(env, dir, backupDir); - - if (result != TDB_SUCCESS) - { - throwTidesDBException(env, result, getErrorMessage(result)); - } -} + (void)cls; -JNIEXPORT void JNICALL Java_com_tidesdb_TidesDB_nativeCheckpoint(JNIEnv *env, jclass cls, - jlong handle, jstring dir) -{ - tidesdb_t *db = (tidesdb_t *)(uintptr_t)handle; - const char *checkpointDir = (*env)->GetStringUTFChars(env, dir, NULL); - if (checkpointDir == NULL) + const char *cfName = (*env)->GetStringUTFChars(env, name, NULL); + if (cfName == NULL) { - throwTidesDBException(env, TDB_ERR_MEMORY, "Failed to get checkpoint directory"); + if (!jvm_exception_pending(env)) + { + throwTidesDBException(env, TDB_ERR_MEMORY, "Failed to read the column family name"); + } return; } - int result = tidesdb_checkpoint(db, checkpointDir); - - (*env)->ReleaseStringUTFChars(env, dir, checkpointDir); + int result = tidesdb_drop_column_family((tidesdb_t *)(uintptr_t)handle, cfName); + (*env)->ReleaseStringUTFChars(env, name, cfName); if (result != TDB_SUCCESS) { - throwTidesDBException(env, result, getErrorMessage(result)); + throwResult(env, result); } } JNIEXPORT void JNICALL Java_com_tidesdb_TidesDB_nativeRenameColumnFamily(JNIEnv *env, jclass cls, - jlong handle, - jstring oldName, - jstring newName) + jlong handle, + jstring oldName, + jstring newName) { - tidesdb_t *db = (tidesdb_t *)(uintptr_t)handle; - const char *oldCfName = (*env)->GetStringUTFChars(env, oldName, NULL); - if (oldCfName == NULL) + (void)cls; + + const char *from = (*env)->GetStringUTFChars(env, oldName, NULL); + if (from == NULL) { - throwTidesDBException(env, TDB_ERR_MEMORY, "Failed to get old column family name"); + if (!jvm_exception_pending(env)) + { + throwTidesDBException(env, TDB_ERR_MEMORY, "Failed to read the column family name"); + } return; } - const char *newCfName = (*env)->GetStringUTFChars(env, newName, NULL); - if (newCfName == NULL) + const char *to = (*env)->GetStringUTFChars(env, newName, NULL); + if (to == NULL) { - (*env)->ReleaseStringUTFChars(env, oldName, oldCfName); - throwTidesDBException(env, TDB_ERR_MEMORY, "Failed to get new column family name"); + (*env)->ReleaseStringUTFChars(env, oldName, from); + if (!jvm_exception_pending(env)) + { + throwTidesDBException(env, TDB_ERR_MEMORY, "Failed to read the column family name"); + } return; } - int result = tidesdb_rename_column_family(db, oldCfName, newCfName); + int result = tidesdb_rename_column_family((tidesdb_t *)(uintptr_t)handle, from, to); - (*env)->ReleaseStringUTFChars(env, oldName, oldCfName); - (*env)->ReleaseStringUTFChars(env, newName, newCfName); + (*env)->ReleaseStringUTFChars(env, newName, to); + (*env)->ReleaseStringUTFChars(env, oldName, from); if (result != TDB_SUCCESS) { - throwTidesDBException(env, result, getErrorMessage(result)); + throwResult(env, result); } } JNIEXPORT void JNICALL Java_com_tidesdb_TidesDB_nativeCloneColumnFamily(JNIEnv *env, jclass cls, - jlong handle, - jstring sourceName, - jstring destName) + jlong handle, + jstring sourceName, + jstring destName) { - tidesdb_t *db = (tidesdb_t *)(uintptr_t)handle; - const char *srcCfName = (*env)->GetStringUTFChars(env, sourceName, NULL); - if (srcCfName == NULL) + (void)cls; + + const char *src = (*env)->GetStringUTFChars(env, sourceName, NULL); + if (src == NULL) { - throwTidesDBException(env, TDB_ERR_MEMORY, "Failed to get source column family name"); + if (!jvm_exception_pending(env)) + { + throwTidesDBException(env, TDB_ERR_MEMORY, "Failed to read the column family name"); + } return; } - const char *dstCfName = (*env)->GetStringUTFChars(env, destName, NULL); - if (dstCfName == NULL) + const char *dst = (*env)->GetStringUTFChars(env, destName, NULL); + if (dst == NULL) { - (*env)->ReleaseStringUTFChars(env, sourceName, srcCfName); - throwTidesDBException(env, TDB_ERR_MEMORY, "Failed to get destination column family name"); + (*env)->ReleaseStringUTFChars(env, sourceName, src); + if (!jvm_exception_pending(env)) + { + throwTidesDBException(env, TDB_ERR_MEMORY, "Failed to read the column family name"); + } return; } - int result = tidesdb_clone_column_family(db, srcCfName, dstCfName); + int result = tidesdb_clone_column_family((tidesdb_t *)(uintptr_t)handle, src, dst); - (*env)->ReleaseStringUTFChars(env, sourceName, srcCfName); - (*env)->ReleaseStringUTFChars(env, destName, dstCfName); + (*env)->ReleaseStringUTFChars(env, destName, dst); + (*env)->ReleaseStringUTFChars(env, sourceName, src); if (result != TDB_SUCCESS) { - throwTidesDBException(env, result, getErrorMessage(result)); + throwResult(env, result); } } -JNIEXPORT jobject JNICALL Java_com_tidesdb_ColumnFamily_nativeGetStats(JNIEnv *env, jclass cls, - jlong handle) +JNIEXPORT jlong JNICALL Java_com_tidesdb_TidesDB_nativeGetColumnFamily(JNIEnv *env, jclass cls, + jlong handle, jstring name) { - tidesdb_column_family_t *cf = (tidesdb_column_family_t *)(uintptr_t)handle; - tidesdb_stats_t *stats = NULL; - - int result = tidesdb_get_stats(cf, &stats); - if (result != TDB_SUCCESS) - { - throwTidesDBException(env, result, getErrorMessage(result)); - return NULL; - } - - jlongArray levelSizes = (*env)->NewLongArray(env, stats->num_levels); - if (levelSizes == NULL) - { - tidesdb_free_stats(stats); - return NULL; - } + (void)cls; - jlong *sizes = NULL; - if (stats->level_sizes != NULL) + const char *cfName = (*env)->GetStringUTFChars(env, name, NULL); + if (cfName == NULL) { - sizes = malloc(stats->num_levels * sizeof(jlong)); - if (sizes == NULL) - { - tidesdb_free_stats(stats); - return NULL; - } - for (int i = 0; i < stats->num_levels; i++) + if (!jvm_exception_pending(env)) { - sizes[i] = (jlong)stats->level_sizes[i]; + throwTidesDBException(env, TDB_ERR_MEMORY, "Failed to read the column family name"); } - (*env)->SetLongArrayRegion(env, levelSizes, 0, stats->num_levels, sizes); - free(sizes); - sizes = NULL; + return 0; } - jintArray levelNumSSTables = (*env)->NewIntArray(env, stats->num_levels); - if (levelNumSSTables == NULL) - { - tidesdb_free_stats(stats); - return NULL; - } + tidesdb_column_family_t *cf = + tidesdb_get_column_family((tidesdb_t *)(uintptr_t)handle, cfName); + (*env)->ReleaseStringUTFChars(env, name, cfName); - jint *nums = NULL; - if (stats->level_num_sstables != NULL) + if (cf == NULL) { - nums = malloc(stats->num_levels * sizeof(jint)); - if (nums == NULL) - { - tidesdb_free_stats(stats); - return NULL; - } - for (int i = 0; i < stats->num_levels; i++) - { - nums[i] = stats->level_num_sstables[i]; - } - (*env)->SetIntArrayRegion(env, levelNumSSTables, 0, stats->num_levels, nums); - free(nums); - nums = NULL; + throwTidesDBException(env, TDB_ERR_NOT_FOUND, "Column family not found"); + return 0; } - jlongArray levelKeyCounts = (*env)->NewLongArray(env, stats->num_levels); - if (levelKeyCounts == NULL) - { - tidesdb_free_stats(stats); - return NULL; - } + return (jlong)(uintptr_t)cf; +} - jlong *counts = NULL; - if (stats->level_key_counts != NULL) - { - counts = malloc(stats->num_levels * sizeof(jlong)); - if (counts == NULL) - { - tidesdb_free_stats(stats); - return NULL; - } - for (int i = 0; i < stats->num_levels; i++) - { - counts[i] = (jlong)stats->level_key_counts[i]; - } - (*env)->SetLongArrayRegion(env, levelKeyCounts, 0, stats->num_levels, counts); - free(counts); - counts = NULL; - } +JNIEXPORT jobjectArray JNICALL Java_com_tidesdb_TidesDB_nativeListColumnFamilies(JNIEnv *env, + jclass cls, + jlong handle) +{ + (void)cls; - jlongArray levelTombstoneCounts = (*env)->NewLongArray(env, stats->num_levels); - if (levelTombstoneCounts == NULL) + char **names = NULL; + int count = 0; + int result = tidesdb_list_column_families((tidesdb_t *)(uintptr_t)handle, &names, &count); + if (result != TDB_SUCCESS) { - tidesdb_free_stats(stats); + throwResult(env, result); return NULL; } - jlong *tombstoneCounts = NULL; - if (stats->level_tombstone_counts != NULL) + jclass stringClass = (*env)->FindClass(env, "java/lang/String"); + jobjectArray array = NULL; + if (stringClass != NULL && count >= 0 && count <= JSIZE_MAX) { - tombstoneCounts = malloc(stats->num_levels * sizeof(jlong)); - if (tombstoneCounts == NULL) - { - tidesdb_free_stats(stats); - return NULL; - } - for (int i = 0; i < stats->num_levels; i++) - { - tombstoneCounts[i] = (jlong)stats->level_tombstone_counts[i]; - } - (*env)->SetLongArrayRegion(env, levelTombstoneCounts, 0, stats->num_levels, - tombstoneCounts); - free(tombstoneCounts); - tombstoneCounts = NULL; + array = (*env)->NewObjectArray(env, (jsize)count, stringClass, NULL); } - /* Build ColumnFamilyConfig from stats->config so callers can round-trip CF settings */ - jobject cfConfigObj = NULL; - if (stats->config != NULL) + if (array != NULL) { - cfConfigObj = buildCfConfigObject(env, stats->config); - if (cfConfigObj == NULL) + for (int i = 0; i < count; i++) { - tidesdb_free_stats(stats); - return NULL; + jstring element = (*env)->NewStringUTF(env, names[i] != NULL ? names[i] : ""); + if (element == NULL) + { + array = NULL; + break; + } + (*env)->SetObjectArrayElement(env, array, (jsize)i, element); + (*env)->DeleteLocalRef(env, element); } } - jclass statsClass = (*env)->FindClass(env, "com/tidesdb/Stats"); - if (statsClass == NULL) - { - tidesdb_free_stats(stats); - return NULL; - } - - jmethodID constructor = - (*env)->GetMethodID(env, statsClass, "", - "(IJ[J[ILcom/tidesdb/ColumnFamilyConfig;JJDD[JDDZJIDJD[JDIJJJJJJJ)V"); - if (constructor == NULL) + for (int i = 0; i < count; i++) { - tidesdb_free_stats(stats); - return NULL; + tidesdb_free(names[i]); } + tidesdb_free(names); - jobject statsObj = (*env)->NewObject( - env, statsClass, constructor, stats->num_levels, (jlong)stats->memtable_size, levelSizes, - levelNumSSTables, cfConfigObj, (jlong)stats->total_keys, (jlong)stats->total_data_size, - stats->avg_key_size, stats->avg_value_size, levelKeyCounts, stats->read_amp, - stats->hit_rate, stats->use_btree != 0, (jlong)stats->btree_total_nodes, - (jint)stats->btree_max_height, stats->btree_avg_height, (jlong)stats->total_tombstones, - (jdouble)stats->tombstone_ratio, levelTombstoneCounts, (jdouble)stats->max_sst_density, - (jint)stats->max_sst_density_level, (jlong)stats->wal_bytes_written, - (jlong)stats->flush_bytes_written, (jlong)stats->compaction_bytes_written, - (jlong)stats->compaction_bytes_read, (jlong)stats->user_bytes_written, - (jlong)stats->flush_count, (jlong)stats->compaction_count); - - tidesdb_free_stats(stats); - - return statsObj; + if (stringClass != NULL) (*env)->DeleteLocalRef(env, stringClass); + return array; } -JNIEXPORT void JNICALL Java_com_tidesdb_ColumnFamily_nativeCompact(JNIEnv *env, jclass cls, - jlong handle) +/* ===== com.tidesdb.TidesDB : transactions ===== */ + +JNIEXPORT jlong JNICALL Java_com_tidesdb_TidesDB_nativeBeginTransaction(JNIEnv *env, jclass cls, + jlong handle) { - tidesdb_column_family_t *cf = (tidesdb_column_family_t *)(uintptr_t)handle; - int result = tidesdb_compact(cf); + (void)cls; + tidesdb_txn_t *txn = NULL; + int result = tidesdb_txn_begin((tidesdb_t *)(uintptr_t)handle, &txn); if (result != TDB_SUCCESS) { - throwTidesDBException(env, result, getErrorMessage(result)); + throwResult(env, result); + return 0; } + return (jlong)(uintptr_t)txn; } -JNIEXPORT void JNICALL Java_com_tidesdb_ColumnFamily_nativeCompactRange(JNIEnv *env, jclass cls, - jlong handle, - jbyteArray startKey, - jbyteArray endKey) +JNIEXPORT jlong JNICALL Java_com_tidesdb_TidesDB_nativeBeginTransactionWithIsolation( + JNIEnv *env, jclass cls, jlong handle, jint isolationLevel) { - tidesdb_column_family_t *cf = (tidesdb_column_family_t *)(uintptr_t)handle; - - /* Map null/empty byte arrays to NULL pointers for unbounded endpoints. The C API - rejects both-NULL with TDB_ERR_INVALID_ARGS, so we don't need to filter here. */ - jbyte *startBytes = NULL; - jsize startLen = 0; - if (startKey != NULL) - { - startLen = (*env)->GetArrayLength(env, startKey); - if (startLen > 0) - { - startBytes = (*env)->GetByteArrayElements(env, startKey, NULL); - if (startBytes == NULL) return; /* JVM exception already pending */ - } - } + (void)cls; - jbyte *endBytes = NULL; - jsize endLen = 0; - if (endKey != NULL) + tidesdb_txn_t *txn = NULL; + int result = tidesdb_txn_begin_with_isolation( + (tidesdb_t *)(uintptr_t)handle, (tidesdb_isolation_level_t)isolationLevel, &txn); + if (result != TDB_SUCCESS) { - endLen = (*env)->GetArrayLength(env, endKey); - if (endLen > 0) - { - endBytes = (*env)->GetByteArrayElements(env, endKey, NULL); - if (endBytes == NULL) - { - if (startBytes != NULL) - (*env)->ReleaseByteArrayElements(env, startKey, startBytes, JNI_ABORT); - return; /* JVM exception already pending */ - } - } + throwResult(env, result); + return 0; } - - int result = tidesdb_compact_range(cf, (const uint8_t *)startBytes, (size_t)startLen, - (const uint8_t *)endBytes, (size_t)endLen); - - if (startBytes != NULL) (*env)->ReleaseByteArrayElements(env, startKey, startBytes, JNI_ABORT); - if (endBytes != NULL) (*env)->ReleaseByteArrayElements(env, endKey, endBytes, JNI_ABORT); - - if (result != TDB_SUCCESS && !jvm_exception_pending(env)) - throwTidesDBException(env, result, getErrorMessage(result)); + return (jlong)(uintptr_t)txn; } -JNIEXPORT void JNICALL Java_com_tidesdb_ColumnFamily_nativeFlushMemtable(JNIEnv *env, jclass cls, - jlong handle) +JNIEXPORT jlong JNICALL Java_com_tidesdb_TidesDB_nativeBeginTransactionCf(JNIEnv *env, jclass cls, + jlong handle, + jlong cfHandle) { - tidesdb_column_family_t *cf = (tidesdb_column_family_t *)(uintptr_t)handle; - int result = tidesdb_flush_memtable(cf); + (void)cls; + tidesdb_txn_t *txn = NULL; + int result = tidesdb_txn_begin_cf((tidesdb_t *)(uintptr_t)handle, + (tidesdb_column_family_t *)(uintptr_t)cfHandle, &txn); if (result != TDB_SUCCESS) { - throwTidesDBException(env, result, getErrorMessage(result)); + throwResult(env, result); + return 0; } + return (jlong)(uintptr_t)txn; } -JNIEXPORT jboolean JNICALL Java_com_tidesdb_ColumnFamily_nativeIsFlushing(JNIEnv *env, jclass cls, - jlong handle) +JNIEXPORT jlong JNICALL Java_com_tidesdb_TidesDB_nativeSnapshotCreate(JNIEnv *env, jclass cls, + jlong handle) { - tidesdb_column_family_t *cf = (tidesdb_column_family_t *)(uintptr_t)handle; - return tidesdb_is_flushing(cf) != 0; -} + (void)cls; -JNIEXPORT jboolean JNICALL Java_com_tidesdb_ColumnFamily_nativeIsCompacting(JNIEnv *env, jclass cls, - jlong handle) -{ - tidesdb_column_family_t *cf = (tidesdb_column_family_t *)(uintptr_t)handle; - return tidesdb_is_compacting(cf) != 0; + tidesdb_snapshot_t *snapshot = NULL; + int result = tidesdb_snapshot_create((tidesdb_t *)(uintptr_t)handle, &snapshot); + if (result != TDB_SUCCESS) + { + throwResult(env, result); + return 0; + } + return (jlong)(uintptr_t)snapshot; } -JNIEXPORT void JNICALL Java_com_tidesdb_ColumnFamily_nativeUpdateRuntimeConfig( - JNIEnv *env, jclass cls, jlong handle, jlong writeBufferSize, jint skipListMaxLevel, - jfloat skipListProbability, jdouble bloomFPR, jint indexSampleRatio, jint syncMode, - jlong syncIntervalUs, jboolean persistToDisk) +JNIEXPORT jlong JNICALL Java_com_tidesdb_TidesDB_nativeBeginTransactionAtSnapshot( + JNIEnv *env, jclass cls, jlong handle, jlong snapshotHandle) { - tidesdb_column_family_t *cf = (tidesdb_column_family_t *)(uintptr_t)handle; - - tidesdb_column_family_config_t config = {.write_buffer_size = (size_t)writeBufferSize, - .skip_list_max_level = skipListMaxLevel, - .skip_list_probability = skipListProbability, - .bloom_fpr = bloomFPR, - .index_sample_ratio = indexSampleRatio, - .sync_mode = syncMode, - .sync_interval_us = (uint64_t)syncIntervalUs}; - - int result = tidesdb_cf_update_runtime_config(cf, &config, persistToDisk ? 1 : 0); + (void)cls; + tidesdb_txn_t *txn = NULL; + int result = tidesdb_txn_begin_at_snapshot((tidesdb_t *)(uintptr_t)handle, + (tidesdb_snapshot_t *)(uintptr_t)snapshotHandle, + &txn); if (result != TDB_SUCCESS) { - throwTidesDBException(env, result, getErrorMessage(result)); + throwResult(env, result); + return 0; } + return (jlong)(uintptr_t)txn; } -JNIEXPORT void JNICALL Java_com_tidesdb_Transaction_nativePut(JNIEnv *env, jclass cls, jlong handle, - jlong cfHandle, jbyteArray key, - jbyteArray value, jlong ttl) +JNIEXPORT jlong JNICALL Java_com_tidesdb_TidesDB_nativeBeginTransactionAtSeq(JNIEnv *env, jclass cls, + jlong handle, + jlong seq) { - tidesdb_txn_t *txn = (tidesdb_txn_t *)(uintptr_t)handle; - tidesdb_column_family_t *cf = (tidesdb_column_family_t *)(uintptr_t)cfHandle; - - jsize keyLen = (*env)->GetArrayLength(env, key); - jsize valueLen = (*env)->GetArrayLength(env, value); - - jbyte *keyBytes = (*env)->GetByteArrayElements(env, key, NULL); - if (keyBytes == NULL) return; /* JVM exception already pending */ + (void)cls; - jbyte *valueBytes = (*env)->GetByteArrayElements(env, value, NULL); - if (valueBytes == NULL) + tidesdb_txn_t *txn = NULL; + int result = tidesdb_txn_begin_at_seq((tidesdb_t *)(uintptr_t)handle, (uint64_t)seq, &txn); + if (result != TDB_SUCCESS) { - (*env)->ReleaseByteArrayElements(env, key, keyBytes, JNI_ABORT); - return; /* JVM exception already pending */ + throwResult(env, result); + return 0; } - - int result = tidesdb_txn_put(txn, cf, (uint8_t *)keyBytes, keyLen, (uint8_t *)valueBytes, - valueLen, (time_t)ttl); - - (*env)->ReleaseByteArrayElements(env, key, keyBytes, JNI_ABORT); - (*env)->ReleaseByteArrayElements(env, value, valueBytes, JNI_ABORT); - - if (result != TDB_SUCCESS && !jvm_exception_pending(env)) - throwTidesDBException(env, result, getErrorMessage(result)); + return (jlong)(uintptr_t)txn; } -JNIEXPORT jbyteArray JNICALL Java_com_tidesdb_Transaction_nativeGet(JNIEnv *env, jclass cls, - jlong handle, jlong cfHandle, - jbyteArray key) +JNIEXPORT jlong JNICALL Java_com_tidesdb_TidesDB_nativeOldestReadableSeq(JNIEnv *env, jclass cls, + jlong handle) { - tidesdb_txn_t *txn = (tidesdb_txn_t *)(uintptr_t)handle; - tidesdb_column_family_t *cf = (tidesdb_column_family_t *)(uintptr_t)cfHandle; - - jsize keyLen = (*env)->GetArrayLength(env, key); - jbyte *keyBytes = (*env)->GetByteArrayElements(env, key, NULL); - if (keyBytes == NULL) return NULL; /* JVM exception already pending */ - - uint8_t *value = NULL; - size_t valueLen = 0; + (void)env; + (void)cls; + return (jlong)tidesdb_oldest_readable_seq((const tidesdb_t *)(uintptr_t)handle); +} - int result = tidesdb_txn_get(txn, cf, (uint8_t *)keyBytes, keyLen, &value, &valueLen); +JNIEXPORT jobjectArray JNICALL Java_com_tidesdb_TidesDB_nativeRecoverPrepared(JNIEnv *env, + jclass cls, + jlong handle) +{ + (void)cls; - (*env)->ReleaseByteArrayElements(env, key, keyBytes, JNI_ABORT); + tidesdb_t *db = (tidesdb_t *)(uintptr_t)handle; + /* the set is fixed when the database opens, so size it first and then fill it */ + int count = 0; + int result = tidesdb_recover_prepared(db, NULL, 0, &count); if (result != TDB_SUCCESS) { - if (!jvm_exception_pending(env)) - throwTidesDBException(env, result, getErrorMessage(result)); - if (value != NULL) free(value); + throwResult(env, result); return NULL; } - if (valueLen > (size_t)JSIZE_MAX) + jclass preparedClass = (*env)->FindClass(env, "com/tidesdb/PreparedTransaction"); + if (preparedClass == NULL) return NULL; + + if (count <= 0) + { + jobjectArray empty = (*env)->NewObjectArray(env, 0, preparedClass, NULL); + (*env)->DeleteLocalRef(env, preparedClass); + return empty; + } + + tidesdb_prepared_txn_t *entries = + (tidesdb_prepared_txn_t *)calloc((size_t)count, sizeof(tidesdb_prepared_txn_t)); + if (entries == NULL) { - free(value); - (*env)->ThrowNew(env, (*env)->FindClass(env, "java/lang/ArrayIndexOutOfBoundsException"), - "value exceeds maximum Java array size"); + (*env)->DeleteLocalRef(env, preparedClass); + throwTidesDBException(env, TDB_ERR_MEMORY, "Failed to allocate the recovery buffer"); return NULL; } - jbyteArray resultArray = (*env)->NewByteArray(env, (jsize)valueLen); - if (resultArray == NULL) + int filled = 0; + result = tidesdb_recover_prepared(db, entries, count, &filled); + if (result != TDB_SUCCESS) { - free(value); - return NULL; /* JVM exception (OOM) already pending */ + free(entries); + (*env)->DeleteLocalRef(env, preparedClass); + throwResult(env, result); + return NULL; } + if (filled > count) filled = count; - (*env)->SetByteArrayRegion(env, resultArray, 0, (jsize)valueLen, (jbyte *)value); - free(value); + jclass txnClass = (*env)->FindClass(env, "com/tidesdb/Transaction"); + jmethodID txnCtor = + txnClass != NULL ? (*env)->GetMethodID(env, txnClass, "", "(J)V") : NULL; + jmethodID preparedCtor = + (*env)->GetMethodID(env, preparedClass, "", "(Lcom/tidesdb/Transaction;[B)V"); - return resultArray; -} + jobjectArray array = NULL; + if (txnCtor != NULL && preparedCtor != NULL) + { + array = (*env)->NewObjectArray(env, (jsize)filled, preparedClass, NULL); + } -JNIEXPORT void JNICALL Java_com_tidesdb_Transaction_nativeDelete(JNIEnv *env, jclass cls, - jlong handle, jlong cfHandle, - jbyteArray key) -{ - tidesdb_txn_t *txn = (tidesdb_txn_t *)(uintptr_t)handle; - tidesdb_column_family_t *cf = (tidesdb_column_family_t *)(uintptr_t)cfHandle; + if (array != NULL) + { + for (int i = 0; i < filled; i++) + { + jobject txn = + (*env)->NewObject(env, txnClass, txnCtor, (jlong)(uintptr_t)entries[i].txn); + if (txn == NULL) + { + array = NULL; + break; + } - jsize keyLen = (*env)->GetArrayLength(env, key); - jbyte *keyBytes = (*env)->GetByteArrayElements(env, key, NULL); - if (keyBytes == NULL) return; /* JVM exception already pending */ + jsize xidSize = entries[i].xid_size > (size_t)JSIZE_MAX ? JSIZE_MAX + : (jsize)entries[i].xid_size; + jbyteArray xid = (*env)->NewByteArray(env, xidSize); + if (xid == NULL) + { + (*env)->DeleteLocalRef(env, txn); + array = NULL; + break; + } + if (xidSize > 0) + { + (*env)->SetByteArrayRegion(env, xid, 0, xidSize, (const jbyte *)entries[i].xid); + } - int result = tidesdb_txn_delete(txn, cf, (uint8_t *)keyBytes, keyLen); + jobject prepared = (*env)->NewObject(env, preparedClass, preparedCtor, txn, xid); + if (prepared == NULL) + { + (*env)->DeleteLocalRef(env, xid); + (*env)->DeleteLocalRef(env, txn); + array = NULL; + break; + } - (*env)->ReleaseByteArrayElements(env, key, keyBytes, JNI_ABORT); + (*env)->SetObjectArrayElement(env, array, (jsize)i, prepared); + (*env)->DeleteLocalRef(env, prepared); + (*env)->DeleteLocalRef(env, xid); + (*env)->DeleteLocalRef(env, txn); + } + } - if (result != TDB_SUCCESS && !jvm_exception_pending(env)) - throwTidesDBException(env, result, getErrorMessage(result)); + free(entries); + if (txnClass != NULL) (*env)->DeleteLocalRef(env, txnClass); + (*env)->DeleteLocalRef(env, preparedClass); + return array; } -JNIEXPORT void JNICALL Java_com_tidesdb_Transaction_nativeSingleDelete(JNIEnv *env, jclass cls, - jlong handle, jlong cfHandle, - jbyteArray key) -{ - tidesdb_txn_t *txn = (tidesdb_txn_t *)(uintptr_t)handle; - tidesdb_column_family_t *cf = (tidesdb_column_family_t *)(uintptr_t)cfHandle; - - jsize keyLen = (*env)->GetArrayLength(env, key); - jbyte *keyBytes = (*env)->GetByteArrayElements(env, key, NULL); - if (keyBytes == NULL) return; /* JVM exception already pending */ - - int result = tidesdb_txn_single_delete(txn, cf, (uint8_t *)keyBytes, keyLen); - - (*env)->ReleaseByteArrayElements(env, key, keyBytes, JNI_ABORT); - - if (result != TDB_SUCCESS && !jvm_exception_pending(env)) - throwTidesDBException(env, result, getErrorMessage(result)); -} +/* ===== com.tidesdb.TidesDB : maintenance ===== */ -JNIEXPORT void JNICALL Java_com_tidesdb_Transaction_nativeCommit(JNIEnv *env, jclass cls, - jlong handle) +JNIEXPORT void JNICALL Java_com_tidesdb_TidesDB_nativeFlushMemtable(JNIEnv *env, jclass cls, + jlong handle) { - tidesdb_txn_t *txn = (tidesdb_txn_t *)(uintptr_t)handle; - int result = tidesdb_txn_commit(txn); - + (void)cls; + int result = tidesdb_flush_memtable((tidesdb_t *)(uintptr_t)handle); if (result != TDB_SUCCESS) { - throwTidesDBException(env, result, getErrorMessage(result)); + throwResult(env, result); } } -JNIEXPORT void JNICALL Java_com_tidesdb_Transaction_nativeRollback(JNIEnv *env, jclass cls, - jlong handle) +JNIEXPORT jboolean JNICALL Java_com_tidesdb_TidesDB_nativeIsFlushing(JNIEnv *env, jclass cls, + jlong handle) { - tidesdb_txn_t *txn = (tidesdb_txn_t *)(uintptr_t)handle; - int result = tidesdb_txn_rollback(txn); + (void)env; + (void)cls; + return tidesdb_is_flushing((tidesdb_t *)(uintptr_t)handle) ? JNI_TRUE : JNI_FALSE; +} +JNIEXPORT void JNICALL Java_com_tidesdb_TidesDB_nativeSyncWal(JNIEnv *env, jclass cls, jlong handle) +{ + (void)cls; + int result = tidesdb_sync_wal((tidesdb_t *)(uintptr_t)handle); if (result != TDB_SUCCESS) { - throwTidesDBException(env, result, getErrorMessage(result)); + throwResult(env, result); } } -JNIEXPORT void JNICALL Java_com_tidesdb_Transaction_nativeSavepoint(JNIEnv *env, jclass cls, - jlong handle, jstring name) +JNIEXPORT void JNICALL Java_com_tidesdb_TidesDB_nativeBackup(JNIEnv *env, jclass cls, jlong handle, + jstring dir) { - tidesdb_txn_t *txn = (tidesdb_txn_t *)(uintptr_t)handle; - const char *spName = (*env)->GetStringUTFChars(env, name, NULL); + (void)cls; - int result = tidesdb_txn_savepoint(txn, spName); + const char *path = (*env)->GetStringUTFChars(env, dir, NULL); + if (path == NULL) + { + if (!jvm_exception_pending(env)) + { + throwTidesDBException(env, TDB_ERR_MEMORY, "Failed to read the backup directory"); + } + return; + } - (*env)->ReleaseStringUTFChars(env, name, spName); + int result = tidesdb_backup((tidesdb_t *)(uintptr_t)handle, path); + (*env)->ReleaseStringUTFChars(env, dir, path); if (result != TDB_SUCCESS) { - throwTidesDBException(env, result, getErrorMessage(result)); + throwResult(env, result); } } -JNIEXPORT void JNICALL Java_com_tidesdb_Transaction_nativeRollbackToSavepoint(JNIEnv *env, - jclass cls, - jlong handle, - jstring name) +JNIEXPORT void JNICALL Java_com_tidesdb_TidesDB_nativeCheckpoint(JNIEnv *env, jclass cls, + jlong handle) { - tidesdb_txn_t *txn = (tidesdb_txn_t *)(uintptr_t)handle; - const char *spName = (*env)->GetStringUTFChars(env, name, NULL); - - int result = tidesdb_txn_rollback_to_savepoint(txn, spName); - - (*env)->ReleaseStringUTFChars(env, name, spName); - + (void)cls; + int result = tidesdb_checkpoint((tidesdb_t *)(uintptr_t)handle); if (result != TDB_SUCCESS) { - throwTidesDBException(env, result, getErrorMessage(result)); + throwResult(env, result); } } -JNIEXPORT void JNICALL Java_com_tidesdb_Transaction_nativeReleaseSavepoint(JNIEnv *env, jclass cls, - jlong handle, - jstring name) -{ - tidesdb_txn_t *txn = (tidesdb_txn_t *)(uintptr_t)handle; - const char *spName = (*env)->GetStringUTFChars(env, name, NULL); - - int result = tidesdb_txn_release_savepoint(txn, spName); +/* ===== com.tidesdb.TidesDB : statistics ===== */ - (*env)->ReleaseStringUTFChars(env, name, spName); +JNIEXPORT jobject JNICALL Java_com_tidesdb_TidesDB_nativeGetDbStats(JNIEnv *env, jclass cls, + jlong handle) +{ + (void)cls; + tidesdb_db_stats_t stats; + memset(&stats, 0, sizeof(stats)); + int result = tidesdb_get_db_stats((tidesdb_t *)(uintptr_t)handle, &stats); if (result != TDB_SUCCESS) { - throwTidesDBException(env, result, getErrorMessage(result)); + throwResult(env, result); + return NULL; } + return buildDbStatsObject(env, &stats); } -JNIEXPORT jlong JNICALL Java_com_tidesdb_Transaction_nativeNewIterator(JNIEnv *env, jclass cls, - jlong handle, jlong cfHandle) +JNIEXPORT jobject JNICALL Java_com_tidesdb_TidesDB_nativeGetCacheStats(JNIEnv *env, jclass cls, + jlong handle) { - tidesdb_txn_t *txn = (tidesdb_txn_t *)(uintptr_t)handle; - tidesdb_column_family_t *cf = (tidesdb_column_family_t *)(uintptr_t)cfHandle; - tidesdb_iter_t *iter = NULL; + (void)cls; - int result = tidesdb_iter_new(txn, cf, &iter); + tidesdb_cache_stats_t stats; + memset(&stats, 0, sizeof(stats)); + int result = tidesdb_get_cache_stats((tidesdb_t *)(uintptr_t)handle, &stats); if (result != TDB_SUCCESS) { - throwTidesDBException(env, result, getErrorMessage(result)); - return 0; + throwResult(env, result); + return NULL; } - return (jlong)(uintptr_t)iter; + jclass cls_ = (*env)->FindClass(env, "com/tidesdb/CacheStats"); + if (cls_ == NULL) return NULL; + + jmethodID ctor = (*env)->GetMethodID(env, cls_, "", "(ZJJJJDJ)V"); + if (ctor == NULL) + { + (*env)->DeleteLocalRef(env, cls_); + return NULL; + } + + jobject result_obj = (*env)->NewObject( + env, cls_, ctor, stats.enabled ? JNI_TRUE : JNI_FALSE, (jlong)stats.total_entries, + (jlong)stats.total_bytes, (jlong)stats.hits, (jlong)stats.misses, (jdouble)stats.hit_rate, + (jlong)stats.num_partitions); + + (*env)->DeleteLocalRef(env, cls_); + return result_obj; } -JNIEXPORT void JNICALL Java_com_tidesdb_Transaction_nativeReset(JNIEnv *env, jclass cls, - jlong handle, jint isolationLevel) +JNIEXPORT jobject JNICALL Java_com_tidesdb_TidesDB_nativeGetStallStats(JNIEnv *env, jclass cls, + jlong handle) { - tidesdb_txn_t *txn = (tidesdb_txn_t *)(uintptr_t)handle; - int result = tidesdb_txn_reset(txn, (tidesdb_isolation_level_t)isolationLevel); + (void)cls; + tidesdb_stall_stats_t stats; + memset(&stats, 0, sizeof(stats)); + int result = tidesdb_get_stall_stats((tidesdb_t *)(uintptr_t)handle, &stats); if (result != TDB_SUCCESS) { - throwTidesDBException(env, result, getErrorMessage(result)); + throwResult(env, result); + return NULL; } -} -JNIEXPORT void JNICALL Java_com_tidesdb_Transaction_nativeFree(JNIEnv *env, jclass cls, - jlong handle) -{ - tidesdb_txn_t *txn = (tidesdb_txn_t *)(uintptr_t)handle; - if (txn != NULL) + jclass statClass = (*env)->FindClass(env, "com/tidesdb/StallStat"); + if (statClass == NULL) return NULL; + + jmethodID statCtor = (*env)->GetMethodID(env, statClass, "", "(JJJ)V"); + jobjectArray array = + statCtor != NULL ? (*env)->NewObjectArray(env, TDB_STALL_COUNT, statClass, NULL) : NULL; + + if (array != NULL) { - tidesdb_txn_free(txn); + for (int i = 0; i < TDB_STALL_COUNT; i++) + { + jobject stat = (*env)->NewObject(env, statClass, statCtor, + (jlong)stats.reasons[i].count, + (jlong)stats.reasons[i].total_us, + (jlong)stats.reasons[i].max_us); + if (stat == NULL) + { + array = NULL; + break; + } + (*env)->SetObjectArrayElement(env, array, (jsize)i, stat); + (*env)->DeleteLocalRef(env, stat); + } } + (*env)->DeleteLocalRef(env, statClass); + if (array == NULL) return NULL; + + jclass statsClass = (*env)->FindClass(env, "com/tidesdb/StallStats"); + if (statsClass == NULL) return NULL; + + jmethodID statsCtor = + (*env)->GetMethodID(env, statsClass, "", "([Lcom/tidesdb/StallStat;)V"); + jobject result_obj = + statsCtor != NULL ? (*env)->NewObject(env, statsClass, statsCtor, array) : NULL; + + (*env)->DeleteLocalRef(env, statsClass); + return result_obj; } -JNIEXPORT void JNICALL Java_com_tidesdb_TidesDBIterator_nativeSeekToFirst(JNIEnv *env, jclass cls, - jlong handle) +JNIEXPORT jobject JNICALL Java_com_tidesdb_TidesDB_nativeGetIoStats(JNIEnv *env, jclass cls, + jlong handle) { - tidesdb_iter_t *iter = (tidesdb_iter_t *)(uintptr_t)handle; - int result = tidesdb_iter_seek_to_first(iter); + (void)cls; + tidesdb_io_stats_t stats; + memset(&stats, 0, sizeof(stats)); + int result = tidesdb_get_io_stats((tidesdb_t *)(uintptr_t)handle, &stats); if (result != TDB_SUCCESS) { - throwTidesDBException(env, result, getErrorMessage(result)); + throwResult(env, result); + return NULL; } -} -JNIEXPORT void JNICALL Java_com_tidesdb_TidesDBIterator_nativeSeekToLast(JNIEnv *env, jclass cls, - jlong handle) -{ - tidesdb_iter_t *iter = (tidesdb_iter_t *)(uintptr_t)handle; - int result = tidesdb_iter_seek_to_last(iter); + jclass statClass = (*env)->FindClass(env, "com/tidesdb/IoStat"); + if (statClass == NULL) return NULL; - if (result != TDB_SUCCESS) + jmethodID statCtor = (*env)->GetMethodID(env, statClass, "", "(JJJJ)V"); + jobjectArray array = + statCtor != NULL ? (*env)->NewObjectArray(env, TDB_IO_COUNT, statClass, NULL) : NULL; + + if (array != NULL) { - throwTidesDBException(env, result, getErrorMessage(result)); + for (int i = 0; i < TDB_IO_COUNT; i++) + { + jobject stat = (*env)->NewObject(env, statClass, statCtor, (jlong)stats.classes[i].ops, + (jlong)stats.classes[i].bytes, + (jlong)stats.classes[i].total_us, + (jlong)stats.classes[i].max_us); + if (stat == NULL) + { + array = NULL; + break; + } + (*env)->SetObjectArrayElement(env, array, (jsize)i, stat); + (*env)->DeleteLocalRef(env, stat); + } } -} - -JNIEXPORT void JNICALL Java_com_tidesdb_TidesDBIterator_nativeSeek(JNIEnv *env, jclass cls, - jlong handle, jbyteArray key) -{ - tidesdb_iter_t *iter = (tidesdb_iter_t *)(uintptr_t)handle; - jsize keyLen = (*env)->GetArrayLength(env, key); - jbyte *keyBytes = (*env)->GetByteArrayElements(env, key, NULL); - if (keyBytes == NULL) return; /* JVM exception already pending */ + (*env)->DeleteLocalRef(env, statClass); + if (array == NULL) return NULL; - int result = tidesdb_iter_seek(iter, (uint8_t *)keyBytes, keyLen); + jclass statsClass = (*env)->FindClass(env, "com/tidesdb/IoStats"); + if (statsClass == NULL) return NULL; - (*env)->ReleaseByteArrayElements(env, key, keyBytes, JNI_ABORT); + jmethodID statsCtor = (*env)->GetMethodID(env, statsClass, "", "([Lcom/tidesdb/IoStat;)V"); + jobject result_obj = + statsCtor != NULL ? (*env)->NewObject(env, statsClass, statsCtor, array) : NULL; - if (result != TDB_SUCCESS && !jvm_exception_pending(env)) - throwTidesDBException(env, result, getErrorMessage(result)); + (*env)->DeleteLocalRef(env, statsClass); + return result_obj; } -JNIEXPORT void JNICALL Java_com_tidesdb_TidesDBIterator_nativeSeekForPrev(JNIEnv *env, jclass cls, - jlong handle, - jbyteArray key) +/** + * Shared body for the key-log and value-log encoding stats calls, which differ + * only in which collector they run. + */ +static jobjectArray buildEncodingStatsArray(JNIEnv *env, tidesdb_encoding_stats_t *entries, + size_t count) { - tidesdb_iter_t *iter = (tidesdb_iter_t *)(uintptr_t)handle; - jsize keyLen = (*env)->GetArrayLength(env, key); - jbyte *keyBytes = (*env)->GetByteArrayElements(env, key, NULL); - if (keyBytes == NULL) return; /* JVM exception already pending */ + jclass cls = (*env)->FindClass(env, "com/tidesdb/EncodingStats"); + if (cls == NULL) return NULL; - int result = tidesdb_iter_seek_for_prev(iter, (uint8_t *)keyBytes, keyLen); + jmethodID ctor = (*env)->GetMethodID(env, cls, "", "([IJJJ)V"); + jobjectArray array = NULL; + if (ctor != NULL && count <= (size_t)JSIZE_MAX) + { + array = (*env)->NewObjectArray(env, (jsize)count, cls, NULL); + } - (*env)->ReleaseByteArrayElements(env, key, keyBytes, JNI_ABORT); + if (array != NULL) + { + for (size_t i = 0; i < count; i++) + { + int idCount = entries[i].id_count; + if (idCount < 0) idCount = 0; + if (idCount > TDB_ENCODING_PIPELINE_MAX) idCount = TDB_ENCODING_PIPELINE_MAX; - if (result != TDB_SUCCESS && !jvm_exception_pending(env)) - throwTidesDBException(env, result, getErrorMessage(result)); + jintArray ids = newIntArrayFromBytes(env, entries[i].ids, (jsize)idCount); + if (ids == NULL) + { + array = NULL; + break; + } + + jobject stat = + (*env)->NewObject(env, cls, ctor, ids, (jlong)entries[i].logical_bytes, + (jlong)entries[i].stored_bytes, (jlong)entries[i].item_count); + if (stat == NULL) + { + (*env)->DeleteLocalRef(env, ids); + array = NULL; + break; + } + + (*env)->SetObjectArrayElement(env, array, (jsize)i, stat); + (*env)->DeleteLocalRef(env, stat); + (*env)->DeleteLocalRef(env, ids); + } + } + + (*env)->DeleteLocalRef(env, cls); + return array; } -JNIEXPORT jboolean JNICALL Java_com_tidesdb_TidesDBIterator_nativeValid(JNIEnv *env, jclass cls, - jlong handle) +JNIEXPORT jobjectArray JNICALL Java_com_tidesdb_TidesDB_nativeGetKlogEncodingStats(JNIEnv *env, + jclass cls, + jlong handle) { - tidesdb_iter_t *iter = (tidesdb_iter_t *)(uintptr_t)handle; - return tidesdb_iter_valid(iter) != 0; + (void)cls; + + tidesdb_encoding_stats_t entries[JNI_MAX_ENCODING_CHAINS]; + memset(entries, 0, sizeof(entries)); + size_t count = 0; + int result = tidesdb_get_klog_encoding_stats((tidesdb_t *)(uintptr_t)handle, entries, + JNI_MAX_ENCODING_CHAINS, &count); + if (result != TDB_SUCCESS) + { + throwResult(env, result); + return NULL; + } + if (count > JNI_MAX_ENCODING_CHAINS) count = JNI_MAX_ENCODING_CHAINS; + return buildEncodingStatsArray(env, entries, count); } -JNIEXPORT void JNICALL Java_com_tidesdb_TidesDBIterator_nativeNext(JNIEnv *env, jclass cls, - jlong handle) +JNIEXPORT jobjectArray JNICALL Java_com_tidesdb_TidesDB_nativeGetVlogEncodingStats(JNIEnv *env, + jclass cls, + jlong handle) { - tidesdb_iter_t *iter = (tidesdb_iter_t *)(uintptr_t)handle; - int result = tidesdb_iter_next(iter); + (void)cls; - /* TDB_ERR_NOT_FOUND is expected when reaching end of iteration -- iterator becomes invalid */ - if (result != TDB_SUCCESS && result != TDB_ERR_NOT_FOUND) + tidesdb_encoding_stats_t entries[JNI_MAX_ENCODING_CHAINS]; + memset(entries, 0, sizeof(entries)); + size_t count = 0; + int result = tidesdb_get_vlog_encoding_stats((tidesdb_t *)(uintptr_t)handle, entries, + JNI_MAX_ENCODING_CHAINS, &count); + if (result != TDB_SUCCESS) { - throwTidesDBException(env, result, getErrorMessage(result)); + throwResult(env, result); + return NULL; } + if (count > JNI_MAX_ENCODING_CHAINS) count = JNI_MAX_ENCODING_CHAINS; + return buildEncodingStatsArray(env, entries, count); } -JNIEXPORT void JNICALL Java_com_tidesdb_TidesDBIterator_nativePrev(JNIEnv *env, jclass cls, - jlong handle) +/* ===== com.tidesdb.ColumnFamily ===== */ + +JNIEXPORT jobject JNICALL Java_com_tidesdb_ColumnFamily_nativeGetStats(JNIEnv *env, jclass cls, + jlong cfHandle) { - tidesdb_iter_t *iter = (tidesdb_iter_t *)(uintptr_t)handle; - int result = tidesdb_iter_prev(iter); + (void)cls; - /* TDB_ERR_NOT_FOUND is expected when reaching start of iteration -- iterator becomes invalid */ - if (result != TDB_SUCCESS && result != TDB_ERR_NOT_FOUND) + tidesdb_cf_stats_t stats; + memset(&stats, 0, sizeof(stats)); + int result = tidesdb_get_cf_stats((tidesdb_column_family_t *)(uintptr_t)cfHandle, &stats); + if (result != TDB_SUCCESS) { - throwTidesDBException(env, result, getErrorMessage(result)); + throwResult(env, result); + return NULL; } + return buildCfStatsObject(env, &stats); } -JNIEXPORT jbyteArray JNICALL Java_com_tidesdb_TidesDBIterator_nativeKey(JNIEnv *env, jclass cls, - jlong handle) +JNIEXPORT jlong JNICALL Java_com_tidesdb_ColumnFamily_nativeEstimateCardinality(JNIEnv *env, + jclass cls, + jlong cfHandle) { - tidesdb_iter_t *iter = (tidesdb_iter_t *)(uintptr_t)handle; - uint8_t *key = NULL; - size_t keyLen = 0; + (void)cls; - int result = tidesdb_iter_key(iter, &key, &keyLen); + uint64_t estimate = 0; + int result = tidesdb_cf_estimate_cardinality((tidesdb_column_family_t *)(uintptr_t)cfHandle, + &estimate); if (result != TDB_SUCCESS) { - if (!jvm_exception_pending(env)) - throwTidesDBException(env, result, getErrorMessage(result)); - return NULL; + throwResult(env, result); + return 0; } + return (jlong)estimate; +} - if (keyLen > (size_t)JSIZE_MAX) +JNIEXPORT void JNICALL Java_com_tidesdb_ColumnFamily_nativeCompact(JNIEnv *env, jclass cls, + jlong dbHandle, jlong cfHandle) +{ + (void)cls; + int result = tidesdb_compact((tidesdb_t *)(uintptr_t)dbHandle, + (tidesdb_column_family_t *)(uintptr_t)cfHandle); + if (result != TDB_SUCCESS) { - (*env)->ThrowNew(env, (*env)->FindClass(env, "java/lang/ArrayIndexOutOfBoundsException"), - "key exceeds maximum Java array size"); - return NULL; + throwResult(env, result); + } +} + +JNIEXPORT void JNICALL Java_com_tidesdb_ColumnFamily_nativeCompactRange(JNIEnv *env, jclass cls, + jlong dbHandle, + jlong cfHandle, + jbyteArray startKey, + jbyteArray endKey) +{ + (void)cls; + + jni_bytes_t start; + jni_bytes_t end; + if (acquireBytes(env, startKey, &start) != 0) return; + if (acquireBytes(env, endKey, &end) != 0) + { + releaseBytes(env, &start); + return; } - jbyteArray resultArray = (*env)->NewByteArray(env, (jsize)keyLen); - if (resultArray == NULL) return NULL; /* JVM exception (OOM) already pending */ + int result = tidesdb_compact_range( + (tidesdb_t *)(uintptr_t)dbHandle, (tidesdb_column_family_t *)(uintptr_t)cfHandle, + (const uint8_t *)start.data, (size_t)start.length, (const uint8_t *)end.data, + (size_t)end.length); - (*env)->SetByteArrayRegion(env, resultArray, 0, (jsize)keyLen, (jbyte *)key); + releaseBytes(env, &end); + releaseBytes(env, &start); - return resultArray; + if (result != TDB_SUCCESS) + { + throwResult(env, result); + } } -JNIEXPORT jbyteArray JNICALL Java_com_tidesdb_TidesDBIterator_nativeValue(JNIEnv *env, jclass cls, - jlong handle) +JNIEXPORT jboolean JNICALL Java_com_tidesdb_ColumnFamily_nativeIsCompacting(JNIEnv *env, jclass cls, + jlong cfHandle) { - tidesdb_iter_t *iter = (tidesdb_iter_t *)(uintptr_t)handle; - uint8_t *value = NULL; - size_t valueLen = 0; + (void)env; + (void)cls; + return tidesdb_is_compacting((tidesdb_column_family_t *)(uintptr_t)cfHandle) ? JNI_TRUE + : JNI_FALSE; +} + +JNIEXPORT void JNICALL Java_com_tidesdb_ColumnFamily_nativeUpdateRuntimeConfig( + JNIEnv *env, jclass cls, jlong dbHandle, jlong cfHandle, jlong levelSizeRatio, jint minLevels, + jint dividingLevelOffset, jboolean keepValuesInline, jlong btreeKlogBlockSize, + jintArray encodingPipeline, jboolean enableBloomFilter, jdouble bloomFpr, + jint defaultIsolationLevel, jint l1FileCountTrigger, jdouble tombstoneDensityTrigger, + jlong tombstoneDensityMinEntries, jboolean persistToDisk) +{ + (void)cls; + + tidesdb_column_family_config_t config; + if (fillCfConfig(env, &config, levelSizeRatio, minLevels, dividingLevelOffset, + keepValuesInline, btreeKlogBlockSize, encodingPipeline, enableBloomFilter, + bloomFpr, defaultIsolationLevel, l1FileCountTrigger, tombstoneDensityTrigger, + tombstoneDensityMinEntries) != 0) + { + return; + } + + int result = tidesdb_cf_update_runtime_config( + (tidesdb_t *)(uintptr_t)dbHandle, (tidesdb_column_family_t *)(uintptr_t)cfHandle, &config, + persistToDisk ? 1 : 0); - int result = tidesdb_iter_value(iter, &value, &valueLen); if (result != TDB_SUCCESS) { - if (!jvm_exception_pending(env)) - throwTidesDBException(env, result, getErrorMessage(result)); - return NULL; + throwResult(env, result); } +} - if (valueLen > (size_t)JSIZE_MAX) +JNIEXPORT jobject JNICALL Java_com_tidesdb_ColumnFamily_nativeRangeStats(JNIEnv *env, jclass cls, + jlong dbHandle, + jlong cfHandle, + jbyteArray keyA, + jbyteArray keyB) +{ + (void)cls; + + jni_bytes_t a; + jni_bytes_t b; + if (acquireBytes(env, keyA, &a) != 0) return NULL; + if (acquireBytes(env, keyB, &b) != 0) { - (*env)->ThrowNew(env, (*env)->FindClass(env, "java/lang/ArrayIndexOutOfBoundsException"), - "value exceeds maximum Java array size"); + releaseBytes(env, &a); return NULL; } - jbyteArray resultArray = (*env)->NewByteArray(env, (jsize)valueLen); - if (resultArray == NULL) return NULL; /* JVM exception (OOM) already pending */ + tidesdb_range_stats_t stats; + memset(&stats, 0, sizeof(stats)); + int result = tidesdb_range_stats( + (tidesdb_t *)(uintptr_t)dbHandle, (tidesdb_column_family_t *)(uintptr_t)cfHandle, + (const uint8_t *)a.data, (size_t)a.length, (const uint8_t *)b.data, (size_t)b.length, + &stats); - (*env)->SetByteArrayRegion(env, resultArray, 0, (jsize)valueLen, (jbyte *)value); + releaseBytes(env, &b); + releaseBytes(env, &a); - return resultArray; -} + if (result != TDB_SUCCESS) + { + throwResult(env, result); + return NULL; + } -JNIEXPORT void JNICALL Java_com_tidesdb_TidesDBIterator_nativeFree(JNIEnv *env, jclass cls, - jlong handle) -{ - tidesdb_iter_t *iter = (tidesdb_iter_t *)(uintptr_t)handle; - if (iter != NULL) + jclass statsClass = (*env)->FindClass(env, "com/tidesdb/RangeStats"); + if (statsClass == NULL) return NULL; + + jmethodID ctor = (*env)->GetMethodID(env, statsClass, "", "(JJZ)V"); + jobject result_obj = NULL; + if (ctor != NULL) { - tidesdb_iter_free(iter); + result_obj = (*env)->NewObject(env, statsClass, ctor, (jlong)stats.sstables_overlapping, + (jlong)stats.estimated_keys, + stats.keys_exact ? JNI_TRUE : JNI_FALSE); } + + (*env)->DeleteLocalRef(env, statsClass); + return result_obj; } +/* ===== commit hooks ===== */ + /** - * Context stored as the commit hook ctx pointer. - * Holds the JavaVM and a global reference to the Java CommitHook object. - * Uses reference-counted quiescent retirement to avoid use-after-free. + * Context stored as the commit hook ctx pointer. Holds the JavaVM and a global + * reference to the Java CommitHook object, with reference-counted quiescent + * retirement so a hook being replaced cannot be freed under a callback that is + * still inside it. */ typedef struct { JavaVM *jvm; jobject hook_obj; /* global reference to CommitHook */ - int refcount; /* callback reference count */ + int refcount; /* callbacks currently inside the trampoline */ int retired; /* 0 = active, 1 = retired (do not enter) */ pthread_mutex_t lock; pthread_cond_t zero_cond; } java_hook_ctx_t; /** - * Retires and destroys a hook context. Waits for in-flight callbacks to drain, - * then deletes the global reference and frees the context. - * Must only be called after the context has been detached from tidesdb (i.e., - * after tidesdb_cf_set_commit_hook has replaced it). + * Retires and destroys a hook context, waiting for in-flight callbacks to drain + * before deleting the global reference. Must only be called once the context has + * been detached from tidesdb, so no new callback can arrive. */ static void retire_and_destroy_hook_ctx(JNIEnv *env, java_hook_ctx_t *ctx) { @@ -1480,9 +1519,19 @@ static void retire_and_destroy_hook_ctx(JNIEnv *env, java_hook_ctx_t *ctx) free(ctx); } +/** Drops a trampoline's claim on the context and wakes a waiting retirement. */ +static void release_hook_ctx(java_hook_ctx_t *ctx) +{ + pthread_mutex_lock(&ctx->lock); + ctx->refcount--; + if (ctx->refcount == 0 && ctx->retired) pthread_cond_signal(&ctx->zero_cond); + pthread_mutex_unlock(&ctx->lock); +} + /** - * C trampoline that bridges the tidesdb_commit_hook_fn callback to the Java CommitHook.onCommit - * method. Fires synchronously on the committing thread (which is always a Java thread). + * Bridges tidesdb_commit_hook_fn to CommitHook.onCommit. Fires synchronously on + * the committing thread, which is normally a Java thread but is attached here + * anyway so an engine-internal caller is also safe. */ static int java_commit_hook_trampoline(const tidesdb_commit_op_t *ops, int num_ops, uint64_t commit_seq, void *ctx) @@ -1491,7 +1540,6 @@ static int java_commit_hook_trampoline(const tidesdb_commit_op_t *ops, int num_o JNIEnv *env = NULL; int need_detach = 0; - /* Check if context is retired under lock, increment refcount if active */ pthread_mutex_lock(&hctx->lock); if (hctx->retired) { @@ -1506,62 +1554,77 @@ static int java_commit_hook_trampoline(const tidesdb_commit_op_t *ops, int num_o { if ((*hctx->jvm)->AttachCurrentThread(hctx->jvm, (void **)&env, NULL) != 0) { - pthread_mutex_lock(&hctx->lock); - hctx->refcount--; - if (hctx->refcount == 0 && hctx->retired) pthread_cond_signal(&hctx->zero_cond); - pthread_mutex_unlock(&hctx->lock); + release_hook_ctx(hctx); return -1; } need_detach = 1; } else if (rc != JNI_OK) { - pthread_mutex_lock(&hctx->lock); - hctx->refcount--; - if (hctx->refcount == 0 && hctx->retired) pthread_cond_signal(&hctx->zero_cond); - pthread_mutex_unlock(&hctx->lock); + release_hook_ctx(hctx); return -1; } jint ret = -1; + jclass commitOpClass = NULL; + jobjectArray opsArray = NULL; + jclass hookClass = NULL; - /* Find CommitOp class and constructor: CommitOp(byte[], byte[], long, boolean) */ - jclass commitOpClass = (*env)->FindClass(env, "com/tidesdb/CommitOp"); + commitOpClass = (*env)->FindClass(env, "com/tidesdb/CommitOp"); if (commitOpClass == NULL) goto cleanup; - jmethodID ctor = (*env)->GetMethodID(env, commitOpClass, "", "([B[BJZ)V"); - if (ctor == NULL) goto cleanup; + jmethodID opCtor = (*env)->GetMethodID(env, commitOpClass, "", "([B[BJZ)V"); + if (opCtor == NULL) goto cleanup; - /* Create CommitOp[] array */ - jobjectArray opsArray = (*env)->NewObjectArray(env, num_ops, commitOpClass, NULL); + if (num_ops < 0) num_ops = 0; + opsArray = (*env)->NewObjectArray(env, (jsize)num_ops, commitOpClass, NULL); if (opsArray == NULL) goto cleanup; for (int i = 0; i < num_ops; i++) { jbyteArray jkey = (*env)->NewByteArray(env, (jsize)ops[i].key_size); - (*env)->SetByteArrayRegion(env, jkey, 0, (jsize)ops[i].key_size, (jbyte *)ops[i].key); + if (jkey == NULL) goto cleanup; + (*env)->SetByteArrayRegion(env, jkey, 0, (jsize)ops[i].key_size, + (const jbyte *)ops[i].key); jbyteArray jvalue = NULL; - if (ops[i].value != NULL && ops[i].value_size > 0) + if (ops[i].value != NULL) { jvalue = (*env)->NewByteArray(env, (jsize)ops[i].value_size); - (*env)->SetByteArrayRegion(env, jvalue, 0, (jsize)ops[i].value_size, - (jbyte *)ops[i].value); + if (jvalue == NULL) + { + (*env)->DeleteLocalRef(env, jkey); + goto cleanup; + } + if (ops[i].value_size > 0) + { + (*env)->SetByteArrayRegion(env, jvalue, 0, (jsize)ops[i].value_size, + (const jbyte *)ops[i].value); + } } - jobject opObj = (*env)->NewObject(env, commitOpClass, ctor, jkey, jvalue, (jlong)ops[i].ttl, + jobject opObj = (*env)->NewObject(env, commitOpClass, opCtor, jkey, jvalue, + (jlong)ops[i].ttl, ops[i].is_delete ? JNI_TRUE : JNI_FALSE); - (*env)->SetObjectArrayElement(env, opsArray, i, opObj); + if (opObj == NULL) + { + if (jvalue != NULL) (*env)->DeleteLocalRef(env, jvalue); + (*env)->DeleteLocalRef(env, jkey); + goto cleanup; + } + (*env)->SetObjectArrayElement(env, opsArray, i, opObj); (*env)->DeleteLocalRef(env, opObj); - (*env)->DeleteLocalRef(env, jkey); if (jvalue != NULL) (*env)->DeleteLocalRef(env, jvalue); + (*env)->DeleteLocalRef(env, jkey); } - /* Call CommitHook.onCommit(CommitOp[], long) */ - jclass hookClass = (*env)->GetObjectClass(env, hctx->hook_obj); + hookClass = (*env)->GetObjectClass(env, hctx->hook_obj); + if (hookClass == NULL) goto cleanup; + jmethodID onCommit = (*env)->GetMethodID(env, hookClass, "onCommit", "([Lcom/tidesdb/CommitOp;J)I"); + if (onCommit == NULL) goto cleanup; ret = (*env)->CallIntMethod(env, hctx->hook_obj, onCommit, opsArray, (jlong)commit_seq); @@ -1571,71 +1634,55 @@ static int java_commit_hook_trampoline(const tidesdb_commit_op_t *ops, int num_o ret = -1; } - (*env)->DeleteLocalRef(env, opsArray); - (*env)->DeleteLocalRef(env, commitOpClass); - (*env)->DeleteLocalRef(env, hookClass); - - if (need_detach) (*hctx->jvm)->DetachCurrentThread(hctx->jvm); - - /* Decrement refcount and signal if retiring */ - pthread_mutex_lock(&hctx->lock); - hctx->refcount--; - if (hctx->refcount == 0 && hctx->retired) pthread_cond_signal(&hctx->zero_cond); - pthread_mutex_unlock(&hctx->lock); - - return (int)ret; - cleanup: if ((*env)->ExceptionCheck(env)) (*env)->ExceptionClear(env); + if (hookClass != NULL) (*env)->DeleteLocalRef(env, hookClass); + if (opsArray != NULL) (*env)->DeleteLocalRef(env, opsArray); + if (commitOpClass != NULL) (*env)->DeleteLocalRef(env, commitOpClass); if (need_detach) (*hctx->jvm)->DetachCurrentThread(hctx->jvm); - /* Decrement refcount and signal if retiring */ - pthread_mutex_lock(&hctx->lock); - hctx->refcount--; - if (hctx->refcount == 0 && hctx->retired) pthread_cond_signal(&hctx->zero_cond); - pthread_mutex_unlock(&hctx->lock); - - return -1; + release_hook_ctx(hctx); + return (int)ret; } JNIEXPORT jlong JNICALL Java_com_tidesdb_ColumnFamily_nativeSetCommitHook(JNIEnv *env, jclass cls, - jlong cfHandle, - jobject hook, - jlong oldCtxHandle) + jlong dbHandle, + jlong cfHandle, + jobject hook, + jlong oldCtxHandle) { + (void)cls; + + tidesdb_t *db = (tidesdb_t *)(uintptr_t)dbHandle; tidesdb_column_family_t *cf = (tidesdb_column_family_t *)(uintptr_t)cfHandle; - /* If hook is NULL, clear the hook */ + /* a NULL hook clears the callback */ if (hook == NULL) { - int result = tidesdb_cf_set_commit_hook(cf, NULL, NULL); + int result = tidesdb_cf_set_commit_hook(db, cf, NULL, NULL); if (result != TDB_SUCCESS) { - throwTidesDBException(env, result, getErrorMessage(result)); - return oldCtxHandle; /* old context remains active */ + throwResult(env, result); + return oldCtxHandle; /* the old context stays active */ } - /* Retire old context after successful detachment */ if (oldCtxHandle != 0) { - java_hook_ctx_t *old_ctx = (java_hook_ctx_t *)(uintptr_t)oldCtxHandle; - retire_and_destroy_hook_ctx(env, old_ctx); + retire_and_destroy_hook_ctx(env, (java_hook_ctx_t *)(uintptr_t)oldCtxHandle); } - return 0; } - /* Allocate new context */ java_hook_ctx_t *new_ctx = (java_hook_ctx_t *)malloc(sizeof(java_hook_ctx_t)); if (new_ctx == NULL) { - throwTidesDBException(env, TDB_ERR_MEMORY, "Failed to allocate commit hook context"); - return oldCtxHandle; /* old context remains active */ + throwTidesDBException(env, TDB_ERR_MEMORY, "Failed to allocate the commit hook context"); + return oldCtxHandle; } - /* Initialize refcount and retired flag */ new_ctx->refcount = 0; new_ctx->retired = 0; + new_ctx->hook_obj = NULL; pthread_mutex_init(&new_ctx->lock, NULL); pthread_cond_init(&new_ctx->zero_cond, NULL); @@ -1644,434 +1691,653 @@ JNIEXPORT jlong JNICALL Java_com_tidesdb_ColumnFamily_nativeSetCommitHook(JNIEnv pthread_mutex_destroy(&new_ctx->lock); pthread_cond_destroy(&new_ctx->zero_cond); free(new_ctx); - throwTidesDBException(env, TDB_ERR_MEMORY, "Failed to get JavaVM"); + throwTidesDBException(env, TDB_ERR_UNKNOWN, "Failed to reach the JavaVM"); return oldCtxHandle; } new_ctx->hook_obj = (*env)->NewGlobalRef(env, hook); if (new_ctx->hook_obj == NULL) { - if ((*env)->ExceptionCheck(env)) (*env)->ExceptionClear(env); + if (jvm_exception_pending(env)) (*env)->ExceptionClear(env); pthread_mutex_destroy(&new_ctx->lock); pthread_cond_destroy(&new_ctx->zero_cond); free(new_ctx); throwTidesDBException(env, TDB_ERR_MEMORY, - "Failed to create global reference for commit hook"); + "Failed to create a global reference for the commit hook"); return oldCtxHandle; } - int result = tidesdb_cf_set_commit_hook(cf, java_commit_hook_trampoline, new_ctx); + int result = tidesdb_cf_set_commit_hook(db, cf, java_commit_hook_trampoline, new_ctx); if (result != TDB_SUCCESS) { (*env)->DeleteGlobalRef(env, new_ctx->hook_obj); pthread_mutex_destroy(&new_ctx->lock); pthread_cond_destroy(&new_ctx->zero_cond); free(new_ctx); - throwTidesDBException(env, result, getErrorMessage(result)); - return oldCtxHandle; /* old context remains active */ + throwResult(env, result); + return oldCtxHandle; /* the old context stays active */ } - /* Retire old context after successful hook replacement */ + /* the engine now holds the new context, so the old one can no longer be entered */ if (oldCtxHandle != 0) { - java_hook_ctx_t *old_ctx = (java_hook_ctx_t *)(uintptr_t)oldCtxHandle; - retire_and_destroy_hook_ctx(env, old_ctx); + retire_and_destroy_hook_ctx(env, (java_hook_ctx_t *)(uintptr_t)oldCtxHandle); } return (jlong)(uintptr_t)new_ctx; } -JNIEXPORT void JNICALL Java_com_tidesdb_ColumnFamily_nativePurge(JNIEnv *env, jclass cls, - jlong handle) +/* ===== com.tidesdb.Transaction ===== */ + +JNIEXPORT void JNICALL Java_com_tidesdb_Transaction_nativePut(JNIEnv *env, jclass cls, jlong handle, + jlong cfHandle, jbyteArray key, + jbyteArray value, jlong ttlSeconds) { - tidesdb_column_family_t *cf = (tidesdb_column_family_t *)(uintptr_t)handle; - int result = tidesdb_purge_cf(cf); + (void)cls; - if (result != TDB_SUCCESS) + jni_bytes_t k; + jni_bytes_t v; + if (acquireBytes(env, key, &k) != 0) return; + if (acquireBytes(env, value, &v) != 0) { - throwTidesDBException(env, result, getErrorMessage(result)); + releaseBytes(env, &k); + return; } -} -JNIEXPORT void JNICALL Java_com_tidesdb_ColumnFamily_nativeSyncWal(JNIEnv *env, jclass cls, - jlong handle) -{ - tidesdb_column_family_t *cf = (tidesdb_column_family_t *)(uintptr_t)handle; - int result = tidesdb_sync_wal(cf); + int result = tidesdb_txn_put((tidesdb_txn_t *)(uintptr_t)handle, + (tidesdb_column_family_t *)(uintptr_t)cfHandle, + (const uint8_t *)k.data, (size_t)k.length, + (const uint8_t *)v.data, (size_t)v.length, (time_t)ttlSeconds); + + releaseBytes(env, &v); + releaseBytes(env, &k); if (result != TDB_SUCCESS) { - throwTidesDBException(env, result, getErrorMessage(result)); + throwResult(env, result); } } -JNIEXPORT void JNICALL Java_com_tidesdb_TidesDB_nativePurge(JNIEnv *env, jclass cls, jlong handle) +/** Shared body for the tracking and non-tracking reads, which differ only in the call. */ +static jbyteArray transactionRead(JNIEnv *env, jlong handle, jlong cfHandle, jbyteArray key, + int track) { - tidesdb_t *db = (tidesdb_t *)(uintptr_t)handle; - int result = tidesdb_purge(db); + jni_bytes_t k; + if (acquireBytes(env, key, &k) != 0) return NULL; + + uint8_t *value = NULL; + size_t valueSize = 0; + int result; + if (track) + { + result = tidesdb_txn_get((tidesdb_txn_t *)(uintptr_t)handle, + (tidesdb_column_family_t *)(uintptr_t)cfHandle, + (const uint8_t *)k.data, (size_t)k.length, &value, &valueSize); + } + else + { + result = tidesdb_txn_get_notrack( + (tidesdb_txn_t *)(uintptr_t)handle, (tidesdb_column_family_t *)(uintptr_t)cfHandle, + (const uint8_t *)k.data, (size_t)k.length, &value, &valueSize); + } + releaseBytes(env, &k); + + if (result == TDB_ERR_NOT_FOUND) + { + return NULL; /* absence is reported as a null return, not an exception */ + } if (result != TDB_SUCCESS) { - throwTidesDBException(env, result, getErrorMessage(result)); + throwResult(env, result); + return NULL; } + + return toByteArrayAndFree(env, value, valueSize); } -JNIEXPORT jobject JNICALL Java_com_tidesdb_TidesDB_nativeGetDbStats(JNIEnv *env, jclass cls, - jlong handle) +JNIEXPORT jbyteArray JNICALL Java_com_tidesdb_Transaction_nativeGet(JNIEnv *env, jclass cls, + jlong handle, jlong cfHandle, + jbyteArray key) { - tidesdb_t *db = (tidesdb_t *)(uintptr_t)handle; - tidesdb_db_stats_t db_stats; - memset(&db_stats, 0, sizeof(db_stats)); + (void)cls; + return transactionRead(env, handle, cfHandle, key, 1); +} + +JNIEXPORT jbyteArray JNICALL Java_com_tidesdb_Transaction_nativeGetNoTrack(JNIEnv *env, jclass cls, + jlong handle, + jlong cfHandle, + jbyteArray key) +{ + (void)cls; + return transactionRead(env, handle, cfHandle, key, 0); +} + +JNIEXPORT jboolean JNICALL Java_com_tidesdb_Transaction_nativeContains(JNIEnv *env, jclass cls, + jlong handle, + jlong cfHandle, + jbyteArray key) +{ + (void)cls; + + jni_bytes_t k; + if (acquireBytes(env, key, &k) != 0) return JNI_FALSE; + + int result = tidesdb_txn_contains((tidesdb_txn_t *)(uintptr_t)handle, + (tidesdb_column_family_t *)(uintptr_t)cfHandle, + (const uint8_t *)k.data, (size_t)k.length); + + releaseBytes(env, &k); + + if (result == TDB_SUCCESS) return JNI_TRUE; + if (result == TDB_ERR_NOT_FOUND) return JNI_FALSE; + + throwResult(env, result); + return JNI_FALSE; +} + +JNIEXPORT jlong JNICALL Java_com_tidesdb_Transaction_nativeReadSnapshot(JNIEnv *env, jclass cls, + jlong handle) +{ + (void)env; + (void)cls; + return (jlong)tidesdb_txn_read_snapshot((const tidesdb_txn_t *)(uintptr_t)handle); +} + +JNIEXPORT void JNICALL Java_com_tidesdb_Transaction_nativeDelete(JNIEnv *env, jclass cls, + jlong handle, jlong cfHandle, + jbyteArray key) +{ + (void)cls; + + jni_bytes_t k; + if (acquireBytes(env, key, &k) != 0) return; + + int result = tidesdb_txn_delete((tidesdb_txn_t *)(uintptr_t)handle, + (tidesdb_column_family_t *)(uintptr_t)cfHandle, + (const uint8_t *)k.data, (size_t)k.length); + + releaseBytes(env, &k); - int result = tidesdb_get_db_stats(db, &db_stats); if (result != TDB_SUCCESS) { - throwTidesDBException(env, result, getErrorMessage(result)); - return NULL; + throwResult(env, result); } +} - jclass dbStatsClass = (*env)->FindClass(env, "com/tidesdb/DbStats"); - if (dbStatsClass == NULL) return NULL; - - jmethodID constructor = - (*env)->GetMethodID(env, dbStatsClass, "", - "(IJJJIIJIIJIJJJJZJIZIJZLjava/lang/String;JJIJJJJZJJJJJJJJJJ)V"); - if (constructor == NULL) return NULL; - - jstring connectorStr = NULL; - if (db_stats.object_store_connector != NULL) - { - connectorStr = (*env)->NewStringUTF(env, db_stats.object_store_connector); - if (connectorStr == NULL) return NULL; - } - - return (*env)->NewObject( - env, dbStatsClass, constructor, (jint)db_stats.num_column_families, - (jlong)db_stats.total_memory, (jlong)db_stats.available_memory, - (jlong)db_stats.resolved_memory_limit, (jint)db_stats.memory_pressure_level, - (jint)db_stats.flush_pending_count, (jlong)db_stats.total_memtable_bytes, - (jint)db_stats.total_immutable_count, (jint)db_stats.total_sstable_count, - (jlong)db_stats.total_data_size_bytes, (jint)db_stats.num_open_sstables, - (jlong)db_stats.global_seq, (jlong)db_stats.txn_memory_bytes, - (jlong)db_stats.compaction_queue_size, (jlong)db_stats.flush_queue_size, - db_stats.unified_memtable_enabled != 0, (jlong)db_stats.unified_memtable_bytes, - (jint)db_stats.unified_immutable_count, db_stats.unified_is_flushing != 0, - (jint)db_stats.unified_next_cf_index, (jlong)db_stats.unified_wal_generation, - db_stats.object_store_enabled != 0, connectorStr, (jlong)db_stats.local_cache_bytes_used, - (jlong)db_stats.local_cache_bytes_max, (jint)db_stats.local_cache_num_files, - (jlong)db_stats.last_uploaded_generation, (jlong)db_stats.upload_queue_depth, - (jlong)db_stats.total_uploads, (jlong)db_stats.total_upload_failures, - db_stats.replica_mode != 0, (jlong)db_stats.primary_epoch, (jlong)db_stats.seen_epoch, - (jlong)db_stats.uwal_bytes_written, (jlong)db_stats.wal_bytes_written, - (jlong)db_stats.flush_bytes_written, (jlong)db_stats.compaction_bytes_written, - (jlong)db_stats.compaction_bytes_read, (jlong)db_stats.user_bytes_written, - (jlong)db_stats.flush_count, (jlong)db_stats.compaction_count); -} - -JNIEXPORT jdouble JNICALL Java_com_tidesdb_ColumnFamily_nativeRangeCost(JNIEnv *env, jclass cls, +JNIEXPORT void JNICALL Java_com_tidesdb_Transaction_nativeSingleDelete(JNIEnv *env, jclass cls, jlong handle, - jbyteArray keyA, - jbyteArray keyB) + jlong cfHandle, + jbyteArray key) { - tidesdb_column_family_t *cf = (tidesdb_column_family_t *)(uintptr_t)handle; + (void)cls; + + jni_bytes_t k; + if (acquireBytes(env, key, &k) != 0) return; - jsize keyALen = (*env)->GetArrayLength(env, keyA); - jsize keyBLen = (*env)->GetArrayLength(env, keyB); + int result = tidesdb_txn_single_delete((tidesdb_txn_t *)(uintptr_t)handle, + (tidesdb_column_family_t *)(uintptr_t)cfHandle, + (const uint8_t *)k.data, (size_t)k.length); - jbyte *keyABytes = (*env)->GetByteArrayElements(env, keyA, NULL); - if (keyABytes == NULL) return 0.0; /* JVM exception already pending */ + releaseBytes(env, &k); - jbyte *keyBBytes = (*env)->GetByteArrayElements(env, keyB, NULL); - if (keyBBytes == NULL) + if (result != TDB_SUCCESS) + { + throwResult(env, result); + } +} + +JNIEXPORT void JNICALL Java_com_tidesdb_Transaction_nativeDeleteRange(JNIEnv *env, jclass cls, + jlong handle, jlong cfHandle, + jbyteArray lo, jbyteArray hi) +{ + (void)cls; + + jni_bytes_t lower; + jni_bytes_t upper; + if (acquireBytes(env, lo, &lower) != 0) return; + if (acquireBytes(env, hi, &upper) != 0) { - (*env)->ReleaseByteArrayElements(env, keyA, keyABytes, JNI_ABORT); - return 0.0; /* JVM exception already pending */ + releaseBytes(env, &lower); + return; } - double cost = 0.0; - int result = - tidesdb_range_cost(cf, (uint8_t *)keyABytes, keyALen, (uint8_t *)keyBBytes, keyBLen, &cost); + int result = tidesdb_txn_delete_range( + (tidesdb_txn_t *)(uintptr_t)handle, (tidesdb_column_family_t *)(uintptr_t)cfHandle, + (const uint8_t *)lower.data, (size_t)lower.length, (const uint8_t *)upper.data, + (size_t)upper.length); - (*env)->ReleaseByteArrayElements(env, keyA, keyABytes, JNI_ABORT); - (*env)->ReleaseByteArrayElements(env, keyB, keyBBytes, JNI_ABORT); + releaseBytes(env, &upper); + releaseBytes(env, &lower); if (result != TDB_SUCCESS) { - if (!jvm_exception_pending(env)) - throwTidesDBException(env, result, getErrorMessage(result)); - return 0.0; + throwResult(env, result); } - - return (jdouble)cost; } -JNIEXPORT void JNICALL Java_com_tidesdb_TidesDB_nativeDeleteColumnFamily(JNIEnv *env, jclass cls, - jlong handle, - jlong cfHandle) +JNIEXPORT void JNICALL Java_com_tidesdb_Transaction_nativeDeletePrefix(JNIEnv *env, jclass cls, + jlong handle, + jlong cfHandle, + jbyteArray prefix) { - tidesdb_t *db = (tidesdb_t *)(uintptr_t)handle; - tidesdb_column_family_t *cf = (tidesdb_column_family_t *)(uintptr_t)cfHandle; + (void)cls; + + jni_bytes_t p; + if (acquireBytes(env, prefix, &p) != 0) return; - int result = tidesdb_delete_column_family(db, cf); + int result = tidesdb_txn_delete_prefix((tidesdb_txn_t *)(uintptr_t)handle, + (tidesdb_column_family_t *)(uintptr_t)cfHandle, + (const uint8_t *)p.data, (size_t)p.length); + + releaseBytes(env, &p); if (result != TDB_SUCCESS) { - throwTidesDBException(env, result, getErrorMessage(result)); + throwResult(env, result); } } -JNIEXPORT void JNICALL Java_com_tidesdb_TidesDB_nativePromoteToPrimary(JNIEnv *env, jclass cls, - jlong handle) +JNIEXPORT jlong JNICALL Java_com_tidesdb_Transaction_nativeNewIterator(JNIEnv *env, jclass cls, + jlong handle, + jlong cfHandle) { - tidesdb_t *db = (tidesdb_t *)(uintptr_t)handle; - - int result = tidesdb_promote_to_primary(db); + (void)cls; + tidesdb_iter_t *iter = NULL; + int result = tidesdb_iter_new((tidesdb_txn_t *)(uintptr_t)handle, + (tidesdb_column_family_t *)(uintptr_t)cfHandle, &iter); if (result != TDB_SUCCESS) { - throwTidesDBException(env, result, getErrorMessage(result)); + throwResult(env, result); + return 0; } + return (jlong)(uintptr_t)iter; } -JNIEXPORT void JNICALL Java_com_tidesdb_TidesDB_nativeCancelBackgroundWork(JNIEnv *env, jclass cls, - jlong handle) +JNIEXPORT jlong JNICALL Java_com_tidesdb_Transaction_nativeNewRangeIterator(JNIEnv *env, jclass cls, + jlong handle, + jlong cfHandle, + jbyteArray lower, + jbyteArray upper) { - tidesdb_t *db = (tidesdb_t *)(uintptr_t)handle; + (void)cls; - int result = tidesdb_cancel_background_work(db); + jni_bytes_t lo; + jni_bytes_t hi; + if (acquireBytes(env, lower, &lo) != 0) return 0; + if (acquireBytes(env, upper, &hi) != 0) + { + releaseBytes(env, &lo); + return 0; + } + + tidesdb_iter_t *iter = NULL; + int result = tidesdb_iter_new_range( + (tidesdb_txn_t *)(uintptr_t)handle, (tidesdb_column_family_t *)(uintptr_t)cfHandle, + (const uint8_t *)lo.data, (size_t)lo.length, (const uint8_t *)hi.data, (size_t)hi.length, + &iter); + + releaseBytes(env, &hi); + releaseBytes(env, &lo); if (result != TDB_SUCCESS) { - throwTidesDBException(env, result, getErrorMessage(result)); + throwResult(env, result); + return 0; } + return (jlong)(uintptr_t)iter; } -JNIEXPORT jlong JNICALL Java_com_tidesdb_TidesDB_nativeRaiseOpenFileLimit(JNIEnv *env, jclass cls, - jlong desired) +JNIEXPORT void JNICALL Java_com_tidesdb_Transaction_nativeCommit(JNIEnv *env, jclass cls, + jlong handle) { - return (jlong)tidesdb_raise_open_file_limit((long)desired); + (void)cls; + int result = tidesdb_txn_commit((tidesdb_txn_t *)(uintptr_t)handle); + if (result != TDB_SUCCESS) + { + throwResult(env, result); + } } -JNIEXPORT jint JNICALL Java_com_tidesdb_Config_nativeDefaultMaxConcurrentFlushes(JNIEnv *env, - jclass cls) +JNIEXPORT void JNICALL Java_com_tidesdb_Transaction_nativeRollback(JNIEnv *env, jclass cls, + jlong handle) { - tidesdb_config_t cfg = tidesdb_default_config(); - return (jint)cfg.max_concurrent_flushes; + (void)cls; + int result = tidesdb_txn_rollback((tidesdb_txn_t *)(uintptr_t)handle); + if (result != TDB_SUCCESS) + { + throwResult(env, result); + } } -JNIEXPORT jdouble JNICALL -Java_com_tidesdb_ColumnFamilyConfig_nativeDefaultTombstoneDensityTrigger(JNIEnv *env, jclass cls) +JNIEXPORT void JNICALL Java_com_tidesdb_Transaction_nativeSetTimeout(JNIEnv *env, jclass cls, + jlong handle, jlong seconds) { - tidesdb_column_family_config_t cfg = tidesdb_default_column_family_config(); - return (jdouble)cfg.tombstone_density_trigger; + (void)cls; + int result = tidesdb_txn_set_timeout((tidesdb_txn_t *)(uintptr_t)handle, (int64_t)seconds); + if (result != TDB_SUCCESS) + { + throwResult(env, result); + } } -JNIEXPORT jlong JNICALL -Java_com_tidesdb_ColumnFamilyConfig_nativeDefaultTombstoneDensityMinEntries(JNIEnv *env, jclass cls) +JNIEXPORT void JNICALL Java_com_tidesdb_Transaction_nativeRequestAbort(JNIEnv *env, jclass cls, + jlong handle) { - tidesdb_column_family_config_t cfg = tidesdb_default_column_family_config(); - return (jlong)cfg.tombstone_density_min_entries; + (void)env; + (void)cls; + tidesdb_txn_request_abort((tidesdb_txn_t *)(uintptr_t)handle); } -/** - * Builds a com.tidesdb.ColumnFamilyConfig from a native config struct via the - * ColumnFamilyConfig.fromNative static factory. Returns a local ref, or NULL on error. - */ -static jobject buildCfConfigObject(JNIEnv *env, const tidesdb_column_family_config_t *cfg) +JNIEXPORT void JNICALL Java_com_tidesdb_Transaction_nativeReset(JNIEnv *env, jclass cls, + jlong handle, jint isolationLevel) { - jclass cfConfigClass = (*env)->FindClass(env, "com/tidesdb/ColumnFamilyConfig"); - if (cfConfigClass == NULL) return NULL; - - jmethodID fromNative = (*env)->GetStaticMethodID( - env, cfConfigClass, "fromNative", - "(JJIIJIZDZIIIJLjava/lang/String;IFIJIIDJZZZ)Lcom/tidesdb/ColumnFamilyConfig;"); - if (fromNative == NULL) + (void)cls; + int result = tidesdb_txn_reset((tidesdb_txn_t *)(uintptr_t)handle, + (tidesdb_isolation_level_t)isolationLevel); + if (result != TDB_SUCCESS) { - (*env)->DeleteLocalRef(env, cfConfigClass); - return NULL; + throwResult(env, result); } +} - jstring comparatorName = (*env)->NewStringUTF(env, cfg->comparator_name); - - jobject obj = (*env)->CallStaticObjectMethod( - env, cfConfigClass, fromNative, (jlong)cfg->write_buffer_size, (jlong)cfg->level_size_ratio, - (jint)cfg->min_levels, (jint)cfg->dividing_level_offset, (jlong)cfg->klog_value_threshold, - (jint)cfg->compression_algorithm, cfg->enable_bloom_filter != 0 ? JNI_TRUE : JNI_FALSE, - (jdouble)cfg->bloom_fpr, cfg->enable_block_indexes != 0 ? JNI_TRUE : JNI_FALSE, - (jint)cfg->index_sample_ratio, (jint)cfg->block_index_prefix_len, (jint)cfg->sync_mode, - (jlong)cfg->sync_interval_us, comparatorName, (jint)cfg->skip_list_max_level, - (jfloat)cfg->skip_list_probability, (jint)cfg->default_isolation_level, - (jlong)cfg->min_disk_space, (jint)cfg->l1_file_count_trigger, - (jint)cfg->l0_queue_stall_threshold, (jdouble)cfg->tombstone_density_trigger, - (jlong)cfg->tombstone_density_min_entries, cfg->use_btree != 0 ? JNI_TRUE : JNI_FALSE, - cfg->object_lazy_compaction != 0 ? JNI_TRUE : JNI_FALSE, - cfg->object_prefetch_compaction != 0 ? JNI_TRUE : JNI_FALSE); - - (*env)->DeleteLocalRef(env, comparatorName); - (*env)->DeleteLocalRef(env, cfConfigClass); - return obj; -} - -JNIEXPORT void JNICALL Java_com_tidesdb_ColumnFamilyConfig_nativeSaveToIni( - JNIEnv *env, jclass cls, jstring iniFile, jstring sectionName, jlong writeBufferSize, - jlong levelSizeRatio, jint minLevels, jint dividingLevelOffset, jlong klogValueThreshold, - jint compressionAlgorithm, jboolean enableBloomFilter, jdouble bloomFPR, - jboolean enableBlockIndexes, jint indexSampleRatio, jint blockIndexPrefixLen, jint syncMode, - jlong syncIntervalUs, jstring comparatorName, jint skipListMaxLevel, jfloat skipListProbability, - jint defaultIsolationLevel, jlong minDiskSpace, jint l1FileCountTrigger, - jint l0QueueStallThreshold, jdouble tombstoneDensityTrigger, jlong tombstoneDensityMinEntries, - jboolean useBtree, jboolean objectLazyCompaction, jboolean objectPrefetchCompaction) -{ - const char *ini = (*env)->GetStringUTFChars(env, iniFile, NULL); - if (ini == NULL) - { - throwTidesDBException(env, TDB_ERR_MEMORY, "Failed to get INI file path"); - return; - } - const char *section = (*env)->GetStringUTFChars(env, sectionName, NULL); - if (section == NULL) +JNIEXPORT jint JNICALL Java_com_tidesdb_Transaction_nativeState(JNIEnv *env, jclass cls, + jlong handle) +{ + (void)cls; + + tidesdb_txn_state_t state = TDB_TXN_STATE_ACTIVE; + int result = tidesdb_txn_state((const tidesdb_txn_t *)(uintptr_t)handle, &state); + if (result != TDB_SUCCESS) { - (*env)->ReleaseStringUTFChars(env, iniFile, ini); - throwTidesDBException(env, TDB_ERR_MEMORY, "Failed to get section name"); - return; + throwResult(env, result); + return 0; } + return (jint)state; +} - const char *compName = NULL; - if (comparatorName != NULL) +JNIEXPORT void JNICALL Java_com_tidesdb_Transaction_nativePrepare(JNIEnv *env, jclass cls, + jlong handle, jbyteArray xid) +{ + (void)cls; + + jni_bytes_t x; + if (acquireBytes(env, xid, &x) != 0) return; + + int result = tidesdb_txn_prepare((tidesdb_txn_t *)(uintptr_t)handle, (const uint8_t *)x.data, + (size_t)x.length); + + releaseBytes(env, &x); + + if (result != TDB_SUCCESS) { - compName = (*env)->GetStringUTFChars(env, comparatorName, NULL); + throwResult(env, result); } +} - tidesdb_column_family_config_t config = { - .write_buffer_size = (size_t)writeBufferSize, - .level_size_ratio = (size_t)levelSizeRatio, - .min_levels = minLevels, - .dividing_level_offset = dividingLevelOffset, - .klog_value_threshold = (size_t)klogValueThreshold, - .compression_algorithm = (compression_algorithm)compressionAlgorithm, - .enable_bloom_filter = enableBloomFilter ? 1 : 0, - .bloom_fpr = bloomFPR, - .enable_block_indexes = enableBlockIndexes ? 1 : 0, - .index_sample_ratio = indexSampleRatio, - .block_index_prefix_len = blockIndexPrefixLen, - .sync_mode = syncMode, - .sync_interval_us = (uint64_t)syncIntervalUs, - .skip_list_max_level = skipListMaxLevel, - .skip_list_probability = skipListProbability, - .default_isolation_level = (tidesdb_isolation_level_t)defaultIsolationLevel, - .min_disk_space = (uint64_t)minDiskSpace, - .l1_file_count_trigger = l1FileCountTrigger, - .l0_queue_stall_threshold = l0QueueStallThreshold, - .tombstone_density_trigger = tombstoneDensityTrigger, - .tombstone_density_min_entries = (uint64_t)tombstoneDensityMinEntries, - .use_btree = useBtree ? 1 : 0, - .object_lazy_compaction = objectLazyCompaction ? 1 : 0, - .object_prefetch_compaction = objectPrefetchCompaction ? 1 : 0}; - - memset(config.comparator_name, 0, TDB_MAX_COMPARATOR_NAME); - if (compName != NULL && strlen(compName) > 0) +JNIEXPORT void JNICALL Java_com_tidesdb_Transaction_nativeCommitPrepared(JNIEnv *env, jclass cls, + jlong handle) +{ + (void)cls; + int result = tidesdb_txn_commit_prepared((tidesdb_txn_t *)(uintptr_t)handle); + if (result != TDB_SUCCESS) { - strncpy(config.comparator_name, compName, TDB_MAX_COMPARATOR_NAME - 1); + throwResult(env, result); } - memset(config.comparator_ctx_str, 0, TDB_MAX_COMPARATOR_CTX); +} - int result = tidesdb_cf_config_save_to_ini(ini, section, &config); +JNIEXPORT void JNICALL Java_com_tidesdb_Transaction_nativeRollbackPrepared(JNIEnv *env, jclass cls, + jlong handle) +{ + (void)cls; + int result = tidesdb_txn_rollback_prepared((tidesdb_txn_t *)(uintptr_t)handle); + if (result != TDB_SUCCESS) + { + throwResult(env, result); + } +} - (*env)->ReleaseStringUTFChars(env, iniFile, ini); - (*env)->ReleaseStringUTFChars(env, sectionName, section); - if (compName != NULL) +/** Shared body for the three savepoint calls, which differ only in the operation. */ +static void savepointCall(JNIEnv *env, jlong handle, jstring name, + int (*op)(tidesdb_txn_t *, const char *)) +{ + const char *spName = (*env)->GetStringUTFChars(env, name, NULL); + if (spName == NULL) { - (*env)->ReleaseStringUTFChars(env, comparatorName, compName); + if (!jvm_exception_pending(env)) + { + throwTidesDBException(env, TDB_ERR_MEMORY, "Failed to read the savepoint name"); + } + return; } + int result = op((tidesdb_txn_t *)(uintptr_t)handle, spName); + (*env)->ReleaseStringUTFChars(env, name, spName); + if (result != TDB_SUCCESS) { - throwTidesDBException(env, result, getErrorMessage(result)); + throwResult(env, result); } } -JNIEXPORT jobject JNICALL Java_com_tidesdb_ColumnFamilyConfig_nativeLoadFromIni(JNIEnv *env, - jclass cls, - jstring iniFile, - jstring sectionName) +JNIEXPORT void JNICALL Java_com_tidesdb_Transaction_nativeSavepoint(JNIEnv *env, jclass cls, + jlong handle, jstring name) { - const char *ini = (*env)->GetStringUTFChars(env, iniFile, NULL); - if (ini == NULL) + (void)cls; + savepointCall(env, handle, name, tidesdb_txn_savepoint); +} + +JNIEXPORT void JNICALL Java_com_tidesdb_Transaction_nativeRollbackToSavepoint(JNIEnv *env, + jclass cls, + jlong handle, + jstring name) +{ + (void)cls; + savepointCall(env, handle, name, tidesdb_txn_rollback_to_savepoint); +} + +JNIEXPORT void JNICALL Java_com_tidesdb_Transaction_nativeReleaseSavepoint(JNIEnv *env, jclass cls, + jlong handle, + jstring name) +{ + (void)cls; + savepointCall(env, handle, name, tidesdb_txn_release_savepoint); +} + +JNIEXPORT void JNICALL Java_com_tidesdb_Transaction_nativeFree(JNIEnv *env, jclass cls, jlong handle) +{ + (void)env; + (void)cls; + tidesdb_txn_free((tidesdb_txn_t *)(uintptr_t)handle); +} + +/* ===== com.tidesdb.TidesDBIterator ===== + * + * every positioning call reports TDB_ERR_NOT_FOUND when the merged stream has no + * entry where it was asked to stand. that is the end of the range rather than a + * failure, so it is swallowed here and the iterator is simply left invalid -- + * isValid() is the single way to ask whether the cursor is on an entry. */ + +/** Runs one positioning call, reporting only the errors that are not end-of-range. */ +static void iteratorSeek(JNIEnv *env, jlong handle, int (*op)(tidesdb_iter_t *)) +{ + int result = op((tidesdb_iter_t *)(uintptr_t)handle); + if (result != TDB_SUCCESS && result != TDB_ERR_NOT_FOUND) { - throwTidesDBException(env, TDB_ERR_MEMORY, "Failed to get INI file path"); - return NULL; + throwResult(env, result); } - const char *section = (*env)->GetStringUTFChars(env, sectionName, NULL); - if (section == NULL) +} + +JNIEXPORT void JNICALL Java_com_tidesdb_TidesDBIterator_nativeSeekToFirst(JNIEnv *env, jclass cls, + jlong handle) +{ + (void)cls; + iteratorSeek(env, handle, tidesdb_iter_seek_to_first); +} + +JNIEXPORT void JNICALL Java_com_tidesdb_TidesDBIterator_nativeSeekToLast(JNIEnv *env, jclass cls, + jlong handle) +{ + (void)cls; + iteratorSeek(env, handle, tidesdb_iter_seek_to_last); +} + +JNIEXPORT void JNICALL Java_com_tidesdb_TidesDBIterator_nativeNext(JNIEnv *env, jclass cls, + jlong handle) +{ + (void)cls; + iteratorSeek(env, handle, tidesdb_iter_next); +} + +JNIEXPORT void JNICALL Java_com_tidesdb_TidesDBIterator_nativePrev(JNIEnv *env, jclass cls, + jlong handle) +{ + (void)cls; + iteratorSeek(env, handle, tidesdb_iter_prev); +} + +/** Runs one keyed positioning call, with the same end-of-range handling. */ +static void iteratorSeekKey(JNIEnv *env, jlong handle, jbyteArray key, + int (*op)(tidesdb_iter_t *, const uint8_t *, size_t)) +{ + jni_bytes_t k; + if (acquireBytes(env, key, &k) != 0) return; + + int result = op((tidesdb_iter_t *)(uintptr_t)handle, (const uint8_t *)k.data, (size_t)k.length); + + releaseBytes(env, &k); + + if (result != TDB_SUCCESS && result != TDB_ERR_NOT_FOUND) { - (*env)->ReleaseStringUTFChars(env, iniFile, ini); - throwTidesDBException(env, TDB_ERR_MEMORY, "Failed to get section name"); - return NULL; + throwResult(env, result); } +} - /* start from engine defaults so fields absent from the INI section keep sane values */ - tidesdb_column_family_config_t config = tidesdb_default_column_family_config(); +JNIEXPORT void JNICALL Java_com_tidesdb_TidesDBIterator_nativeSeek(JNIEnv *env, jclass cls, + jlong handle, jbyteArray key) +{ + (void)cls; + iteratorSeekKey(env, handle, key, tidesdb_iter_seek); +} - int result = tidesdb_cf_config_load_from_ini(ini, section, &config); +JNIEXPORT void JNICALL Java_com_tidesdb_TidesDBIterator_nativeSeekForPrev(JNIEnv *env, jclass cls, + jlong handle, + jbyteArray key) +{ + (void)cls; + iteratorSeekKey(env, handle, key, tidesdb_iter_seek_for_prev); +} - (*env)->ReleaseStringUTFChars(env, iniFile, ini); - (*env)->ReleaseStringUTFChars(env, sectionName, section); +JNIEXPORT jboolean JNICALL Java_com_tidesdb_TidesDBIterator_nativeValid(JNIEnv *env, jclass cls, + jlong handle) +{ + (void)env; + (void)cls; + return tidesdb_iter_valid((tidesdb_iter_t *)(uintptr_t)handle) ? JNI_TRUE : JNI_FALSE; +} +JNIEXPORT jbyteArray JNICALL Java_com_tidesdb_TidesDBIterator_nativeKey(JNIEnv *env, jclass cls, + jlong handle) +{ + (void)cls; + + uint8_t *key = NULL; + size_t keySize = 0; + int result = tidesdb_iter_key((tidesdb_iter_t *)(uintptr_t)handle, &key, &keySize); if (result != TDB_SUCCESS) { - throwTidesDBException(env, result, getErrorMessage(result)); + throwResult(env, result); return NULL; } - - return buildCfConfigObject(env, &config); + return toByteArrayAndFree(env, key, keySize); } -JNIEXPORT jobject JNICALL Java_com_tidesdb_TidesDBIterator_nativeKeyValue(JNIEnv *env, jclass cls, +JNIEXPORT jbyteArray JNICALL Java_com_tidesdb_TidesDBIterator_nativeValue(JNIEnv *env, jclass cls, jlong handle) { - tidesdb_iter_t *iter = (tidesdb_iter_t *)(uintptr_t)handle; - uint8_t *key = NULL; - size_t keyLen = 0; - uint8_t *value = NULL; - size_t valueLen = 0; + (void)cls; - int result = tidesdb_iter_key_value(iter, &key, &keyLen, &value, &valueLen); + uint8_t *value = NULL; + size_t valueSize = 0; + int result = tidesdb_iter_value((tidesdb_iter_t *)(uintptr_t)handle, &value, &valueSize); if (result != TDB_SUCCESS) { - if (!jvm_exception_pending(env)) - throwTidesDBException(env, result, getErrorMessage(result)); + throwResult(env, result); return NULL; } + return toByteArrayAndFree(env, value, valueSize); +} + +JNIEXPORT jobject JNICALL Java_com_tidesdb_TidesDBIterator_nativeKeyValue(JNIEnv *env, jclass cls, + jlong handle) +{ + (void)cls; - if (keyLen > (size_t)JSIZE_MAX) + uint8_t *key = NULL; + size_t keySize = 0; + uint8_t *value = NULL; + size_t valueSize = 0; + int result = tidesdb_iter_key_value((tidesdb_iter_t *)(uintptr_t)handle, &key, &keySize, &value, + &valueSize); + if (result != TDB_SUCCESS) { - (*env)->ThrowNew(env, (*env)->FindClass(env, "java/lang/ArrayIndexOutOfBoundsException"), - "key exceeds maximum Java array size"); + throwResult(env, result); return NULL; } - if (valueLen > (size_t)JSIZE_MAX) + jbyteArray jKey = toByteArrayAndFree(env, key, keySize); + jbyteArray jValue = toByteArrayAndFree(env, value, valueSize); + if (jKey == NULL || jValue == NULL) { - (*env)->ThrowNew(env, (*env)->FindClass(env, "java/lang/ArrayIndexOutOfBoundsException"), - "value exceeds maximum Java array size"); return NULL; } - jbyteArray jkey = (*env)->NewByteArray(env, (jsize)keyLen); - if (jkey == NULL) return NULL; /* JVM exception (OOM) already pending */ + jclass cls_ = (*env)->FindClass(env, "com/tidesdb/KeyValue"); + if (cls_ == NULL) return NULL; - jbyteArray jvalue = (*env)->NewByteArray(env, (jsize)valueLen); - if (jvalue == NULL) return NULL; /* JVM exception (OOM) already pending; jkey will be GCed */ + jmethodID ctor = (*env)->GetMethodID(env, cls_, "", "([B[B)V"); + jobject result_obj = ctor != NULL ? (*env)->NewObject(env, cls_, ctor, jKey, jValue) : NULL; - (*env)->SetByteArrayRegion(env, jkey, 0, (jsize)keyLen, (jbyte *)key); - (*env)->SetByteArrayRegion(env, jvalue, 0, (jsize)valueLen, (jbyte *)value); + (*env)->DeleteLocalRef(env, cls_); + return result_obj; +} - jclass kvClass = (*env)->FindClass(env, "com/tidesdb/KeyValue"); - if (kvClass == NULL) return NULL; +JNIEXPORT void JNICALL Java_com_tidesdb_TidesDBIterator_nativeFree(JNIEnv *env, jclass cls, + jlong handle) +{ + (void)env; + (void)cls; + tidesdb_iter_free((tidesdb_iter_t *)(uintptr_t)handle); +} - jmethodID ctor = (*env)->GetMethodID(env, kvClass, "", "([B[B)V"); - if (ctor == NULL) return NULL; +/* ===== com.tidesdb.StallReason / com.tidesdb.IoClass ===== */ - jobject result_obj = (*env)->NewObject(env, kvClass, ctor, jkey, jvalue); - return result_obj; +JNIEXPORT jstring JNICALL Java_com_tidesdb_StallReason_nativeName(JNIEnv *env, jclass cls, + jint reason) +{ + (void)cls; + return (*env)->NewStringUTF(env, tidesdb_stall_reason_name((tidesdb_stall_reason_t)reason)); +} + +JNIEXPORT jstring JNICALL Java_com_tidesdb_IoClass_nativeName(JNIEnv *env, jclass cls, jint cls_id) +{ + (void)cls; + return (*env)->NewStringUTF(env, tidesdb_io_class_name((tidesdb_io_class_t)cls_id)); +} + +/* ===== com.tidesdb.Snapshot ===== */ + +JNIEXPORT jlong JNICALL Java_com_tidesdb_Snapshot_nativeSeq(JNIEnv *env, jclass cls, jlong handle) +{ + (void)env; + (void)cls; + return (jlong)tidesdb_snapshot_seq((const tidesdb_snapshot_t *)(uintptr_t)handle); +} + +JNIEXPORT void JNICALL Java_com_tidesdb_Snapshot_nativeRelease(JNIEnv *env, jclass cls, jlong handle) +{ + (void)env; + (void)cls; + tidesdb_snapshot_release((tidesdb_snapshot_t *)(uintptr_t)handle); } diff --git a/src/main/java/com/tidesdb/CfStats.java b/src/main/java/com/tidesdb/CfStats.java new file mode 100644 index 0000000..f27e213 --- /dev/null +++ b/src/main/java/com/tidesdb/CfStats.java @@ -0,0 +1,418 @@ +/** + * + * Copyright (C) TidesDB + * + * Original Author: Alex Gaetano Padula + * + * Licensed under the Mozilla Public License, v. 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.mozilla.org/en-US/MPL/2.0/ + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.tidesdb; + +import java.util.Arrays; + +/** + * Per-column-family statistics returned by {@link ColumnFamily#getStats()}. + * + *

Shared-memtable, value-log, and MVCC figures are database-level and live in + * {@link DbStats} instead. + * + *

The per-level arrays are indexed by level minus one, and only the first + * {@link #getNumLevels()} entries are meaningful. + */ +public class CfStats { + + /** + * The maximum number of SSTable levels a column family keeps, which is the + * length of every per-level array on this class. + */ + public static final int MAX_LEVELS = 8; + + private final int numLevels; + private final ColumnFamilyConfig config; + private final long[] levelSizes; + private final int[] levelNumSstables; + private final long[] levelKeyCounts; + private final long[] levelTombstoneCounts; + private final long totalKeys; + private final long totalDataSize; + private final double avgKeySize; + private final double avgValueSize; + private final double readAmp; + private final long btreeTotalNodes; + private final long btreeMaxHeight; + private final double btreeAvgHeight; + private final long totalTombstones; + private final double tombstoneRatio; + private final double maxSstDensity; + private final int maxSstDensityLevel; + private final long walBytesWritten; + private final long flushBytesWritten; + private final long compactionBytesWritten; + private final long compactionBytesRead; + private final long userBytesWritten; + private final long compactionCount; + private final long unflushedKeyCount; + private final long filterResidentBytes; + + /** + * Creates a new {@code CfStats}. Typically called by the JNI bridge rather + * than application code. + * + * @param numLevels number of levels in the column family + * @param config a copy of the column family configuration + * @param levelSizes on-disk size of each level, indexed by level minus one + * @param levelNumSstables SSTable count of each level + * @param levelKeyCounts key count of each level + * @param levelTombstoneCounts tombstone count of each level + * @param totalKeys total distinct keys across every SSTable + * @param totalDataSize the family's own on-disk size + * @param avgKeySize average key length in bytes, over distinct keys + * @param avgValueSize average value length in bytes, over distinct keys + * @param readAmp point-lookup read amplification + * @param btreeTotalNodes total btree nodes across the column family + * @param btreeMaxHeight maximum btree height + * @param btreeAvgHeight average btree height + * @param totalTombstones sum of tombstone counts across every SSTable + * @param tombstoneRatio tombstones over total keys + * @param maxSstDensity worst per-SSTable tombstone density observed + * @param maxSstDensityLevel 1-based level where that density was observed + * @param walBytesWritten this family's share of the shared write-ahead log + * @param flushBytesWritten on-disk bytes this family's flushes wrote to L1 + * @param compactionBytesWritten on-disk bytes this family's compactions wrote + * @param compactionBytesRead on-disk bytes this family's compactions read + * @param userBytesWritten logical key and value bytes committed + * @param compactionCount compactions this family has run + * @param unflushedKeyCount distinct keys resident in the shared memtables + * @param filterResidentBytes memory this family's filters hold outside the cache + */ + public CfStats(int numLevels, ColumnFamilyConfig config, long[] levelSizes, + int[] levelNumSstables, long[] levelKeyCounts, long[] levelTombstoneCounts, + long totalKeys, long totalDataSize, double avgKeySize, double avgValueSize, + double readAmp, long btreeTotalNodes, long btreeMaxHeight, double btreeAvgHeight, + long totalTombstones, double tombstoneRatio, double maxSstDensity, + int maxSstDensityLevel, long walBytesWritten, long flushBytesWritten, + long compactionBytesWritten, long compactionBytesRead, long userBytesWritten, + long compactionCount, long unflushedKeyCount, long filterResidentBytes) { + this.numLevels = numLevels; + this.config = config; + this.levelSizes = levelSizes == null ? new long[MAX_LEVELS] : levelSizes.clone(); + this.levelNumSstables = levelNumSstables == null ? new int[MAX_LEVELS] : levelNumSstables.clone(); + this.levelKeyCounts = levelKeyCounts == null ? new long[MAX_LEVELS] : levelKeyCounts.clone(); + this.levelTombstoneCounts = + levelTombstoneCounts == null ? new long[MAX_LEVELS] : levelTombstoneCounts.clone(); + this.totalKeys = totalKeys; + this.totalDataSize = totalDataSize; + this.avgKeySize = avgKeySize; + this.avgValueSize = avgValueSize; + this.readAmp = readAmp; + this.btreeTotalNodes = btreeTotalNodes; + this.btreeMaxHeight = btreeMaxHeight; + this.btreeAvgHeight = btreeAvgHeight; + this.totalTombstones = totalTombstones; + this.tombstoneRatio = tombstoneRatio; + this.maxSstDensity = maxSstDensity; + this.maxSstDensityLevel = maxSstDensityLevel; + this.walBytesWritten = walBytesWritten; + this.flushBytesWritten = flushBytesWritten; + this.compactionBytesWritten = compactionBytesWritten; + this.compactionBytesRead = compactionBytesRead; + this.userBytesWritten = userBytesWritten; + this.compactionCount = compactionCount; + this.unflushedKeyCount = unflushedKeyCount; + this.filterResidentBytes = filterResidentBytes; + } + + /** + * Returns the number of levels in the column family. Only the first this + * many entries of each per-level array are meaningful. + * + * @return the level count + */ + public int getNumLevels() { + return numLevels; + } + + /** + * Returns a copy of the column family's configuration as the engine holds + * it, including its persisted name. + * + * @return the configuration + */ + public ColumnFamilyConfig getConfig() { + return config; + } + + /** + * Returns the on-disk size of each level, indexed by level minus one. + * + * @return a copy of the per-level sizes in bytes + */ + public long[] getLevelSizes() { + return levelSizes.clone(); + } + + /** + * Returns the SSTable count of each level, indexed by level minus one. + * + * @return a copy of the per-level SSTable counts + */ + public int[] getLevelNumSstables() { + return levelNumSstables.clone(); + } + + /** + * Returns the key count of each level, indexed by level minus one. + * + * @return a copy of the per-level key counts + */ + public long[] getLevelKeyCounts() { + return levelKeyCounts.clone(); + } + + /** + * Returns the tombstone count of each level, indexed by level minus one. + * + * @return a copy of the per-level tombstone counts + */ + public long[] getLevelTombstoneCounts() { + return levelTombstoneCounts.clone(); + } + + /** + * Returns the total distinct keys across every SSTable in the column family. + * Add {@link #getUnflushedKeyCount()} for the live logical key count + * including what is still in memory. + * + * @return the key count + */ + public long getTotalKeys() { + return totalKeys; + } + + /** + * Returns the family's own on-disk size, the sum of its key logs. Values + * below the separation threshold are inside those bytes already; what + * spilled lives in the shared value log, reported by + * {@link DbStats#getVlogFileSize()}. + * + * @return the on-disk size in bytes + */ + public long getTotalDataSize() { + return totalDataSize; + } + + /** + * Returns the average key length in bytes, over distinct keys. + * + * @return the average key size + */ + public double getAvgKeySize() { + return avgKeySize; + } + + /** + * Returns the average value length in bytes, over distinct keys. + * Tombstones contribute zero. + * + * @return the average value size + */ + public double getAvgValueSize() { + return avgValueSize; + } + + /** + * Returns the point-lookup read amplification, the SSTables a worst-case get + * may probe. + * + * @return the read amplification + */ + public double getReadAmp() { + return readAmp; + } + + /** + * Returns the total btree nodes across the column family. + * + * @return the node count + */ + public long getBtreeTotalNodes() { + return btreeTotalNodes; + } + + /** + * Returns the maximum btree height. + * + * @return the maximum height + */ + public long getBtreeMaxHeight() { + return btreeMaxHeight; + } + + /** + * Returns the average btree height. + * + * @return the average height + */ + public double getBtreeAvgHeight() { + return btreeAvgHeight; + } + + /** + * Returns the sum of tombstone counts across every SSTable. + * + * @return the tombstone count + */ + public long getTotalTombstones() { + return totalTombstones; + } + + /** + * Returns tombstones over total keys, or 0 when there are no keys. + * + * @return the tombstone ratio + */ + public double getTombstoneRatio() { + return tombstoneRatio; + } + + /** + * Returns the worst per-SSTable tombstone density observed. + * + * @return the maximum density + */ + public double getMaxSstDensity() { + return maxSstDensity; + } + + /** + * Returns the 1-based level where {@link #getMaxSstDensity()} was observed. + * + * @return the level, or 0 if none + */ + public int getMaxSstDensityLevel() { + return maxSstDensityLevel; + } + + /** + * Returns this family's share of the shared write-ahead log, as the encoded + * size of its own entries. This is an attribution rather than a measurement: + * the batch header and the block framing belong to no single family, so + * these do not sum to what the log wrote. {@link DbStats#getWalBytesWritten()} + * is the measured figure. + * + * @return the attributed WAL byte count + */ + public long getWalBytesWritten() { + return walBytesWritten; + } + + /** + * Returns the on-disk bytes this family's flushes wrote to L1. + * + * @return the flush byte count + */ + public long getFlushBytesWritten() { + return flushBytesWritten; + } + + /** + * Returns the on-disk bytes this family's compactions wrote. + * + * @return the compaction output byte count + */ + public long getCompactionBytesWritten() { + return compactionBytesWritten; + } + + /** + * Returns the on-disk bytes this family's compactions read as input. + * + * @return the compaction input byte count + */ + public long getCompactionBytesRead() { + return compactionBytesRead; + } + + /** + * Returns the logical key and value bytes committed to this family. + * + * @return the user byte count + */ + public long getUserBytesWritten() { + return userBytesWritten; + } + + /** + * Returns the number of compactions this family has run. + * + * @return the compaction count + */ + public long getCompactionCount() { + return compactionCount; + } + + /** + * Returns the distinct keys resident in the shared memtables for this family + * and not yet in any SSTable. + * + * @return the unflushed key count + */ + public long getUnflushedKeyCount() { + return unflushedKeyCount; + } + + /** + * Returns the memory this family's partition range filters hold outside the + * block cache. The filter bit arrays themselves are not counted here: those + * are fetched per probe and live in the cache. A directory is built on a + * table's first probe, so this reads zero for a family nothing has read from + * yet and rises as tables are touched. + * + * @return the resident filter bytes + */ + public long getFilterResidentBytes() { + return filterResidentBytes; + } + + @Override + public String toString() { + return "CfStats{" + + "numLevels=" + numLevels + + ", levelSizes=" + Arrays.toString(levelSizes) + + ", levelNumSstables=" + Arrays.toString(levelNumSstables) + + ", levelKeyCounts=" + Arrays.toString(levelKeyCounts) + + ", levelTombstoneCounts=" + Arrays.toString(levelTombstoneCounts) + + ", totalKeys=" + totalKeys + + ", totalDataSize=" + totalDataSize + + ", avgKeySize=" + avgKeySize + + ", avgValueSize=" + avgValueSize + + ", readAmp=" + readAmp + + ", btreeTotalNodes=" + btreeTotalNodes + + ", btreeMaxHeight=" + btreeMaxHeight + + ", btreeAvgHeight=" + btreeAvgHeight + + ", totalTombstones=" + totalTombstones + + ", tombstoneRatio=" + tombstoneRatio + + ", maxSstDensity=" + maxSstDensity + + ", maxSstDensityLevel=" + maxSstDensityLevel + + ", walBytesWritten=" + walBytesWritten + + ", flushBytesWritten=" + flushBytesWritten + + ", compactionBytesWritten=" + compactionBytesWritten + + ", compactionBytesRead=" + compactionBytesRead + + ", userBytesWritten=" + userBytesWritten + + ", compactionCount=" + compactionCount + + ", unflushedKeyCount=" + unflushedKeyCount + + ", filterResidentBytes=" + filterResidentBytes + + ", config=" + config + + '}'; + } +} diff --git a/src/main/java/com/tidesdb/ColumnFamily.java b/src/main/java/com/tidesdb/ColumnFamily.java index 7065111..9a57b49 100644 --- a/src/main/java/com/tidesdb/ColumnFamily.java +++ b/src/main/java/com/tidesdb/ColumnFamily.java @@ -19,37 +19,35 @@ package com.tidesdb; /** - * Represents a column family in TidesDB. A column family is an isolated - * key-value store within a database, with its own independent configuration. + * A column family in TidesDB: an isolated key-value store within a database, + * with its own independent configuration. * - *

A {@code ColumnFamily} is a handle returned by {@link TidesDB#getColumnFamily(String)} - * and is not independently closeable. There is no Java-side guard against using - * a column family after its owning database has been closed; callers must manage - * the lifecycle externally. + *

A {@code ColumnFamily} is a handle returned by + * {@link TidesDB#getColumnFamily(String)} and is not independently closeable. It + * is invalid once its owning database is closed or the family is dropped. + * + *

The memtable and write-ahead log are shared across every family, so + * flushing and WAL syncing live on {@link TidesDB} rather than here. * *

This class is not guaranteed to be thread-safe. */ public class ColumnFamily { - + static { NativeLibrary.load(); } - + private final long nativeHandle; private final String name; private long commitHookCtxHandle = 0; private final TidesDB owner; - - ColumnFamily(long nativeHandle, String name) { - this(nativeHandle, name, null); - } - + ColumnFamily(long nativeHandle, String name, TidesDB owner) { this.nativeHandle = nativeHandle; this.name = name; this.owner = owner; } - + /** * Checks that the owning database is open. Throws if the owner is closed. */ @@ -58,204 +56,192 @@ void checkOwnerOpen() { throw new IllegalStateException("TidesDB instance is closed"); } } - + + private long ownerHandle() { + checkOwnerOpen(); + return owner == null ? 0 : owner.getNativeHandle(); + } + /** - * Gets the name of this column family. + * Returns the name of this column family. * * @return the column family name, never {@code null} */ public String getName() { return name; } - + /** - * Retrieves statistics about this column family. + * Collects statistics for this column family. * - * @return column family statistics, never {@code null} - * @throws TidesDBException if the native stats retrieval fails + * @return the statistics, never {@code null} + * @throws IllegalStateException if the owning database is closed + * @throws TidesDBException if the native stats retrieval fails, including + * {@link TidesDBException#ERR_LOCKED} when descriptor pressure kept a + * level from being read */ - public Stats getStats() throws TidesDBException { + public CfStats getStats() throws TidesDBException { checkOwnerOpen(); return nativeGetStats(nativeHandle); } - + /** - * Manually triggers compaction for this column family. + * Estimates the distinct key count of this column family. * - * @throws TidesDBException if the native compaction fails + * @return the estimated distinct key count + * @throws IllegalStateException if the owning database is closed + * @throws TidesDBException if the estimate fails, including + * {@link TidesDBException#ERR_LOCKED} when descriptor pressure kept a + * level from being read */ - public void compact() throws TidesDBException { + public long estimateCardinality() throws TidesDBException { checkOwnerOpen(); - nativeCompact(nativeHandle); + return nativeEstimateCardinality(nativeHandle); } /** - * Synchronously compacts every SSTable whose key range overlaps {@code [startKey, endKey)}. - * Blocks the calling thread until the merge commits or fails - does not enqueue work - * onto the compaction thread pool. - * - *

A {@code null} or empty endpoint means unbounded on that side. Both endpoints - * being {@code null} or empty is rejected with {@link TidesDBException}; callers - * wanting full-CF compaction must use {@link #compact()}.

+ * Synchronously runs one forced compaction pass on this column family, + * merging even when no trigger is due. * - * @param startKey lower bound of the range, or {@code null}/empty for unbounded - * @param endKey upper bound (exclusive), or {@code null}/empty for unbounded - * @throws TidesDBException if the range is invalid, another compaction is running, - * or the merge fails + * @throws IllegalStateException if the owning database is closed + * @throws TidesDBException with {@link TidesDBException#ERR_LOCKED} if a + * compaction is already running, in which case the work asked for is + * usually already under way */ - public void compactRange(byte[] startKey, byte[] endKey) throws TidesDBException { - checkOwnerOpen(); - nativeCompactRange(nativeHandle, startKey, endKey); + public void compact() throws TidesDBException { + nativeCompact(ownerHandle(), nativeHandle); } - + /** - * Manually triggers a memtable flush for this column family. + * Synchronously compacts every SSTable overlapping + * {@code [startKey, endKey)}, merging toward the largest level affected. + * Blocks the calling thread until the merge commits or fails. * - * @throws TidesDBException if the native flush fails - */ - public void flushMemtable() throws TidesDBException { - checkOwnerOpen(); - nativeFlushMemtable(nativeHandle); - } - - /** - * Checks if a flush operation is currently in progress for this column family. + *

A {@code null} or empty endpoint is unbounded on that side. Both being + * unbounded is rejected in favour of {@link #compact()}. * - * @return true if flushing is in progress + * @param startKey the range start, or {@code null}/empty for unbounded + * @param endKey the range end, or {@code null}/empty for unbounded + * @throws IllegalStateException if the owning database is closed + * @throws TidesDBException if the range is invalid, a compaction is already + * running, or the merge fails */ - public boolean isFlushing() { - checkOwnerOpen(); - return nativeIsFlushing(nativeHandle); + public void compactRange(byte[] startKey, byte[] endKey) throws TidesDBException { + nativeCompactRange(ownerHandle(), nativeHandle, startKey, endKey); } - + /** - * Checks if a compaction operation is currently in progress for this column family. + * Reports whether a compaction is in progress on this column family. * - * @return true if compaction is in progress + * @return {@code true} if compacting + * @throws IllegalStateException if the owning database is closed */ public boolean isCompacting() { checkOwnerOpen(); return nativeIsCompacting(nativeHandle); } - + /** - * Updates runtime-safe configuration settings for this column family. - * Configuration changes are applied to new operations only. - * - *

Updatable settings (safe to change at runtime):

- *
    - *
  • writeBufferSize - Memtable flush threshold
    • - *
  • skipListMaxLevel - Skip list level for new memtables
    • - *
  • skipListProbability - Skip list probability for new memtables
    • - *
  • bloomFPR - False positive rate for new SSTables
    • - *
  • indexSampleRatio - Index sampling ratio for new SSTables
    • - *
  • syncMode - Durability mode
    • - *
  • syncIntervalUs - Sync interval in microseconds
    • - *
+ * Applies a new configuration to this column family at runtime. Every field + * may change, since byte-wise key ordering keeps all SSTables mergeable. The + * family name and id are preserved; a rename is separate. * - * @param config the new configuration - * @param persistToDisk if true, saves changes to config.ini + * @param config the configuration to apply; must not be {@code null}. Its + * {@code name} field is ignored + * @param persistToDisk {@code true} to persist the new config in the + * manifest, {@code false} to apply in memory only + * @throws IllegalArgumentException if {@code config} is {@code null} + * @throws IllegalStateException if the owning database is closed * @throws TidesDBException if the update fails */ - public void updateRuntimeConfig(ColumnFamilyConfig config, boolean persistToDisk) throws TidesDBException { - checkOwnerOpen(); + public void updateRuntimeConfig(ColumnFamilyConfig config, boolean persistToDisk) + throws TidesDBException { if (config == null) { throw new IllegalArgumentException("Config cannot be null"); } - nativeUpdateRuntimeConfig(nativeHandle, - config.getWriteBufferSize(), - config.getSkipListMaxLevel(), - config.getSkipListProbability(), - config.getBloomFPR(), - config.getIndexSampleRatio(), - config.getSyncMode().getValue(), - config.getSyncIntervalUs(), + nativeUpdateRuntimeConfig(ownerHandle(), nativeHandle, + config.getLevelSizeRatio(), + config.getMinLevels(), + config.getDividingLevelOffset(), + config.isKeepValuesInline(), + config.getBtreeKlogBlockSize(), + config.getEncodingPipeline(), + config.isEnableBloomFilter(), + config.getBloomFpr(), + config.getDefaultIsolationLevel().getValue(), + config.getL1FileCountTrigger(), + config.getTombstoneDensityTrigger(), + config.getTombstoneDensityMinEntries(), persistToDisk); } - - /** - * Forces a synchronous flush and aggressive compaction for this column family. - * Unlike {@link #compact()} and {@link #flushMemtable()} (which are non-blocking), - * purge blocks until all flush and compaction I/O is complete. - * - * @throws TidesDBException if the purge fails - */ - public void purge() throws TidesDBException { - checkOwnerOpen(); - nativePurge(nativeHandle); - } - + /** - * Forces an immediate fsync of the active write-ahead log for this column family. - * Useful for explicit durability control when using SYNC_NONE or SYNC_INTERVAL modes. + * Describes the key range {@code [keyA, keyB)} for a query planner, reporting + * both what a scan of it would cost and how many live keys it holds. * - * @throws TidesDBException if the WAL sync fails - */ - public void syncWal() throws TidesDBException { - checkOwnerOpen(); - nativeSyncWal(nativeHandle); - } - - /** - * Estimates the computational cost of iterating between two keys in this column family. - * The returned value is an opaque double - meaningful only for comparison with other - * values from the same method. Uses only in-memory metadata and performs no disk I/O. - * Key order does not matter - the method normalizes the range internally. + *

The count is memtable-aware. A range small enough to walk is counted + * exactly and reports {@link RangeStats#isKeysExact()}; a wider one is + * estimated from SSTable metadata without walking, so the call stays cheap + * enough for plan time whatever the range covers. * - * @param keyA first key (bound of range) - * @param keyB second key (bound of range) - * @return estimated traversal cost (higher = more expensive), 0.0 if no overlapping data - * @throws TidesDBException if the estimation fails + * @param keyA the range start, inclusive; must not be {@code null} or empty + * @param keyB the range end, exclusive; must not be {@code null} or empty + * @return the range statistics, never {@code null} + * @throws IllegalArgumentException if either bound is {@code null} or empty + * @throws IllegalStateException if the owning database is closed + * @throws TidesDBException if the call fails, including + * {@link TidesDBException#ERR_LOCKED} if the layout moved mid-scan */ - public double rangeCost(byte[] keyA, byte[] keyB) throws TidesDBException { - checkOwnerOpen(); + public RangeStats rangeStats(byte[] keyA, byte[] keyB) throws TidesDBException { if (keyA == null || keyA.length == 0) { throw new IllegalArgumentException("keyA cannot be null or empty"); } if (keyB == null || keyB.length == 0) { throw new IllegalArgumentException("keyB cannot be null or empty"); } - return nativeRangeCost(nativeHandle, keyA, keyB); + return nativeRangeStats(ownerHandle(), nativeHandle, keyA, keyB); } - + /** - * Sets a commit hook (Change Data Capture) for this column family. - * The hook fires synchronously after every transaction commit, receiving the full - * batch of committed operations atomically. Keep the callback fast to avoid - * stalling writers. + * Sets a commit hook for this column family. The hook fires synchronously + * after every transaction commit, receiving the full batch of committed + * operations atomically. Keep the callback fast to avoid stalling writers. * *

Hooks are runtime-only and not persisted. After a database restart, - * hooks must be re-registered by the application.

+ * hooks must be re-registered by the application. * - * @param hook the commit hook callback + * @param hook the commit hook callback; must not be {@code null} + * @throws IllegalArgumentException if {@code hook} is {@code null} + * @throws IllegalStateException if the owning database is closed * @throws TidesDBException if the hook cannot be set */ public void setCommitHook(CommitHook hook) throws TidesDBException { - checkOwnerOpen(); if (hook == null) { throw new IllegalArgumentException("Hook cannot be null, use clearCommitHook() instead"); } + long dbHandle = ownerHandle(); boolean firstInstall = (commitHookCtxHandle == 0); - commitHookCtxHandle = nativeSetCommitHook(nativeHandle, hook, commitHookCtxHandle); + commitHookCtxHandle = nativeSetCommitHook(dbHandle, nativeHandle, hook, commitHookCtxHandle); if (firstInstall && owner != null) { owner.registerHookColumnFamily(this); } } - + /** - * Clears the commit hook for this column family. - * After this call, no further commit callbacks will fire. + * Clears the commit hook for this column family. After this call, no further + * commit callbacks fire. * + * @throws IllegalStateException if the owning database is closed * @throws TidesDBException if the hook cannot be cleared */ public void clearCommitHook() throws TidesDBException { - checkOwnerOpen(); - commitHookCtxHandle = nativeSetCommitHook(nativeHandle, null, commitHookCtxHandle); + long dbHandle = ownerHandle(); + commitHookCtxHandle = nativeSetCommitHook(dbHandle, nativeHandle, null, commitHookCtxHandle); if (owner != null) { owner.unregisterHookColumnFamily(this); } } - + /** * Best-effort hook detach during database close. The hook is removed from * the engine without caller interaction. Errors are swallowed. @@ -263,29 +249,44 @@ public void clearCommitHook() throws TidesDBException { void clearHookOnClose() { if (commitHookCtxHandle != 0) { try { - nativeSetCommitHook(nativeHandle, null, commitHookCtxHandle); + long dbHandle = owner == null ? 0 : owner.getNativeHandle(); + nativeSetCommitHook(dbHandle, nativeHandle, null, commitHookCtxHandle); } catch (TidesDBException ignored) { // Best-effort during shutdown. } commitHookCtxHandle = 0; } } - + long getNativeHandle() { return nativeHandle; } - - private static native Stats nativeGetStats(long handle) throws TidesDBException; - private static native void nativeCompact(long handle) throws TidesDBException; - private static native void nativeCompactRange(long handle, byte[] startKey, byte[] endKey) throws TidesDBException; - private static native void nativeFlushMemtable(long handle) throws TidesDBException; - private static native boolean nativeIsFlushing(long handle); - private static native boolean nativeIsCompacting(long handle); - private static native void nativeUpdateRuntimeConfig(long handle, long writeBufferSize, - int skipListMaxLevel, float skipListProbability, double bloomFPR, int indexSampleRatio, - int syncMode, long syncIntervalUs, boolean persistToDisk) throws TidesDBException; - private static native double nativeRangeCost(long handle, byte[] keyA, byte[] keyB) throws TidesDBException; - private static native long nativeSetCommitHook(long handle, CommitHook hook, long oldCtxHandle) throws TidesDBException; - private static native void nativePurge(long handle) throws TidesDBException; - private static native void nativeSyncWal(long handle) throws TidesDBException; + + @Override + public String toString() { + return "ColumnFamily{name='" + name + "'}"; + } + + private static native CfStats nativeGetStats(long cfHandle) throws TidesDBException; + + private static native long nativeEstimateCardinality(long cfHandle) throws TidesDBException; + + private static native void nativeCompact(long dbHandle, long cfHandle) throws TidesDBException; + + private static native void nativeCompactRange(long dbHandle, long cfHandle, byte[] startKey, + byte[] endKey) throws TidesDBException; + + private static native boolean nativeIsCompacting(long cfHandle); + + private static native void nativeUpdateRuntimeConfig(long dbHandle, long cfHandle, + long levelSizeRatio, int minLevels, int dividingLevelOffset, boolean keepValuesInline, + long btreeKlogBlockSize, int[] encodingPipeline, boolean enableBloomFilter, double bloomFpr, + int defaultIsolationLevel, int l1FileCountTrigger, double tombstoneDensityTrigger, + long tombstoneDensityMinEntries, boolean persistToDisk) throws TidesDBException; + + private static native RangeStats nativeRangeStats(long dbHandle, long cfHandle, byte[] keyA, + byte[] keyB) throws TidesDBException; + + private static native long nativeSetCommitHook(long dbHandle, long cfHandle, CommitHook hook, + long oldCtxHandle) throws TidesDBException; } diff --git a/src/main/java/com/tidesdb/ColumnFamilyConfig.java b/src/main/java/com/tidesdb/ColumnFamilyConfig.java index 809038f..828a90c 100644 --- a/src/main/java/com/tidesdb/ColumnFamilyConfig.java +++ b/src/main/java/com/tidesdb/ColumnFamilyConfig.java @@ -18,14 +18,19 @@ */ package com.tidesdb; +import java.util.Arrays; + /** - * Configuration for a column family. Use {@link #builder()} to construct a - * configuration with custom values, or {@link #defaultConfig()} to obtain a - * configuration with default settings. + * Per-column-family configuration. Every field is mutable at runtime via + * {@link ColumnFamily#updateRuntimeConfig(ColumnFamilyConfig, boolean)}, since + * keys are ordered byte-wise and SSTables are therefore always mergeable. + * + *

The memtable, write-ahead log, and their sync and skip-list settings are + * database-level and live on {@link Config}, not here. * - *

Instances are immutable once built. Unlike {@link Config.Builder}, the - * {@link Builder#build()} method does not perform Java-side validation of the - * field values. + *

Use {@link #builder()} to construct a configuration with custom values, or + * {@link #defaultConfig()} to obtain one carrying the native library's own + * defaults. Instances are immutable once built. */ public class ColumnFamilyConfig { @@ -33,424 +38,328 @@ public class ColumnFamilyConfig { NativeLibrary.load(); } - private long writeBufferSize; - private long levelSizeRatio; - private int minLevels; - private int dividingLevelOffset; - private long klogValueThreshold; - private CompressionAlgorithm compressionAlgorithm; - private boolean enableBloomFilter; - private double bloomFPR; - private boolean enableBlockIndexes; - private int indexSampleRatio; - private int blockIndexPrefixLen; - private SyncMode syncMode; - private long syncIntervalUs; - private String comparatorName; - private int skipListMaxLevel; - private float skipListProbability; - private IsolationLevel defaultIsolationLevel; - private long minDiskSpace; - private int l1FileCountTrigger; - private int l0QueueStallThreshold; - private double tombstoneDensityTrigger; - private long tombstoneDensityMinEntries; - private boolean useBtree; - private boolean objectLazyCompaction; - private boolean objectPrefetchCompaction; + /** + * The longest a column family name may be, including its terminator. + */ + public static final int MAX_NAME_LENGTH = 128; + + /** + * The most stacked encodings a column family may apply. + */ + public static final int MAX_ENCODING_PIPELINE = 8; + + private final String name; + private final long levelSizeRatio; + private final int minLevels; + private final int dividingLevelOffset; + private final boolean keepValuesInline; + private final long btreeKlogBlockSize; + private final int[] encodingPipeline; + private final boolean enableBloomFilter; + private final double bloomFpr; + private final IsolationLevel defaultIsolationLevel; + private final int l1FileCountTrigger; + private final double tombstoneDensityTrigger; + private final long tombstoneDensityMinEntries; private ColumnFamilyConfig(Builder builder) { - this.writeBufferSize = builder.writeBufferSize; + this.name = builder.name; this.levelSizeRatio = builder.levelSizeRatio; this.minLevels = builder.minLevels; this.dividingLevelOffset = builder.dividingLevelOffset; - this.klogValueThreshold = builder.klogValueThreshold; - this.compressionAlgorithm = builder.compressionAlgorithm; + this.keepValuesInline = builder.keepValuesInline; + this.btreeKlogBlockSize = builder.btreeKlogBlockSize; + this.encodingPipeline = builder.encodingPipeline.clone(); this.enableBloomFilter = builder.enableBloomFilter; - this.bloomFPR = builder.bloomFPR; - this.enableBlockIndexes = builder.enableBlockIndexes; - this.indexSampleRatio = builder.indexSampleRatio; - this.blockIndexPrefixLen = builder.blockIndexPrefixLen; - this.syncMode = builder.syncMode; - this.syncIntervalUs = builder.syncIntervalUs; - this.comparatorName = builder.comparatorName; - this.skipListMaxLevel = builder.skipListMaxLevel; - this.skipListProbability = builder.skipListProbability; + this.bloomFpr = builder.bloomFpr; this.defaultIsolationLevel = builder.defaultIsolationLevel; - this.minDiskSpace = builder.minDiskSpace; this.l1FileCountTrigger = builder.l1FileCountTrigger; - this.l0QueueStallThreshold = builder.l0QueueStallThreshold; this.tombstoneDensityTrigger = builder.tombstoneDensityTrigger; this.tombstoneDensityMinEntries = builder.tombstoneDensityMinEntries; - this.useBtree = builder.useBtree; - this.objectLazyCompaction = builder.objectLazyCompaction; - this.objectPrefetchCompaction = builder.objectPrefetchCompaction; } /** - * Creates a default column family configuration. The tombstone density defaults - * are sourced from the underlying C library so that this binding tracks the - * engine's defaults automatically. + * Returns a configuration carrying the native library's own defaults, as + * returned by {@code tidesdb_default_column_family_config()}. Use + * {@link #toBuilder()} to adjust individual fields. * - * @return a new {@code ColumnFamilyConfig} with default values + * @return a new {@code ColumnFamilyConfig} holding the native defaults */ public static ColumnFamilyConfig defaultConfig() { - return new Builder() - .writeBufferSize(128 * 1024 * 1024) - .levelSizeRatio(10) - .minLevels(5) - .dividingLevelOffset(2) - .klogValueThreshold(512) - .compressionAlgorithm(CompressionAlgorithm.LZ4_COMPRESSION) - .enableBloomFilter(true) - .bloomFPR(0.01) - .enableBlockIndexes(true) - .indexSampleRatio(1) - .blockIndexPrefixLen(16) - .syncMode(SyncMode.SYNC_FULL) - .syncIntervalUs(1000000) - .comparatorName("") - .skipListMaxLevel(12) - .skipListProbability(0.25f) - .defaultIsolationLevel(IsolationLevel.READ_COMMITTED) - .minDiskSpace(100 * 1024 * 1024) - .l1FileCountTrigger(4) - .l0QueueStallThreshold(20) - .tombstoneDensityTrigger(nativeDefaultTombstoneDensityTrigger()) - .tombstoneDensityMinEntries(nativeDefaultTombstoneDensityMinEntries()) - .useBtree(false) - .objectLazyCompaction(false) - .objectPrefetchCompaction(true) - .build(); + return nativeDefaultConfig(); } /** - * Creates a new builder with default values. - * - * @return a new {@code Builder} + * Reads {@code tidesdb_default_column_family_config()}. */ - public static Builder builder() { - return new Builder(); - } + private static native ColumnFamilyConfig nativeDefaultConfig(); /** - * Constructs a ColumnFamilyConfig from raw native primitives. Used by the JNI - * layer when reading back the configuration embedded in tidesdb_stats_t. + * Assembles a configuration from the flat field list the JNI bridge reads + * out of {@code tidesdb_column_family_config_t}. Called from native code + * only. */ - static ColumnFamilyConfig fromNative(long writeBufferSize, long levelSizeRatio, int minLevels, - int dividingLevelOffset, long klogValueThreshold, - int compressionAlgorithm, boolean enableBloomFilter, - double bloomFPR, boolean enableBlockIndexes, - int indexSampleRatio, int blockIndexPrefixLen, - int syncMode, long syncIntervalUs, String comparatorName, - int skipListMaxLevel, float skipListProbability, - int defaultIsolationLevel, long minDiskSpace, - int l1FileCountTrigger, int l0QueueStallThreshold, + static ColumnFamilyConfig fromNative(String name, long levelSizeRatio, int minLevels, + int dividingLevelOffset, boolean keepValuesInline, + long btreeKlogBlockSize, int[] encodingPipeline, + boolean enableBloomFilter, double bloomFpr, + int defaultIsolationLevel, int l1FileCountTrigger, double tombstoneDensityTrigger, - long tombstoneDensityMinEntries, boolean useBtree, - boolean objectLazyCompaction, - boolean objectPrefetchCompaction) { + long tombstoneDensityMinEntries) { return new Builder() - .writeBufferSize(writeBufferSize) + .name(name) .levelSizeRatio(levelSizeRatio) .minLevels(minLevels) .dividingLevelOffset(dividingLevelOffset) - .klogValueThreshold(klogValueThreshold) - .compressionAlgorithm(CompressionAlgorithm.fromValue(compressionAlgorithm)) + .keepValuesInline(keepValuesInline) + .btreeKlogBlockSize(btreeKlogBlockSize) + .encodingPipelineIds(encodingPipeline) .enableBloomFilter(enableBloomFilter) - .bloomFPR(bloomFPR) - .enableBlockIndexes(enableBlockIndexes) - .indexSampleRatio(indexSampleRatio) - .blockIndexPrefixLen(blockIndexPrefixLen) - .syncMode(SyncMode.fromValue(syncMode)) - .syncIntervalUs(syncIntervalUs) - .comparatorName(comparatorName == null ? "" : comparatorName) - .skipListMaxLevel(skipListMaxLevel) - .skipListProbability(skipListProbability) + .bloomFpr(bloomFpr) .defaultIsolationLevel(IsolationLevel.fromValue(defaultIsolationLevel)) - .minDiskSpace(minDiskSpace) .l1FileCountTrigger(l1FileCountTrigger) - .l0QueueStallThreshold(l0QueueStallThreshold) .tombstoneDensityTrigger(tombstoneDensityTrigger) .tombstoneDensityMinEntries(tombstoneDensityMinEntries) - .useBtree(useBtree) - .objectLazyCompaction(objectLazyCompaction) - .objectPrefetchCompaction(objectPrefetchCompaction) .build(); } /** - * Returns the write-buffer (memtable) size in bytes. - * - * @return the write-buffer size in bytes - */ - public long getWriteBufferSize() { return writeBufferSize; } - - /** - * Returns the size ratio between adjacent LSM levels. + * Creates a new builder with the native library's defaults as its starting + * point. * - * @return the level size ratio - */ - public long getLevelSizeRatio() { return levelSizeRatio; } - - /** - * Returns the minimum number of LSM levels. - * - * @return the minimum level count - */ - public int getMinLevels() { return minLevels; } - - /** - * Returns the dividing level offset used for tiering decisions. - * - * @return the dividing level offset - */ - public int getDividingLevelOffset() { return dividingLevelOffset; } - - /** - * Returns the key-log value threshold in bytes. Values at or below this - * size are stored inline in the key log. - * - * @return the key-log value threshold in bytes + * @return a new {@code Builder} */ - public long getKlogValueThreshold() { return klogValueThreshold; } + public static Builder builder() { + return defaultConfig().toBuilder(); + } /** - * Returns the compression algorithm used for SSTables. + * Returns a builder pre-populated with this configuration's values. * - * @return the compression algorithm + * @return a new {@code Builder} carrying these values */ - public CompressionAlgorithm getCompressionAlgorithm() { return compressionAlgorithm; } + public Builder toBuilder() { + return new Builder() + .name(name) + .levelSizeRatio(levelSizeRatio) + .minLevels(minLevels) + .dividingLevelOffset(dividingLevelOffset) + .keepValuesInline(keepValuesInline) + .btreeKlogBlockSize(btreeKlogBlockSize) + .encodingPipelineIds(encodingPipeline) + .enableBloomFilter(enableBloomFilter) + .bloomFpr(bloomFpr) + .defaultIsolationLevel(defaultIsolationLevel) + .l1FileCountTrigger(l1FileCountTrigger) + .tombstoneDensityTrigger(tombstoneDensityTrigger) + .tombstoneDensityMinEntries(tombstoneDensityMinEntries); + } /** - * Returns whether Bloom filters are enabled for this column family. + * Returns the column family's persisted identity. Empty on a configuration + * you built yourself: the name passed to + * {@link TidesDB#createColumnFamily(String, ColumnFamilyConfig)} is + * authoritative and this field is ignored there. It carries the family's + * name on a configuration read back from {@link CfStats#getConfig()}. * - * @return {@code true} if Bloom filters are enabled + * @return the column family name, never {@code null} */ - public boolean isEnableBloomFilter() { return enableBloomFilter; } + public String getName() { + return name; + } /** - * Returns the Bloom filter false-positive rate. + * Returns the target size ratio between successive levels. * - * @return the false-positive rate + * @return the level size ratio */ - public double getBloomFPR() { return bloomFPR; } + public long getLevelSizeRatio() { + return levelSizeRatio; + } /** - * Returns whether block indexes are enabled for this column family. + * Returns the floor on the level count. The tree deepens as it fills and + * sheds levels again as data is deleted, and this is the depth it will not + * shed below. The engine keeps its own floor of a flush tier plus one level + * for merges to land in, so a smaller value has no further effect. * - * @return {@code true} if block indexes are enabled + * @return the minimum level count */ - public boolean isEnableBlockIndexes() { return enableBlockIndexes; } + public int getMinLevels() { + return minLevels; + } /** - * Returns the index sample ratio for block indexes. + * Returns how far above the largest level the dividing level sits, so 1 + * means X = L - 2. The dividing level is where a merge writes output + * partitioned to the largest level's file boundaries, which is what lets + * later merges take one group of overlapping files at a time. * - * @return the index sample ratio + * @return the dividing level offset */ - public int getIndexSampleRatio() { return indexSampleRatio; } + public int getDividingLevelOffset() { + return dividingLevelOffset; + } /** - * Returns the block index prefix length. + * Returns whether every value is held in the key log whatever its size, + * ignoring the database's {@link Config#getValueSeparationThreshold()}. * - * @return the block index prefix length - */ - public int getBlockIndexPrefixLen() { return blockIndexPrefixLen; } - - /** - * Returns the sync mode for durability control. + *

A separated value costs a scan one value-log read per row, so a family + * that is scanned far more than it is merged can be worth keeping whole even + * though its values are large. The cost is the one the threshold exists to + * avoid, that compaction rewrites those bytes on every merge. * - * @return the sync mode + * @return {@code true} when values stay inline regardless of size */ - public SyncMode getSyncMode() { return syncMode; } + public boolean isKeepValuesInline() { + return keepValuesInline; + } /** - * Returns the sync interval in microseconds. Only meaningful when - * {@code syncMode} is {@link SyncMode#SYNC_INTERVAL}. + * Returns the target size in bytes of a btree key-log node. Raise it + * alongside the database's {@link Config#getValueSeparationThreshold()} + * rather than on its own. * - * @return the sync interval in microseconds + * @return the block size in bytes, or 0 to leave the choice to the btree */ - public long getSyncIntervalUs() { return syncIntervalUs; } + public long getBtreeKlogBlockSize() { + return btreeKlogBlockSize; + } /** - * Returns the custom comparator name. An empty string indicates the - * default lexicographic comparator. + * Returns the encoding ids applied in order to btree key-log nodes and + * undone in reverse on read. Ids in the range of {@link CompressionAlgorithm} + * name a built-in compression codec. * - * @return the comparator name, or an empty string for the default + * @return a copy of the pipeline, empty when data is stored verbatim */ - public String getComparatorName() { return comparatorName; } + public int[] getEncodingPipeline() { + return encodingPipeline.clone(); + } /** - * Returns the maximum level for the skip-list memtable. + * Returns whether a partition-range filter is built for point-get pruning. * - * @return the skip-list maximum level + * @return {@code true} when the bloom filter is enabled */ - public int getSkipListMaxLevel() { return skipListMaxLevel; } + public boolean isEnableBloomFilter() { + return enableBloomFilter; + } /** - * Returns the probability parameter for skip-list level promotion. + * Returns the target bloom false-positive rate when the filter is enabled. * - * @return the skip-list probability + * @return the false-positive rate */ - public float getSkipListProbability() { return skipListProbability; } + public double getBloomFpr() { + return bloomFpr; + } /** - * Returns the default isolation level for transactions on this column - * family. + * Returns the isolation applied to a transaction opened against this family + * without an explicit level. * * @return the default isolation level */ - public IsolationLevel getDefaultIsolationLevel() { return defaultIsolationLevel; } - - /** - * Returns the minimum disk space in bytes required before writes are - * rejected. - * - * @return the minimum disk space in bytes - */ - public long getMinDiskSpace() { return minDiskSpace; } + public IsolationLevel getDefaultIsolationLevel() { + return defaultIsolationLevel; + } /** - * Returns the L1 file-count trigger for compaction. + * Returns the L1 SSTable count that triggers compaction. * * @return the L1 file count trigger */ - public int getL1FileCountTrigger() { return l1FileCountTrigger; } - - /** - * Returns the L0 queue stall threshold. When the L0 file count reaches - * this threshold, writes are stalled until compaction reduces the count. - * - * @return the L0 queue stall threshold - */ - public int getL0QueueStallThreshold() { return l0QueueStallThreshold; } - public double getTombstoneDensityTrigger() { return tombstoneDensityTrigger; } - public long getTombstoneDensityMinEntries() { return tombstoneDensityMinEntries; } - public boolean isUseBtree() { return useBtree; } - public boolean isObjectLazyCompaction() { return objectLazyCompaction; } - public boolean isObjectPrefetchCompaction() { return objectPrefetchCompaction; } + public int getL1FileCountTrigger() { + return l1FileCountTrigger; + } /** - * Saves this column family configuration to an INI file under the given section. - * If the file already exists it is overwritten. The written file can be read back - * with {@link #loadFromIni(String, String)}. - * - *

Note: not every field round-trips. The persisted fields are the ones the engine - * stores in a column family's {@code config.ini} (write buffer size, level ratios, - * compression, bloom/index settings, sync mode, skip list parameters, isolation level, - * compaction triggers, tombstone density, B+tree and object-store flags, and the - * comparator name). Runtime-only fields such as commit hooks are not persisted.

+ * Returns the ratio in [0, 1] above which an SSTable's tombstone density + * escalates compaction. * - * @param iniFile path to the INI file to write - * @param sectionName section name to write the configuration under - * @throws TidesDBException if the file cannot be written + * @return the density trigger, or 0 to disable it */ - public void saveToIni(String iniFile, String sectionName) throws TidesDBException { - if (iniFile == null || iniFile.isEmpty()) { - throw new IllegalArgumentException("INI file path cannot be null or empty"); - } - if (sectionName == null || sectionName.isEmpty()) { - throw new IllegalArgumentException("Section name cannot be null or empty"); - } - nativeSaveToIni(iniFile, sectionName, - writeBufferSize, levelSizeRatio, minLevels, dividingLevelOffset, klogValueThreshold, - compressionAlgorithm.getValue(), enableBloomFilter, bloomFPR, enableBlockIndexes, - indexSampleRatio, blockIndexPrefixLen, syncMode.getValue(), syncIntervalUs, - comparatorName, skipListMaxLevel, skipListProbability, - defaultIsolationLevel.getValue(), minDiskSpace, l1FileCountTrigger, - l0QueueStallThreshold, tombstoneDensityTrigger, tombstoneDensityMinEntries, - useBtree, objectLazyCompaction, objectPrefetchCompaction); + public double getTombstoneDensityTrigger() { + return tombstoneDensityTrigger; } /** - * Loads a column family configuration from an INI file section previously written by - * {@link #saveToIni(String, String)} (or produced by the engine for an existing column - * family). Fields absent from the section fall back to the engine defaults. + * Returns the minimum entry count for an SSTable to be judged by the density + * trigger, filtering tiny-SSTable noise. * - * @param iniFile path to the INI file to read - * @param sectionName section name to read the configuration from - * @return the loaded configuration - * @throws TidesDBException if the file cannot be read or the section is missing + * @return the minimum entry count, or 0 to impose no minimum */ - public static ColumnFamilyConfig loadFromIni(String iniFile, String sectionName) throws TidesDBException { - if (iniFile == null || iniFile.isEmpty()) { - throw new IllegalArgumentException("INI file path cannot be null or empty"); - } - if (sectionName == null || sectionName.isEmpty()) { - throw new IllegalArgumentException("Section name cannot be null or empty"); - } - return nativeLoadFromIni(iniFile, sectionName); + public long getTombstoneDensityMinEntries() { + return tombstoneDensityMinEntries; } - private static native double nativeDefaultTombstoneDensityTrigger(); - private static native long nativeDefaultTombstoneDensityMinEntries(); - private static native void nativeSaveToIni(String iniFile, String sectionName, - long writeBufferSize, long levelSizeRatio, int minLevels, int dividingLevelOffset, - long klogValueThreshold, int compressionAlgorithm, boolean enableBloomFilter, - double bloomFPR, boolean enableBlockIndexes, int indexSampleRatio, int blockIndexPrefixLen, - int syncMode, long syncIntervalUs, String comparatorName, int skipListMaxLevel, - float skipListProbability, int defaultIsolationLevel, long minDiskSpace, - int l1FileCountTrigger, int l0QueueStallThreshold, double tombstoneDensityTrigger, - long tombstoneDensityMinEntries, boolean useBtree, boolean objectLazyCompaction, - boolean objectPrefetchCompaction) throws TidesDBException; - private static native ColumnFamilyConfig nativeLoadFromIni(String iniFile, String sectionName) throws TidesDBException; + @Override + public String toString() { + return "ColumnFamilyConfig{" + + "name='" + name + '\'' + + ", levelSizeRatio=" + levelSizeRatio + + ", minLevels=" + minLevels + + ", dividingLevelOffset=" + dividingLevelOffset + + ", keepValuesInline=" + keepValuesInline + + ", btreeKlogBlockSize=" + btreeKlogBlockSize + + ", encodingPipeline=" + Arrays.toString(encodingPipeline) + + ", enableBloomFilter=" + enableBloomFilter + + ", bloomFpr=" + bloomFpr + + ", defaultIsolationLevel=" + defaultIsolationLevel + + ", l1FileCountTrigger=" + l1FileCountTrigger + + ", tombstoneDensityTrigger=" + tombstoneDensityTrigger + + ", tombstoneDensityMinEntries=" + tombstoneDensityMinEntries + + '}'; + } /** - * Builder for {@link ColumnFamilyConfig}. All fields have sensible - * defaults matching {@link #defaultConfig()}. Call {@link #build()} to - * create the immutable configuration. - * - *

Unlike {@link Config.Builder}, the {@link #build()} method does not - * perform Java-side validation of field values. + * Builder for {@link ColumnFamilyConfig}. Call {@link #build()} to create + * the immutable configuration; {@code build()} validates all fields. */ public static class Builder { - private long writeBufferSize = 128 * 1024 * 1024; + + private String name = ""; + private long levelSizeRatio = 0; + private int minLevels = 0; + private int dividingLevelOffset = 0; + private boolean keepValuesInline = false; + private long btreeKlogBlockSize = 0; + private int[] encodingPipeline = new int[0]; + private boolean enableBloomFilter = false; + private double bloomFpr = 0.0; + private IsolationLevel defaultIsolationLevel = IsolationLevel.READ_COMMITTED; + private int l1FileCountTrigger = 0; + private double tombstoneDensityTrigger = 0.0; + private long tombstoneDensityMinEntries = 0; /** - * Creates a new builder with default values. + * Creates a new builder with zeroed values. Prefer + * {@link ColumnFamilyConfig#builder()}, which starts from the native + * library's defaults. */ public Builder() { } - private long levelSizeRatio = 10; - private int minLevels = 5; - private int dividingLevelOffset = 2; - private long klogValueThreshold = 512; - private CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.LZ4_COMPRESSION; - private boolean enableBloomFilter = true; - private double bloomFPR = 0.01; - private boolean enableBlockIndexes = true; - private int indexSampleRatio = 1; - private int blockIndexPrefixLen = 16; - private SyncMode syncMode = SyncMode.SYNC_FULL; - private long syncIntervalUs = 1000000; - private String comparatorName = ""; - private int skipListMaxLevel = 12; - private float skipListProbability = 0.25f; - private IsolationLevel defaultIsolationLevel = IsolationLevel.READ_COMMITTED; - private long minDiskSpace = 100 * 1024 * 1024; - private int l1FileCountTrigger = 4; - private int l0QueueStallThreshold = 20; - private double tombstoneDensityTrigger = 0.0; - private long tombstoneDensityMinEntries = 1024; - private boolean useBtree = false; - private boolean objectLazyCompaction = false; - private boolean objectPrefetchCompaction = true; /** - * Sets the write-buffer (memtable) size in bytes. + * Sets the column family name. Ignored by + * {@link TidesDB#createColumnFamily(String, ColumnFamilyConfig)} and by + * {@link ColumnFamily#updateRuntimeConfig(ColumnFamilyConfig, boolean)}, + * both of which take the name from elsewhere. * - * @param writeBufferSize the size in bytes + * @param name the name; {@code null} is treated as empty * @return this builder */ - public Builder writeBufferSize(long writeBufferSize) { - this.writeBufferSize = writeBufferSize; + public Builder name(String name) { + this.name = name == null ? "" : name; return this; } /** - * Sets the size ratio between adjacent LSM levels. + * Sets the target size ratio between successive levels. * - * @param levelSizeRatio the level size ratio + * @param levelSizeRatio the ratio * @return this builder */ public Builder levelSizeRatio(long levelSizeRatio) { @@ -459,7 +368,7 @@ public Builder levelSizeRatio(long levelSizeRatio) { } /** - * Sets the minimum number of LSM levels. + * Sets the floor on the level count. * * @param minLevels the minimum level count * @return this builder @@ -470,7 +379,7 @@ public Builder minLevels(int minLevels) { } /** - * Sets the dividing level offset for tiering decisions. + * Sets how far above the largest level the dividing level sits. * * @param dividingLevelOffset the offset * @return this builder @@ -481,146 +390,110 @@ public Builder dividingLevelOffset(int dividingLevelOffset) { } /** - * Sets the key-log value threshold in bytes. - * - * @param klogValueThreshold the threshold in bytes - * @return this builder - */ - public Builder klogValueThreshold(long klogValueThreshold) { - this.klogValueThreshold = klogValueThreshold; - return this; - } - - /** - * Sets the compression algorithm for SSTables. - * - * @param compressionAlgorithm the compression algorithm; must not be - * {@code null} - * @return this builder - */ - public Builder compressionAlgorithm(CompressionAlgorithm compressionAlgorithm) { - this.compressionAlgorithm = compressionAlgorithm; - return this; - } - - /** - * Enables or disables Bloom filters. - * - * @param enableBloomFilter {@code true} to enable - * @return this builder - */ - public Builder enableBloomFilter(boolean enableBloomFilter) { - this.enableBloomFilter = enableBloomFilter; - return this; - } - - /** - * Sets the Bloom filter false-positive rate. - * - * @param bloomFPR the false-positive rate - * @return this builder - */ - public Builder bloomFPR(double bloomFPR) { - this.bloomFPR = bloomFPR; - return this; - } - - /** - * Enables or disables block indexes. - * - * @param enableBlockIndexes {@code true} to enable - * @return this builder - */ - public Builder enableBlockIndexes(boolean enableBlockIndexes) { - this.enableBlockIndexes = enableBlockIndexes; - return this; - } - - /** - * Sets the index sample ratio for block indexes. + * Sets whether every value is held in the key log whatever its size. * - * @param indexSampleRatio the sample ratio + * @param keepValuesInline {@code true} to keep values inline * @return this builder */ - public Builder indexSampleRatio(int indexSampleRatio) { - this.indexSampleRatio = indexSampleRatio; + public Builder keepValuesInline(boolean keepValuesInline) { + this.keepValuesInline = keepValuesInline; return this; } /** - * Sets the block index prefix length. + * Sets the target size of a btree key-log node. * - * @param blockIndexPrefixLen the prefix length + * @param btreeKlogBlockSize the size in bytes, or 0 to leave the choice + * to the btree * @return this builder */ - public Builder blockIndexPrefixLen(int blockIndexPrefixLen) { - this.blockIndexPrefixLen = blockIndexPrefixLen; + public Builder btreeKlogBlockSize(long btreeKlogBlockSize) { + this.btreeKlogBlockSize = btreeKlogBlockSize; return this; } /** - * Sets the sync mode for durability control. + * Sets the encoding pipeline from built-in compression algorithms, + * applied in the order given. * - * @param syncMode the sync mode; must not be {@code null} + * @param algorithms the algorithms; an empty list stores data verbatim * @return this builder */ - public Builder syncMode(SyncMode syncMode) { - this.syncMode = syncMode; + public Builder encodingPipeline(CompressionAlgorithm... algorithms) { + if (algorithms == null) { + this.encodingPipeline = new int[0]; + return this; + } + int[] ids = new int[algorithms.length]; + for (int i = 0; i < algorithms.length; i++) { + if (algorithms[i] == null) { + throw new IllegalArgumentException("Encoding pipeline entry cannot be null"); + } + ids[i] = algorithms[i].getValue(); + } + this.encodingPipeline = ids; return this; } /** - * Sets the sync interval in microseconds. Only meaningful when - * {@code syncMode} is {@link SyncMode#SYNC_INTERVAL}. + * Sets the encoding pipeline from raw encoding ids, applied in the order + * given. Use this for an encoding the {@link CompressionAlgorithm} enum + * does not name. * - * @param syncIntervalUs the interval in microseconds + * @param ids the encoding ids, each in [0, 255]; an empty array stores + * data verbatim * @return this builder */ - public Builder syncIntervalUs(long syncIntervalUs) { - this.syncIntervalUs = syncIntervalUs; + public Builder encodingPipelineIds(int... ids) { + this.encodingPipeline = ids == null ? new int[0] : ids.clone(); return this; } /** - * Sets the custom comparator name. An empty string selects the - * default lexicographic comparator. + * Sets a single-codec encoding pipeline, the common case. Passing + * {@link CompressionAlgorithm#NONE} clears the pipeline so + * data is stored verbatim. * - * @param comparatorName the comparator name + * @param algorithm the algorithm; must not be {@code null} * @return this builder */ - public Builder comparatorName(String comparatorName) { - this.comparatorName = comparatorName; - return this; + public Builder compression(CompressionAlgorithm algorithm) { + if (algorithm == null) { + throw new IllegalArgumentException("Compression algorithm cannot be null"); + } + if (algorithm == CompressionAlgorithm.NONE) { + return encodingPipelineIds(); + } + return encodingPipeline(algorithm); } /** - * Sets the maximum level for the skip-list memtable. + * Sets whether a partition-range filter is built for point-get pruning. * - * @param skipListMaxLevel the maximum level + * @param enableBloomFilter {@code true} to enable the filter * @return this builder */ - public Builder skipListMaxLevel(int skipListMaxLevel) { - this.skipListMaxLevel = skipListMaxLevel; + public Builder enableBloomFilter(boolean enableBloomFilter) { + this.enableBloomFilter = enableBloomFilter; return this; } /** - * Sets the probability parameter for skip-list level promotion. + * Sets the target bloom false-positive rate. * - * @param skipListProbability the promotion probability + * @param bloomFpr the rate, in (0.0, 1.0) when the filter is enabled * @return this builder */ - public Builder skipListProbability(float skipListProbability) { - this.skipListProbability = skipListProbability; + public Builder bloomFpr(double bloomFpr) { + this.bloomFpr = bloomFpr; return this; } /** - * Sets the default isolation level for transactions on this column - * family. + * Sets the isolation applied to a transaction opened against this family + * without an explicit level. * - * @param defaultIsolationLevel the isolation level; must not be - * {@code null} + * @param defaultIsolationLevel the level; must not be {@code null} * @return this builder */ public Builder defaultIsolationLevel(IsolationLevel defaultIsolationLevel) { @@ -629,21 +502,9 @@ public Builder defaultIsolationLevel(IsolationLevel defaultIsolationLevel) { } /** - * Sets the minimum disk space in bytes required before writes are - * rejected. - * - * @param minDiskSpace the minimum disk space in bytes - * @return this builder - */ - public Builder minDiskSpace(long minDiskSpace) { - this.minDiskSpace = minDiskSpace; - return this; - } - - /** - * Sets the L1 file-count trigger for compaction. + * Sets the L1 SSTable count that triggers compaction. * - * @param l1FileCountTrigger the file count trigger + * @param l1FileCountTrigger the trigger count * @return this builder */ public Builder l1FileCountTrigger(int l1FileCountTrigger) { @@ -652,47 +513,34 @@ public Builder l1FileCountTrigger(int l1FileCountTrigger) { } /** - * Sets the L0 queue stall threshold. When the L0 file count reaches - * this threshold, writes are stalled until compaction reduces the count. + * Sets the tombstone density above which compaction is escalated. * - * @param l0QueueStallThreshold the stall threshold + * @param tombstoneDensityTrigger the ratio in [0.0, 1.0], or 0 to disable * @return this builder */ - public Builder l0QueueStallThreshold(int l0QueueStallThreshold) { - this.l0QueueStallThreshold = l0QueueStallThreshold; - return this; - } - public Builder tombstoneDensityTrigger(double tombstoneDensityTrigger) { this.tombstoneDensityTrigger = tombstoneDensityTrigger; return this; } + /** + * Sets the minimum entry count for an SSTable to be judged by the + * density trigger. + * + * @param tombstoneDensityMinEntries the minimum, or 0 for no minimum + * @return this builder + */ public Builder tombstoneDensityMinEntries(long tombstoneDensityMinEntries) { this.tombstoneDensityMinEntries = tombstoneDensityMinEntries; return this; } - public Builder useBtree(boolean useBtree) { - this.useBtree = useBtree; - return this; - } - - public Builder objectLazyCompaction(boolean objectLazyCompaction) { - this.objectLazyCompaction = objectLazyCompaction; - return this; - } - - public Builder objectPrefetchCompaction(boolean objectPrefetchCompaction) { - this.objectPrefetchCompaction = objectPrefetchCompaction; - return this; - } - /** - * Creates the immutable {@link ColumnFamilyConfig}. No Java-side - * validation of field values is performed. + * Validates all fields and creates the immutable + * {@link ColumnFamilyConfig}. * * @return a new {@code ColumnFamilyConfig} + * @throws IllegalArgumentException if any field is invalid */ public ColumnFamilyConfig build() { validate(); @@ -700,69 +548,60 @@ public ColumnFamilyConfig build() { } private void validate() { - // Nullable-enum checks (do first, before field-level checks) - if (compressionAlgorithm == null) { - throw new IllegalArgumentException("compressionAlgorithm must not be null"); - } - if (syncMode == null) { - throw new IllegalArgumentException("syncMode must not be null"); - } - if (defaultIsolationLevel == null) { - throw new IllegalArgumentException("defaultIsolationLevel must not be null"); - } - - // Non-negative (zero sentinel OK) - if (klogValueThreshold < 0) { - throw new IllegalArgumentException("klogValueThreshold must not be negative, was: " + klogValueThreshold); + if (name.length() >= MAX_NAME_LENGTH) { + throw new IllegalArgumentException( + "name must be shorter than " + MAX_NAME_LENGTH + " characters, was: " + + name.length()); } - if (syncIntervalUs < 0) { - throw new IllegalArgumentException("syncIntervalUs must not be negative, was: " + syncIntervalUs); + if (levelSizeRatio < 0) { + throw new IllegalArgumentException( + "levelSizeRatio must not be negative, was: " + levelSizeRatio); } - if (minDiskSpace < 0) { - throw new IllegalArgumentException("minDiskSpace must not be negative, was: " + minDiskSpace); + if (minLevels < 0) { + throw new IllegalArgumentException( + "minLevels must not be negative, was: " + minLevels); } if (dividingLevelOffset < 0) { - throw new IllegalArgumentException("dividingLevelOffset must not be negative, was: " + dividingLevelOffset); - } - if (blockIndexPrefixLen < 0) { - throw new IllegalArgumentException("blockIndexPrefixLen must not be negative, was: " + blockIndexPrefixLen); - } - if (skipListMaxLevel < 0) { - throw new IllegalArgumentException("skipListMaxLevel must not be negative, was: " + skipListMaxLevel); + throw new IllegalArgumentException( + "dividingLevelOffset must not be negative, was: " + dividingLevelOffset); } - - // Positive-required (zero rejected) - if (writeBufferSize <= 0) { - throw new IllegalArgumentException("writeBufferSize must be positive, was: " + writeBufferSize); - } - if (levelSizeRatio <= 0) { - throw new IllegalArgumentException("levelSizeRatio must be positive, was: " + levelSizeRatio); - } - if (minLevels <= 0) { - throw new IllegalArgumentException("minLevels must be positive, was: " + minLevels); + if (btreeKlogBlockSize < 0) { + throw new IllegalArgumentException( + "btreeKlogBlockSize must not be negative, was: " + btreeKlogBlockSize); } - if (indexSampleRatio <= 0) { - throw new IllegalArgumentException("indexSampleRatio must be positive, was: " + indexSampleRatio); + if (encodingPipeline.length > MAX_ENCODING_PIPELINE) { + throw new IllegalArgumentException( + "encodingPipeline must hold at most " + MAX_ENCODING_PIPELINE + + " entries, was: " + encodingPipeline.length); } - if (l1FileCountTrigger <= 0) { - throw new IllegalArgumentException("l1FileCountTrigger must be positive, was: " + l1FileCountTrigger); + for (int id : encodingPipeline) { + if (id < 0 || id > 255) { + throw new IllegalArgumentException( + "encoding id must be in [0, 255], was: " + id); + } } - if (l0QueueStallThreshold <= 0) { - throw new IllegalArgumentException("l0QueueStallThreshold must be positive, was: " + l0QueueStallThreshold); + if (Double.isNaN(bloomFpr) || Double.isInfinite(bloomFpr) + || bloomFpr < 0.0 || bloomFpr >= 1.0) { + throw new IllegalArgumentException( + "bloomFpr must be finite and in [0.0, 1.0), was: " + bloomFpr); } - if (tombstoneDensityMinEntries <= 0) { - throw new IllegalArgumentException("tombstoneDensityMinEntries must be positive, was: " + tombstoneDensityMinEntries); + if (defaultIsolationLevel == null) { + throw new IllegalArgumentException("defaultIsolationLevel cannot be null"); } - - // Float/double range and finiteness - if (Double.isNaN(bloomFPR) || Double.isInfinite(bloomFPR) || bloomFPR < 0.0 || bloomFPR > 1.0) { - throw new IllegalArgumentException("bloomFPR must be finite and in [0.0, 1.0], was: " + bloomFPR); + if (l1FileCountTrigger < 0) { + throw new IllegalArgumentException( + "l1FileCountTrigger must not be negative, was: " + l1FileCountTrigger); } - if (Float.isNaN(skipListProbability) || Float.isInfinite(skipListProbability) || skipListProbability < 0.0f || skipListProbability > 1.0f) { - throw new IllegalArgumentException("skipListProbability must be finite and in [0.0, 1.0], was: " + skipListProbability); + if (Double.isNaN(tombstoneDensityTrigger) || Double.isInfinite(tombstoneDensityTrigger) + || tombstoneDensityTrigger < 0.0 || tombstoneDensityTrigger > 1.0) { + throw new IllegalArgumentException( + "tombstoneDensityTrigger must be finite and in [0.0, 1.0], was: " + + tombstoneDensityTrigger); } - if (Double.isNaN(tombstoneDensityTrigger) || Double.isInfinite(tombstoneDensityTrigger) || tombstoneDensityTrigger < 0.0 || tombstoneDensityTrigger > 1.0) { - throw new IllegalArgumentException("tombstoneDensityTrigger must be finite and in [0.0, 1.0], was: " + tombstoneDensityTrigger); + if (tombstoneDensityMinEntries < 0) { + throw new IllegalArgumentException( + "tombstoneDensityMinEntries must not be negative, was: " + + tombstoneDensityMinEntries); } } } diff --git a/src/main/java/com/tidesdb/CompressionAlgorithm.java b/src/main/java/com/tidesdb/CompressionAlgorithm.java index 1d2f58f..5c445f2 100644 --- a/src/main/java/com/tidesdb/CompressionAlgorithm.java +++ b/src/main/java/com/tidesdb/CompressionAlgorithm.java @@ -19,59 +19,68 @@ package com.tidesdb; /** - * Compression algorithm for column family SSTables. Each constant maps to an - * integer used by the JNI bridge. + * A built-in compression codec, usable as an entry in a column family's encoding + * pipeline. Each constant's {@link #getValue()} is its encoding id, which is + * persisted in SSTable and value-log metadata. + * + *

Every constant is always defined, but whether a backend can actually be + * used is a build-time choice of the native library. Ask + * {@link TidesDB#isCompressionAvailable(CompressionAlgorithm)} before choosing + * one, rather than discovering it when a node fails to decode. {@link #NONE} is + * always available. */ public enum CompressionAlgorithm { /** - * No compression. + * No compression; data is stored verbatim. */ - NO_COMPRESSION(0), + NONE(0), /** * Snappy compression. */ - SNAPPY_COMPRESSION(1), + SNAPPY(1), /** * LZ4 compression with default settings. */ - LZ4_COMPRESSION(2), + LZ4(2), /** * Zstandard compression. */ - ZSTD_COMPRESSION(3), + ZSTD(3), /** - * LZ4 compression optimized for speed. + * LZ4 compression optimised for speed. */ - LZ4_FAST_COMPRESSION(4); - + LZ4_FAST(4); + private final int value; - + CompressionAlgorithm(int value) { this.value = value; } - + /** - * Returns the JNI numeric mapping for this compression algorithm. + * Returns this codec's encoding id, the value stored in SSTable and + * value-log metadata and accepted by + * {@link ColumnFamilyConfig.Builder#encodingPipelineIds(int...)}. * - * @return the integer value passed to the native library + * @return the encoding id */ public int getValue() { return value; } - + /** * Returns the {@link CompressionAlgorithm} constant matching the given - * JNI integer value. + * encoding id. * - * @param value the JNI integer value + * @param value the encoding id * @return the matching constant * @throws IllegalArgumentException if {@code value} does not map to any - * known constant + * built-in codec */ public static CompressionAlgorithm fromValue(int value) { for (CompressionAlgorithm algo : values()) { diff --git a/src/main/java/com/tidesdb/Config.java b/src/main/java/com/tidesdb/Config.java index 19c8173..3618240 100644 --- a/src/main/java/com/tidesdb/Config.java +++ b/src/main/java/com/tidesdb/Config.java @@ -19,12 +19,25 @@ package com.tidesdb; /** - * Configuration for opening a {@link TidesDB} instance. Use {@link #builder(String)} - * to construct a configuration with custom values, or {@link #defaultConfig()} to - * obtain a configuration with default settings. + * Database-level configuration for opening a {@link TidesDB} instance. The + * memtable, write-ahead log, block cache, value log, and worker pool are + * database-level and shared by every column family, so their settings live here + * rather than on {@link ColumnFamilyConfig}. * - *

Instances are immutable once built. {@link Builder#build()} validates all fields - * and throws {@link IllegalArgumentException} for invalid values. + *

Use {@link #builder(String)} to construct a configuration with custom + * values, or {@link #defaultConfig(String)} to obtain one carrying the native + * library's own defaults. + * + *

Instances are immutable once built. {@link Builder#build()} validates all + * fields and throws {@link IllegalArgumentException} for invalid values. + * + *

A field left at zero is resolved to its default by the engine rather than + * taken literally, for {@code numFlushThreads}, {@code numCompactionThreads}, + * {@code blockCacheSize}, {@code maxOpenSSTables}, {@code vlogSegmentSize}, + * {@code memtableWriteBufferSize}, {@code memtableSkipListMaxLevel} and + * {@code memtableSkipListProbability}. The rest are used as given, + * {@code memtableSyncMode} included, where zero is the meaningful value + * {@link SyncMode#SYNC_NONE}. */ public class Config { @@ -32,26 +45,24 @@ public class Config { NativeLibrary.load(); } - private String dbPath; - private int numFlushThreads; - private int numCompactionThreads; - private LogLevel logLevel; - private long blockCacheSize; - private long maxOpenSSTables; - private boolean logToFile; - private long logTruncationAt; - private long maxMemoryUsage; - private boolean unifiedMemtable; - private long unifiedMemtableWriteBufferSize; - private int unifiedMemtableSkipListMaxLevel; - private float unifiedMemtableSkipListProbability; - private int unifiedMemtableSyncMode; - private long unifiedMemtableSyncIntervalUs; - private String objectStoreFsPath; - private ObjectStoreConfig objectStoreConfig; - private S3Config objectStoreS3Config; - private int maxConcurrentFlushes; - private boolean finishCompactionsOnClose; + private final String dbPath; + private final int numFlushThreads; + private final int numCompactionThreads; + private final LogLevel logLevel; + private final long blockCacheSize; + private final long maxOpenSSTables; + private final boolean logToFile; + private final long logTruncationAt; + private final long memtableWriteBufferSize; + private final int memtableSkipListMaxLevel; + private final float memtableSkipListProbability; + private final SyncMode memtableSyncMode; + private final long memtableSyncIntervalUs; + private final long valueSeparationThreshold; + private final long vlogSegmentSize; + private final int memtableL0QueueStallThreshold; + private final int memtableIdleFlushSeconds; + private final long txnTimeoutSeconds; private Config(Builder builder) { this.dbPath = builder.dbPath; @@ -62,52 +73,76 @@ private Config(Builder builder) { this.maxOpenSSTables = builder.maxOpenSSTables; this.logToFile = builder.logToFile; this.logTruncationAt = builder.logTruncationAt; - this.maxMemoryUsage = builder.maxMemoryUsage; - this.unifiedMemtable = builder.unifiedMemtable; - this.unifiedMemtableWriteBufferSize = builder.unifiedMemtableWriteBufferSize; - this.unifiedMemtableSkipListMaxLevel = builder.unifiedMemtableSkipListMaxLevel; - this.unifiedMemtableSkipListProbability = builder.unifiedMemtableSkipListProbability; - this.unifiedMemtableSyncMode = builder.unifiedMemtableSyncMode; - this.unifiedMemtableSyncIntervalUs = builder.unifiedMemtableSyncIntervalUs; - this.objectStoreFsPath = builder.objectStoreFsPath; - this.objectStoreConfig = builder.objectStoreConfig; - this.objectStoreS3Config = builder.objectStoreS3Config; - this.maxConcurrentFlushes = builder.maxConcurrentFlushes; - this.finishCompactionsOnClose = builder.finishCompactionsOnClose; + this.memtableWriteBufferSize = builder.memtableWriteBufferSize; + this.memtableSkipListMaxLevel = builder.memtableSkipListMaxLevel; + this.memtableSkipListProbability = builder.memtableSkipListProbability; + this.memtableSyncMode = builder.memtableSyncMode; + this.memtableSyncIntervalUs = builder.memtableSyncIntervalUs; + this.valueSeparationThreshold = builder.valueSeparationThreshold; + this.vlogSegmentSize = builder.vlogSegmentSize; + this.memtableL0QueueStallThreshold = builder.memtableL0QueueStallThreshold; + this.memtableIdleFlushSeconds = builder.memtableIdleFlushSeconds; + this.txnTimeoutSeconds = builder.txnTimeoutSeconds; } /** - * Creates a default configuration with the following values: - *

    - *
  • {@code numFlushThreads}: 2
    • - *
  • {@code numCompactionThreads}: 2
    • - *
  • {@code logLevel}: {@link LogLevel#INFO}
    • - *
  • {@code blockCacheSize}: 67108864 bytes (64 MiB)
    • - *
  • {@code maxOpenSSTables}: 256
    • - *
  • {@code maxConcurrentFlushes}: sourced from the native C library via - * {@code tidesdb_default_config()}
    • - *
+ * Creates a configuration carrying the native library's own defaults, as + * returned by {@code tidesdb_default_config()}, with {@code dbPath} applied + * on top. Use {@link #toBuilder()} to adjust individual fields. * - * @return a new {@code Config} with default values + * @param dbPath the database file-system path; must not be {@code null} or empty + * @return a new {@code Config} holding the native defaults + */ + public static Config defaultConfig(String dbPath) { + if (dbPath == null || dbPath.isEmpty()) { + throw new IllegalArgumentException("Database path cannot be null or empty"); + } + return nativeDefaultConfig().toBuilder().dbPath(dbPath).build(); + } + + /** + * Reads {@code tidesdb_default_config()}. The returned configuration carries + * an empty {@code dbPath}. */ - public static Config defaultConfig() { + private static native Config nativeDefaultConfig(); + + /** + * Assembles a configuration from the flat field list the JNI bridge reads + * out of {@code tidesdb_config_t}. Called from native code only. + */ + static Config fromNative(int numFlushThreads, int numCompactionThreads, int logLevel, + long blockCacheSize, long maxOpenSSTables, boolean logToFile, + long logTruncationAt, long memtableWriteBufferSize, + int memtableSkipListMaxLevel, float memtableSkipListProbability, + int memtableSyncMode, long memtableSyncIntervalUs, + long valueSeparationThreshold, long vlogSegmentSize, + int memtableL0QueueStallThreshold, int memtableIdleFlushSeconds, + long txnTimeoutSeconds) { return new Builder() - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .logToFile(false) - .logTruncationAt(24 * 1024 * 1024) - .maxMemoryUsage(0) - .maxConcurrentFlushes(nativeDefaultMaxConcurrentFlushes()) + .numFlushThreads(numFlushThreads) + .numCompactionThreads(numCompactionThreads) + .logLevel(LogLevel.fromValue(logLevel)) + .blockCacheSize(blockCacheSize) + .maxOpenSSTables(maxOpenSSTables) + .logToFile(logToFile) + .logTruncationAt(logTruncationAt) + .memtableWriteBufferSize(memtableWriteBufferSize) + .memtableSkipListMaxLevel(memtableSkipListMaxLevel) + .memtableSkipListProbability(memtableSkipListProbability) + .memtableSyncMode(SyncMode.fromValue(memtableSyncMode)) + .memtableSyncIntervalUs(memtableSyncIntervalUs) + .valueSeparationThreshold(valueSeparationThreshold) + .vlogSegmentSize(vlogSegmentSize) + .memtableL0QueueStallThreshold(memtableL0QueueStallThreshold) + .memtableIdleFlushSeconds(memtableIdleFlushSeconds) + .txnTimeoutSeconds(txnTimeoutSeconds) .build(); } - private static native int nativeDefaultMaxConcurrentFlushes(); - /** - * Creates a new builder with the given database path. + * Creates a new builder with the given database path. Every other field + * starts at zero, which the engine resolves to its own default where the + * class documentation says so. * * @param dbPath the database file-system path; must not be {@code null} * @return a new {@code Builder} @@ -115,7 +150,34 @@ public static Config defaultConfig() { public static Builder builder(String dbPath) { return new Builder().dbPath(dbPath); } - + + /** + * Returns a builder pre-populated with this configuration's values. + * + * @return a new {@code Builder} carrying these values + */ + public Builder toBuilder() { + return new Builder() + .dbPath(dbPath) + .numFlushThreads(numFlushThreads) + .numCompactionThreads(numCompactionThreads) + .logLevel(logLevel) + .blockCacheSize(blockCacheSize) + .maxOpenSSTables(maxOpenSSTables) + .logToFile(logToFile) + .logTruncationAt(logTruncationAt) + .memtableWriteBufferSize(memtableWriteBufferSize) + .memtableSkipListMaxLevel(memtableSkipListMaxLevel) + .memtableSkipListProbability(memtableSkipListProbability) + .memtableSyncMode(memtableSyncMode) + .memtableSyncIntervalUs(memtableSyncIntervalUs) + .valueSeparationThreshold(valueSeparationThreshold) + .vlogSegmentSize(vlogSegmentSize) + .memtableL0QueueStallThreshold(memtableL0QueueStallThreshold) + .memtableIdleFlushSeconds(memtableIdleFlushSeconds) + .txnTimeoutSeconds(txnTimeoutSeconds); + } + /** * Returns the database file-system path. * @@ -126,25 +188,25 @@ public String getDbPath() { } /** - * Returns the number of flush threads. + * Returns the number of flush worker threads. * - * @return the flush thread count + * @return the flush thread count, or 0 for the engine default */ public int getNumFlushThreads() { return numFlushThreads; } /** - * Returns the number of compaction threads. + * Returns the number of compaction worker threads. * - * @return the compaction thread count + * @return the compaction thread count, or 0 for the engine default */ public int getNumCompactionThreads() { return numCompactionThreads; } /** - * Returns the log level. + * Returns the minimum severity to emit. * * @return the log level */ @@ -153,123 +215,216 @@ public LogLevel getLogLevel() { } /** - * Returns the block cache size in bytes. + * Returns the size in bytes of the database-level block cache for hot + * SSTable blocks. * - * @return the block cache size in bytes + * @return the block cache size in bytes, or 0 for the engine default */ public long getBlockCacheSize() { return blockCacheSize; } /** - * Returns the maximum number of open SSTables. + * Returns the maximum number of concurrently open SSTable file handles. + * Lowered at open to what this process's open-file ceiling leaves; raise the + * ceiling with {@link TidesDB#raiseOpenFileLimit(long)} first if the larger + * figure is the one you want. * - * @return the maximum open SSTable count + * @return the maximum open SSTable count, or 0 for the engine default */ public long getMaxOpenSSTables() { return maxOpenSSTables; } - + + /** + * Returns whether the log is written to a file named {@code LOG} inside the + * database directory rather than to stderr. The sink is process-wide rather + * than per-database. + * + * @return {@code true} when logging to a file + */ public boolean isLogToFile() { return logToFile; } - + + /** + * Returns the size in bytes past which the log file is truncated and + * reopened. Ignored unless {@link #isLogToFile()} is set. + * + * @return the truncation threshold in bytes, or 0 for never + */ public long getLogTruncationAt() { return logTruncationAt; } - - public long getMaxMemoryUsage() { - return maxMemoryUsage; - } - public boolean isUnifiedMemtable() { - return unifiedMemtable; - } - - public long getUnifiedMemtableWriteBufferSize() { - return unifiedMemtableWriteBufferSize; + /** + * Returns the memory the active memtable may occupy before it is rotated. + * This is a memory budget rather than a promise about the size of what a + * rotation flushes: an entry costs its key and value plus about a hundred + * bytes of skip list node, pointer arrays and version struct. + * + * @return the write buffer size in bytes, or 0 for the engine default + */ + public long getMemtableWriteBufferSize() { + return memtableWriteBufferSize; } - public int getUnifiedMemtableSkipListMaxLevel() { - return unifiedMemtableSkipListMaxLevel; + /** + * Returns the skip list max level for the memtable. + * + * @return the max level, or 0 for the engine default + */ + public int getMemtableSkipListMaxLevel() { + return memtableSkipListMaxLevel; } - public float getUnifiedMemtableSkipListProbability() { - return unifiedMemtableSkipListProbability; + /** + * Returns the skip list level probability for the memtable. + * + * @return the probability, or 0 for the engine default + */ + public float getMemtableSkipListProbability() { + return memtableSkipListProbability; } - public int getUnifiedMemtableSyncMode() { - return unifiedMemtableSyncMode; + /** + * Returns the durability mode for the write-ahead log. + * + * @return the sync mode + */ + public SyncMode getMemtableSyncMode() { + return memtableSyncMode; } - public long getUnifiedMemtableSyncIntervalUs() { - return unifiedMemtableSyncIntervalUs; + /** + * Returns the fsync interval for {@link SyncMode#SYNC_INTERVAL}, in + * microseconds. Ignored under the other sync modes. + * + * @return the interval in microseconds, or 0 for a one second default + */ + public long getMemtableSyncIntervalUs() { + return memtableSyncIntervalUs; } - public String getObjectStoreFsPath() { - return objectStoreFsPath; + /** + * Returns the size at or above which values are stored in the shared value + * log and referenced from the key log, so a value stays inline only while it + * is strictly under it. + * + *

Keep it at or under a quarter of a family's + * {@link ColumnFamilyConfig#getBtreeKlogBlockSize()}: an inlined value + * approaching the node size leaves a node holding one entry and spends the + * btree fan-out that makes a lookup cheap. The pairing is advisory, and a + * config that breaks it only logs a warning. + * + * @return the threshold in bytes, or 0 for the engine default + */ + public long getValueSeparationThreshold() { + return valueSeparationThreshold; } - public ObjectStoreConfig getObjectStoreConfig() { - return objectStoreConfig; + /** + * Returns the size at which the value log seals its active segment and opens + * a fresh one. A reclaim drains every segment worth draining, so this does + * not change how much space the store settles at; it changes what reclaiming + * costs. + * + * @return the segment size in bytes, or 0 for the engine default + */ + public long getVlogSegmentSize() { + return vlogSegmentSize; } /** - * Returns the S3-compatible object store connector configuration, or null if the database - * is not backed by S3. + * Returns the immutable-queue depth at which writes stall for backpressure. + * Left at 0 the queue is unbounded and a writer outrunning the flush threads + * is never paced, so it is a value to set deliberately rather than leave. * - * @return the S3 connector config, or null + * @return the stall threshold, or 0 to never stall */ - public S3Config getObjectStoreS3Config() { - return objectStoreS3Config; + public int getMemtableL0QueueStallThreshold() { + return memtableL0QueueStallThreshold; } - public int getMaxConcurrentFlushes() { - return maxConcurrentFlushes; + /** + * Returns how long the active memtable may sit unwritten before the engine + * rotates it on its own. A database that stops taking writes otherwise holds + * that data in memory indefinitely. + * + * @return the idle flush interval in seconds, or 0 to never rotate on idle + */ + public int getMemtableIdleFlushSeconds() { + return memtableIdleFlushSeconds; } /** - * Returns the close behavior for in-flight compactions. + * Returns how long a transaction may stay active before the next operation + * on it expires it. An abandoned transaction holds its snapshot and its + * write reservations, which keeps the reclamation floor down and stops + * compaction dropping old versions. * - * @return true if {@code close()} waits for in-flight compactions to finish; - * false (default) cancels them at their next checkpoint for a fast shutdown + * @return the timeout in seconds, or 0 for no timeout */ - public boolean isFinishCompactionsOnClose() { - return finishCompactionsOnClose; + public long getTxnTimeoutSeconds() { + return txnTimeoutSeconds; + } + + @Override + public String toString() { + return "Config{" + + "dbPath='" + dbPath + '\'' + + ", numFlushThreads=" + numFlushThreads + + ", numCompactionThreads=" + numCompactionThreads + + ", logLevel=" + logLevel + + ", blockCacheSize=" + blockCacheSize + + ", maxOpenSSTables=" + maxOpenSSTables + + ", logToFile=" + logToFile + + ", logTruncationAt=" + logTruncationAt + + ", memtableWriteBufferSize=" + memtableWriteBufferSize + + ", memtableSkipListMaxLevel=" + memtableSkipListMaxLevel + + ", memtableSkipListProbability=" + memtableSkipListProbability + + ", memtableSyncMode=" + memtableSyncMode + + ", memtableSyncIntervalUs=" + memtableSyncIntervalUs + + ", valueSeparationThreshold=" + valueSeparationThreshold + + ", vlogSegmentSize=" + vlogSegmentSize + + ", memtableL0QueueStallThreshold=" + memtableL0QueueStallThreshold + + ", memtableIdleFlushSeconds=" + memtableIdleFlushSeconds + + ", txnTimeoutSeconds=" + txnTimeoutSeconds + + '}'; } /** - * Builder for {@link Config}. All fields have sensible defaults. Call + * Builder for {@link Config}. Every field starts at zero, which the engine + * resolves to its own default where {@link Config} says so. Call * {@link #build()} to create the immutable configuration; {@code build()} * validates all fields. */ public static class Builder { + private String dbPath = ""; + private int numFlushThreads = 0; + private int numCompactionThreads = 0; + private LogLevel logLevel = LogLevel.INFO; + private long blockCacheSize = 0; + private long maxOpenSSTables = 0; + private boolean logToFile = false; + private long logTruncationAt = 0; + private long memtableWriteBufferSize = 0; + private int memtableSkipListMaxLevel = 0; + private float memtableSkipListProbability = 0.0f; + private SyncMode memtableSyncMode = SyncMode.SYNC_NONE; + private long memtableSyncIntervalUs = 0; + private long valueSeparationThreshold = 0; + private long vlogSegmentSize = 0; + private int memtableL0QueueStallThreshold = 0; + private int memtableIdleFlushSeconds = 0; + private long txnTimeoutSeconds = 0; /** * Creates a new builder with default values. */ public Builder() { } - private int numFlushThreads = 2; - private int numCompactionThreads = 2; - private LogLevel logLevel = LogLevel.INFO; - private long blockCacheSize = 64 * 1024 * 1024; - private long maxOpenSSTables = 256; - private boolean logToFile = false; - private long logTruncationAt = 24 * 1024 * 1024; - private long maxMemoryUsage = 0; - private boolean unifiedMemtable = false; - private long unifiedMemtableWriteBufferSize = 0; - private int unifiedMemtableSkipListMaxLevel = 0; - private float unifiedMemtableSkipListProbability = 0; - private int unifiedMemtableSyncMode = 0; - private long unifiedMemtableSyncIntervalUs = 0; - private String objectStoreFsPath = null; - private ObjectStoreConfig objectStoreConfig = null; - private S3Config objectStoreS3Config = null; - private int maxConcurrentFlushes = 0; - private boolean finishCompactionsOnClose = false; /** * Sets the database file-system path. @@ -281,31 +436,31 @@ public Builder dbPath(String dbPath) { this.dbPath = dbPath; return this; } - + /** - * Sets the number of flush threads. + * Sets the number of flush worker threads. * - * @param numFlushThreads the thread count; must be positive + * @param numFlushThreads the thread count, or 0 for the engine default * @return this builder */ public Builder numFlushThreads(int numFlushThreads) { this.numFlushThreads = numFlushThreads; return this; } - + /** - * Sets the number of compaction threads. + * Sets the number of compaction worker threads. * - * @param numCompactionThreads the thread count; must be positive + * @param numCompactionThreads the thread count, or 0 for the engine default * @return this builder */ public Builder numCompactionThreads(int numCompactionThreads) { this.numCompactionThreads = numCompactionThreads; return this; } - + /** - * Sets the log level. + * Sets the minimum severity to emit. * * @param logLevel the log level; must not be {@code null} * @return this builder @@ -314,114 +469,167 @@ public Builder logLevel(LogLevel logLevel) { this.logLevel = logLevel; return this; } - + /** - * Sets the block cache size in bytes. + * Sets the size of the database-level block cache for hot SSTable blocks. * - * @param blockCacheSize the size in bytes; must not be negative + * @param blockCacheSize the size in bytes, or 0 for the engine default * @return this builder */ public Builder blockCacheSize(long blockCacheSize) { this.blockCacheSize = blockCacheSize; return this; } - + /** - * Sets the maximum number of open SSTables. + * Sets the maximum number of concurrently open SSTable file handles. * - * @param maxOpenSSTables the maximum count; must be positive + * @param maxOpenSSTables the maximum count, or 0 for the engine default * @return this builder */ public Builder maxOpenSSTables(long maxOpenSSTables) { this.maxOpenSSTables = maxOpenSSTables; return this; } - + + /** + * Sets whether the log is written to a file named {@code LOG} inside the + * database directory rather than to stderr. + * + * @param logToFile {@code true} to log to a file + * @return this builder + */ public Builder logToFile(boolean logToFile) { this.logToFile = logToFile; return this; } - + + /** + * Sets the size past which the log file is truncated and reopened. + * + * @param logTruncationAt the threshold in bytes, or 0 for never + * @return this builder + */ public Builder logTruncationAt(long logTruncationAt) { this.logTruncationAt = logTruncationAt; return this; } - - public Builder maxMemoryUsage(long maxMemoryUsage) { - this.maxMemoryUsage = maxMemoryUsage; - return this; - } - - public Builder unifiedMemtable(boolean unifiedMemtable) { - this.unifiedMemtable = unifiedMemtable; - return this; - } - public Builder unifiedMemtableWriteBufferSize(long unifiedMemtableWriteBufferSize) { - this.unifiedMemtableWriteBufferSize = unifiedMemtableWriteBufferSize; + /** + * Sets the memory the active memtable may occupy before it is rotated. + * + * @param memtableWriteBufferSize the size in bytes, or 0 for the engine default + * @return this builder + */ + public Builder memtableWriteBufferSize(long memtableWriteBufferSize) { + this.memtableWriteBufferSize = memtableWriteBufferSize; return this; } - public Builder unifiedMemtableSkipListMaxLevel(int unifiedMemtableSkipListMaxLevel) { - this.unifiedMemtableSkipListMaxLevel = unifiedMemtableSkipListMaxLevel; + /** + * Sets the skip list max level for the memtable. + * + * @param memtableSkipListMaxLevel the max level, or 0 for the engine default + * @return this builder + */ + public Builder memtableSkipListMaxLevel(int memtableSkipListMaxLevel) { + this.memtableSkipListMaxLevel = memtableSkipListMaxLevel; return this; } - public Builder unifiedMemtableSkipListProbability(float unifiedMemtableSkipListProbability) { - this.unifiedMemtableSkipListProbability = unifiedMemtableSkipListProbability; + /** + * Sets the skip list level probability for the memtable. + * + * @param memtableSkipListProbability the probability in [0.0, 1.0], or 0 + * for the engine default + * @return this builder + */ + public Builder memtableSkipListProbability(float memtableSkipListProbability) { + this.memtableSkipListProbability = memtableSkipListProbability; return this; } - public Builder unifiedMemtableSyncMode(int unifiedMemtableSyncMode) { - this.unifiedMemtableSyncMode = unifiedMemtableSyncMode; + /** + * Sets the durability mode for the write-ahead log. + * + * @param memtableSyncMode the sync mode; must not be {@code null} + * @return this builder + */ + public Builder memtableSyncMode(SyncMode memtableSyncMode) { + this.memtableSyncMode = memtableSyncMode; return this; } - public Builder unifiedMemtableSyncIntervalUs(long unifiedMemtableSyncIntervalUs) { - this.unifiedMemtableSyncIntervalUs = unifiedMemtableSyncIntervalUs; + /** + * Sets the fsync interval for {@link SyncMode#SYNC_INTERVAL}. + * + * @param memtableSyncIntervalUs the interval in microseconds, or 0 for a + * one second default + * @return this builder + */ + public Builder memtableSyncIntervalUs(long memtableSyncIntervalUs) { + this.memtableSyncIntervalUs = memtableSyncIntervalUs; return this; } - public Builder objectStoreFsPath(String objectStoreFsPath) { - this.objectStoreFsPath = objectStoreFsPath; + /** + * Sets the size at or above which values are stored in the shared value + * log and referenced from the key log. + * + * @param valueSeparationThreshold the threshold in bytes, or 0 for the + * engine default + * @return this builder + */ + public Builder valueSeparationThreshold(long valueSeparationThreshold) { + this.valueSeparationThreshold = valueSeparationThreshold; return this; } - public Builder objectStoreConfig(ObjectStoreConfig objectStoreConfig) { - this.objectStoreConfig = objectStoreConfig; + /** + * Sets the size at which the value log seals its active segment. + * + * @param vlogSegmentSize the size in bytes, or 0 for the engine default + * @return this builder + */ + public Builder vlogSegmentSize(long vlogSegmentSize) { + this.vlogSegmentSize = vlogSegmentSize; return this; } /** - * Backs the database with an S3-compatible object store connector (AWS S3, MinIO, etc.). - * Takes precedence over {@link #objectStoreFsPath(String)} when both are set. Pair with - * {@link #objectStoreConfig(ObjectStoreConfig)} to tune cache, multipart, and replication - * behavior. Requires the native library to be built with {@code TIDESDB_WITH_S3=ON}. + * Sets the immutable-queue depth at which writes stall for backpressure. * - * @param objectStoreS3Config the S3 connector configuration, or null for none + * @param memtableL0QueueStallThreshold the depth, or 0 to never stall * @return this builder */ - public Builder objectStoreS3Config(S3Config objectStoreS3Config) { - this.objectStoreS3Config = objectStoreS3Config; + public Builder memtableL0QueueStallThreshold(int memtableL0QueueStallThreshold) { + this.memtableL0QueueStallThreshold = memtableL0QueueStallThreshold; return this; } - public Builder maxConcurrentFlushes(int maxConcurrentFlushes) { - this.maxConcurrentFlushes = maxConcurrentFlushes; + /** + * Sets how long the active memtable may sit unwritten before the engine + * rotates it on its own. + * + * @param memtableIdleFlushSeconds the interval in seconds, or 0 to never + * rotate on idle + * @return this builder + */ + public Builder memtableIdleFlushSeconds(int memtableIdleFlushSeconds) { + this.memtableIdleFlushSeconds = memtableIdleFlushSeconds; return this; } /** - * Sets the close behavior for in-flight compactions. + * Sets how long a transaction may stay active before the next operation + * on it expires it. A single transaction can override this with + * {@link Transaction#setTimeout(long)}. * - * @param finishCompactionsOnClose false (default) cancels in-flight compactions at their - * next checkpoint for a fast shutdown (no data is lost; recovery handles a mid-merge - * state). true lets in-flight compactions run to completion before {@code close()} - * returns. + * @param txnTimeoutSeconds the timeout in seconds, or 0 for no timeout * @return this builder */ - public Builder finishCompactionsOnClose(boolean finishCompactionsOnClose) { - this.finishCompactionsOnClose = finishCompactionsOnClose; + public Builder txnTimeoutSeconds(long txnTimeoutSeconds) { + this.txnTimeoutSeconds = txnTimeoutSeconds; return this; } @@ -436,52 +644,76 @@ public Config build() { return new Config(this); } - private void validate() throws IllegalArgumentException { + private void validate() { if (dbPath == null) { throw new IllegalArgumentException("Database path cannot be null"); } - if (dbPath.isEmpty()) { - dbPath = ""; - } - if (numFlushThreads <= 0) { - throw new IllegalArgumentException("Number of flush threads must be positive"); + if (numFlushThreads < 0) { + throw new IllegalArgumentException( + "numFlushThreads must not be negative, was: " + numFlushThreads); } - if (numCompactionThreads <= 0) { - throw new IllegalArgumentException("Number of compaction threads must be positive"); + if (numCompactionThreads < 0) { + throw new IllegalArgumentException( + "numCompactionThreads must not be negative, was: " + numCompactionThreads); } if (logLevel == null) { throw new IllegalArgumentException("Log level cannot be null"); } if (blockCacheSize < 0) { - throw new IllegalArgumentException("Block cache size cannot be negative"); + throw new IllegalArgumentException( + "blockCacheSize must not be negative, was: " + blockCacheSize); } - if (maxOpenSSTables <= 0) { - throw new IllegalArgumentException("Max open SSTables must be positive"); + if (maxOpenSSTables < 0) { + throw new IllegalArgumentException( + "maxOpenSSTables must not be negative, was: " + maxOpenSSTables); } if (logTruncationAt < 0) { - throw new IllegalArgumentException("logTruncationAt must not be negative, was: " + logTruncationAt); + throw new IllegalArgumentException( + "logTruncationAt must not be negative, was: " + logTruncationAt); + } + if (memtableWriteBufferSize < 0) { + throw new IllegalArgumentException( + "memtableWriteBufferSize must not be negative, was: " + memtableWriteBufferSize); + } + if (memtableSkipListMaxLevel < 0) { + throw new IllegalArgumentException( + "memtableSkipListMaxLevel must not be negative, was: " + memtableSkipListMaxLevel); + } + if (Float.isNaN(memtableSkipListProbability) + || Float.isInfinite(memtableSkipListProbability) + || memtableSkipListProbability < 0.0f + || memtableSkipListProbability > 1.0f) { + throw new IllegalArgumentException( + "memtableSkipListProbability must be finite and in [0.0, 1.0], was: " + + memtableSkipListProbability); } - if (maxMemoryUsage < 0) { - throw new IllegalArgumentException("maxMemoryUsage must not be negative, was: " + maxMemoryUsage); + if (memtableSyncMode == null) { + throw new IllegalArgumentException("memtableSyncMode cannot be null"); } - if (unifiedMemtableWriteBufferSize < 0) { - throw new IllegalArgumentException("unifiedMemtableWriteBufferSize must not be negative, was: " + unifiedMemtableWriteBufferSize); + if (memtableSyncIntervalUs < 0) { + throw new IllegalArgumentException( + "memtableSyncIntervalUs must not be negative, was: " + memtableSyncIntervalUs); } - if (unifiedMemtableSyncIntervalUs < 0) { - throw new IllegalArgumentException("unifiedMemtableSyncIntervalUs must not be negative, was: " + unifiedMemtableSyncIntervalUs); + if (valueSeparationThreshold < 0) { + throw new IllegalArgumentException( + "valueSeparationThreshold must not be negative, was: " + valueSeparationThreshold); } - if (unifiedMemtableSkipListMaxLevel < 0) { - throw new IllegalArgumentException("unifiedMemtableSkipListMaxLevel must not be negative, was: " + unifiedMemtableSkipListMaxLevel); + if (vlogSegmentSize < 0) { + throw new IllegalArgumentException( + "vlogSegmentSize must not be negative, was: " + vlogSegmentSize); } - if (Float.isNaN(unifiedMemtableSkipListProbability) || Float.isInfinite(unifiedMemtableSkipListProbability) - || unifiedMemtableSkipListProbability < 0.0f || unifiedMemtableSkipListProbability > 1.0f) { - throw new IllegalArgumentException("unifiedMemtableSkipListProbability must be finite and in [0.0, 1.0], was: " + unifiedMemtableSkipListProbability); + if (memtableL0QueueStallThreshold < 0) { + throw new IllegalArgumentException( + "memtableL0QueueStallThreshold must not be negative, was: " + + memtableL0QueueStallThreshold); } - if (unifiedMemtableSyncMode < 0) { - throw new IllegalArgumentException("unifiedMemtableSyncMode must not be negative, was: " + unifiedMemtableSyncMode); + if (memtableIdleFlushSeconds < 0) { + throw new IllegalArgumentException( + "memtableIdleFlushSeconds must not be negative, was: " + memtableIdleFlushSeconds); } - if (maxConcurrentFlushes < 0) { - throw new IllegalArgumentException("maxConcurrentFlushes must not be negative, was: " + maxConcurrentFlushes); + if (txnTimeoutSeconds < 0) { + throw new IllegalArgumentException( + "txnTimeoutSeconds must not be negative, was: " + txnTimeoutSeconds); } } } diff --git a/src/main/java/com/tidesdb/DbStats.java b/src/main/java/com/tidesdb/DbStats.java index 98a280d..ae1823c 100644 --- a/src/main/java/com/tidesdb/DbStats.java +++ b/src/main/java/com/tidesdb/DbStats.java @@ -19,378 +19,553 @@ package com.tidesdb; /** - * Database-level aggregate statistics across the entire TidesDB instance. + * Database-level statistics returned by {@link TidesDB#getDbStats()}. The + * memtable, the value log, and the MVCC clock are database-level and shared, so + * their figures live here rather than on {@link CfStats}. */ public class DbStats { private final int numColumnFamilies; - private final long totalMemory; - private final long availableMemory; - private final long resolvedMemoryLimit; - private final int memoryPressureLevel; - private final int flushPendingCount; - private final long totalMemtableBytes; - private final int totalImmutableCount; + private final int immutableMemtableCount; + private final int compactionPendingCount; private final int totalSstableCount; private final long totalDataSizeBytes; private final int numOpenSstables; private final long globalSeq; + private final long minSnapshotSeq; + private final int activeTxnCount; private final long txnMemoryBytes; - private final long compactionQueueSize; - private final long flushQueueSize; - private final boolean unifiedMemtableEnabled; - private final long unifiedMemtableBytes; - private final int unifiedImmutableCount; - private final boolean unifiedIsFlushing; - private final int unifiedNextCfIndex; - private final long unifiedWalGeneration; - private final boolean objectStoreEnabled; - private final String objectStoreConnector; - private final long localCacheBytesUsed; - private final long localCacheBytesMax; - private final int localCacheNumFiles; - private final long lastUploadedGeneration; - private final long uploadQueueDepth; - private final long totalUploads; - private final long totalUploadFailures; - private final boolean replicaMode; - private final long primaryEpoch; - private final long seenEpoch; - private final long uwalBytesWritten; - private final long walBytesWritten; + private final long memtableBytes; + private final boolean flushing; + private final long nextCfIndex; + private final long walGeneration; + private final long flushCount; + private final long compactionCount; private final long flushBytesWritten; private final long compactionBytesWritten; private final long compactionBytesRead; + private final long walBytesWritten; private final long userBytesWritten; - private final long flushCount; - private final long compactionCount; + private final long vlogFileSize; + private final long vlogValueCount; + private final long vlogUsedBytes; + private final long vlogStoredBytes; + private final long vlogLiveBytes; + private final long vlogSegmentCount; + private final long vlogBytesWritten; + private final long vlogDeadBytes; + private final long vlogReclaimCalls; + private final long vlogReclaimPasses; + private final long vlogSegmentsRetired; + private final long vlogSegmentsDrainable; + private final long writesThrottled; + private final long writesBlocked; + private final long writeStallUs; + private final long writeStallCeilingHits; - public DbStats(int numColumnFamilies, long totalMemory, long availableMemory, - long resolvedMemoryLimit, int memoryPressureLevel, int flushPendingCount, - long totalMemtableBytes, int totalImmutableCount, int totalSstableCount, - long totalDataSizeBytes, int numOpenSstables, long globalSeq, - long txnMemoryBytes, long compactionQueueSize, long flushQueueSize, - boolean unifiedMemtableEnabled, long unifiedMemtableBytes, - int unifiedImmutableCount, boolean unifiedIsFlushing, - int unifiedNextCfIndex, long unifiedWalGeneration, - boolean objectStoreEnabled, String objectStoreConnector, - long localCacheBytesUsed, long localCacheBytesMax, int localCacheNumFiles, - long lastUploadedGeneration, long uploadQueueDepth, - long totalUploads, long totalUploadFailures, boolean replicaMode, - long primaryEpoch, long seenEpoch, - long uwalBytesWritten, long walBytesWritten, long flushBytesWritten, - long compactionBytesWritten, long compactionBytesRead, long userBytesWritten, - long flushCount, long compactionCount) { + /** + * Creates a new {@code DbStats}. Typically called by the JNI bridge rather + * than application code. + * + * @param numColumnFamilies number of column families + * @param immutableMemtableCount immutable memtables awaiting flush + * @param compactionPendingCount compaction jobs queued for the worker pool + * @param totalSstableCount total SSTables across every family and level + * @param totalDataSizeBytes on-disk key-log bytes summed across every family + * @param numOpenSstables currently open SSTable file handles + * @param globalSeq current database-global sequence number + * @param minSnapshotSeq the oldest live-transaction snapshot + * @param activeTxnCount live transactions joined to the registry + * @param txnMemoryBytes bytes held by in-flight transactions + * @param memtableBytes memory the active memtable occupies + * @param flushing whether an immutable is queued or flushing + * @param nextCfIndex next column family id to be assigned + * @param walGeneration current write-ahead-log generation counter + * @param flushCount SSTables flushed across every family + * @param compactionCount compactions run across every family + * @param flushBytesWritten flush output bytes summed across every family + * @param compactionBytesWritten compaction output bytes summed across every family + * @param compactionBytesRead compaction input bytes summed across every family + * @param walBytesWritten framed write-ahead-log bytes, counted at the append + * @param userBytesWritten logical committed bytes summed across every family + * @param vlogFileSize total value-log file size in bytes + * @param vlogValueCount values currently indexed in the value log + * @param vlogUsedBytes uncompressed length the indexed values represent + * @param vlogStoredBytes the on-disk length those same values occupy + * @param vlogLiveBytes value-log bytes the live SSTables still reference + * @param vlogSegmentCount value-log segment files currently open + * @param vlogBytesWritten value-log bytes ever appended + * @param vlogDeadBytes value-log bytes beyond what the live values account for + * @param vlogReclaimCalls value-log reclaims attempted, lifetime + * @param vlogReclaimPasses value-log reclaim passes that drained a segment + * @param vlogSegmentsRetired value-log segment files a reclaim has unlinked + * @param vlogSegmentsDrainable sealed segments worth rewriting tables for + * @param writesThrottled commits the L0 admission policy made dwell + * @param writesBlocked commits the L0 admission policy made wait + * @param writeStallUs total microseconds commits were held in L0 admission + * @param writeStallCeilingHits commits admitted only because the wait ceiling expired + */ + public DbStats(int numColumnFamilies, int immutableMemtableCount, int compactionPendingCount, + int totalSstableCount, long totalDataSizeBytes, int numOpenSstables, + long globalSeq, long minSnapshotSeq, int activeTxnCount, long txnMemoryBytes, + long memtableBytes, boolean flushing, long nextCfIndex, long walGeneration, + long flushCount, long compactionCount, long flushBytesWritten, + long compactionBytesWritten, long compactionBytesRead, long walBytesWritten, + long userBytesWritten, long vlogFileSize, long vlogValueCount, + long vlogUsedBytes, long vlogStoredBytes, long vlogLiveBytes, + long vlogSegmentCount, long vlogBytesWritten, long vlogDeadBytes, + long vlogReclaimCalls, long vlogReclaimPasses, long vlogSegmentsRetired, + long vlogSegmentsDrainable, long writesThrottled, long writesBlocked, + long writeStallUs, long writeStallCeilingHits) { this.numColumnFamilies = numColumnFamilies; - this.totalMemory = totalMemory; - this.availableMemory = availableMemory; - this.resolvedMemoryLimit = resolvedMemoryLimit; - this.memoryPressureLevel = memoryPressureLevel; - this.flushPendingCount = flushPendingCount; - this.totalMemtableBytes = totalMemtableBytes; - this.totalImmutableCount = totalImmutableCount; + this.immutableMemtableCount = immutableMemtableCount; + this.compactionPendingCount = compactionPendingCount; this.totalSstableCount = totalSstableCount; this.totalDataSizeBytes = totalDataSizeBytes; this.numOpenSstables = numOpenSstables; this.globalSeq = globalSeq; + this.minSnapshotSeq = minSnapshotSeq; + this.activeTxnCount = activeTxnCount; this.txnMemoryBytes = txnMemoryBytes; - this.compactionQueueSize = compactionQueueSize; - this.flushQueueSize = flushQueueSize; - this.unifiedMemtableEnabled = unifiedMemtableEnabled; - this.unifiedMemtableBytes = unifiedMemtableBytes; - this.unifiedImmutableCount = unifiedImmutableCount; - this.unifiedIsFlushing = unifiedIsFlushing; - this.unifiedNextCfIndex = unifiedNextCfIndex; - this.unifiedWalGeneration = unifiedWalGeneration; - this.objectStoreEnabled = objectStoreEnabled; - this.objectStoreConnector = objectStoreConnector; - this.localCacheBytesUsed = localCacheBytesUsed; - this.localCacheBytesMax = localCacheBytesMax; - this.localCacheNumFiles = localCacheNumFiles; - this.lastUploadedGeneration = lastUploadedGeneration; - this.uploadQueueDepth = uploadQueueDepth; - this.totalUploads = totalUploads; - this.totalUploadFailures = totalUploadFailures; - this.replicaMode = replicaMode; - this.primaryEpoch = primaryEpoch; - this.seenEpoch = seenEpoch; - this.uwalBytesWritten = uwalBytesWritten; - this.walBytesWritten = walBytesWritten; + this.memtableBytes = memtableBytes; + this.flushing = flushing; + this.nextCfIndex = nextCfIndex; + this.walGeneration = walGeneration; + this.flushCount = flushCount; + this.compactionCount = compactionCount; this.flushBytesWritten = flushBytesWritten; this.compactionBytesWritten = compactionBytesWritten; this.compactionBytesRead = compactionBytesRead; + this.walBytesWritten = walBytesWritten; this.userBytesWritten = userBytesWritten; - this.flushCount = flushCount; - this.compactionCount = compactionCount; + this.vlogFileSize = vlogFileSize; + this.vlogValueCount = vlogValueCount; + this.vlogUsedBytes = vlogUsedBytes; + this.vlogStoredBytes = vlogStoredBytes; + this.vlogLiveBytes = vlogLiveBytes; + this.vlogSegmentCount = vlogSegmentCount; + this.vlogBytesWritten = vlogBytesWritten; + this.vlogDeadBytes = vlogDeadBytes; + this.vlogReclaimCalls = vlogReclaimCalls; + this.vlogReclaimPasses = vlogReclaimPasses; + this.vlogSegmentsRetired = vlogSegmentsRetired; + this.vlogSegmentsDrainable = vlogSegmentsDrainable; + this.writesThrottled = writesThrottled; + this.writesBlocked = writesBlocked; + this.writeStallUs = writeStallUs; + this.writeStallCeilingHits = writeStallCeilingHits; } + /** + * Returns the number of column families. + * + * @return the column family count + */ public int getNumColumnFamilies() { return numColumnFamilies; } - public long getTotalMemory() { - return totalMemory; - } - - public long getAvailableMemory() { - return availableMemory; - } - - public long getResolvedMemoryLimit() { - return resolvedMemoryLimit; - } - - public int getMemoryPressureLevel() { - return memoryPressureLevel; - } - - public int getFlushPendingCount() { - return flushPendingCount; - } - - public long getTotalMemtableBytes() { - return totalMemtableBytes; + /** + * Returns the number of immutable memtables awaiting flush, which is the L0 + * queue depth. + * + * @return the immutable memtable count + */ + public int getImmutableMemtableCount() { + return immutableMemtableCount; } - public int getTotalImmutableCount() { - return totalImmutableCount; + /** + * Returns the number of compaction jobs queued for the worker pool. + * + * @return the pending compaction count + */ + public int getCompactionPendingCount() { + return compactionPendingCount; } + /** + * Returns the total SSTables across every column family and level. + * + * @return the SSTable count + */ public int getTotalSstableCount() { return totalSstableCount; } + /** + * Returns the on-disk key-log bytes summed across every column family and + * level. The value log is reported separately by {@link #getVlogFileSize()}, + * since it is shared rather than owned by any one family. + * + * @return the on-disk data size in bytes + */ public long getTotalDataSizeBytes() { return totalDataSizeBytes; } + /** + * Returns the number of currently open SSTable file handles. + * + * @return the open SSTable count + */ public int getNumOpenSstables() { return numOpenSstables; } + /** + * Returns the current database-global sequence number, the MVCC clock. + * + * @return the global sequence + */ public long getGlobalSeq() { return globalSeq; } - public long getTxnMemoryBytes() { - return txnMemoryBytes; + /** + * Returns the oldest live-transaction snapshot, which is the compaction + * garbage-collection floor. + * + *

The engine's sequence is an unsigned 64-bit value, so when no snapshot + * is registered the floor sits at {@code UINT64_MAX} and arrives here as + * {@code -1}. Compare sequences with + * {@link Long#compareUnsigned(long, long)} rather than {@code <}. + * + * @return the minimum snapshot sequence, or {@code -1} when nothing holds + * the floor + */ + public long getMinSnapshotSeq() { + return minSnapshotSeq; } - public long getCompactionQueueSize() { - return compactionQueueSize; + /** + * Returns the live transactions joined to the registry, which is + * repeatable-read and stronger only. Read-uncommitted and read-committed + * transactions need no snapshot reservation and so are not counted. + * + * @return the active transaction count + */ + public int getActiveTxnCount() { + return activeTxnCount; } - public long getFlushQueueSize() { - return flushQueueSize; + /** + * Returns the bytes held by in-flight transactions, over the same registered + * set as {@link #getActiveTxnCount()}. + * + * @return the transaction memory in bytes + */ + public long getTxnMemoryBytes() { + return txnMemoryBytes; } - public boolean isUnifiedMemtableEnabled() { - return unifiedMemtableEnabled; + /** + * Returns the memory the active memtable occupies, the figure + * {@link Config#getMemtableWriteBufferSize()} is compared against, so it + * counts skip list nodes and version structs as well as key and value bytes. + * + * @return the memtable size in bytes + */ + public long getMemtableBytes() { + return memtableBytes; } - public long getUnifiedMemtableBytes() { - return unifiedMemtableBytes; + /** + * Returns whether an immutable memtable is queued or flushing. + * + * @return {@code true} while flushing + */ + public boolean isFlushing() { + return flushing; } - public int getUnifiedImmutableCount() { - return unifiedImmutableCount; + /** + * Returns the next column family id to be assigned. + * + * @return the next column family index + */ + public long getNextCfIndex() { + return nextCfIndex; } - public boolean isUnifiedIsFlushing() { - return unifiedIsFlushing; + /** + * Returns the current write-ahead-log generation counter. + * + * @return the WAL generation + */ + public long getWalGeneration() { + return walGeneration; } - public int getUnifiedNextCfIndex() { - return unifiedNextCfIndex; + /** + * Returns the SSTables flushed across every column family. + * + * @return the flush count + */ + public long getFlushCount() { + return flushCount; } - public long getUnifiedWalGeneration() { - return unifiedWalGeneration; + /** + * Returns the compactions run across every column family. + * + * @return the compaction count + */ + public long getCompactionCount() { + return compactionCount; } - public boolean isObjectStoreEnabled() { - return objectStoreEnabled; + /** + * Returns the flush output bytes summed across every column family. + * + * @return the flush byte count + */ + public long getFlushBytesWritten() { + return flushBytesWritten; } - public String getObjectStoreConnector() { - return objectStoreConnector; + /** + * Returns the compaction output bytes summed across every column family. + * + * @return the compaction output byte count + */ + public long getCompactionBytesWritten() { + return compactionBytesWritten; } - public long getLocalCacheBytesUsed() { - return localCacheBytesUsed; + /** + * Returns the compaction input bytes summed across every column family. + * + * @return the compaction input byte count + */ + public long getCompactionBytesRead() { + return compactionBytesRead; } - public long getLocalCacheBytesMax() { - return localCacheBytesMax; + /** + * Returns the framed write-ahead-log bytes, counted at the append. The log + * is reclaimed with its memtable and never appears in the on-disk totals, + * but the device was still asked to write it, and a write-amplification + * figure that omits it understates by a whole copy of the data. + * + * @return the WAL byte count + */ + public long getWalBytesWritten() { + return walBytesWritten; } - public int getLocalCacheNumFiles() { - return localCacheNumFiles; + /** + * Returns the logical committed bytes summed across every column family. + * + * @return the user byte count + */ + public long getUserBytesWritten() { + return userBytesWritten; } - public long getLastUploadedGeneration() { - return lastUploadedGeneration; + /** + * Returns the total value-log file size in bytes. + * + * @return the value-log size + */ + public long getVlogFileSize() { + return vlogFileSize; } - public long getUploadQueueDepth() { - return uploadQueueDepth; + /** + * Returns the values currently indexed in the value log. + * + * @return the indexed value count + */ + public long getVlogValueCount() { + return vlogValueCount; } - public long getTotalUploads() { - return totalUploads; + /** + * Returns the uncompressed length the indexed values represent. This counts + * everything the index still names, reachable or not, so it is not a measure + * of live data. + * + * @return the used byte count + */ + public long getVlogUsedBytes() { + return vlogUsedBytes; } - public long getTotalUploadFailures() { - return totalUploadFailures; + /** + * Returns the on-disk length those same indexed values occupy. Read against + * {@link #getVlogUsedBytes()} it is the encoding pipeline's realised ratio. + * + * @return the stored byte count + */ + public long getVlogStoredBytes() { + return vlogStoredBytes; } - public boolean isReplicaMode() { - return replicaMode; + /** + * Returns the value-log bytes the live SSTables still reference. This is the + * figure space amplification is against: a store can hold many gigabytes of + * values no tree can reach, and only this tells them apart from the ones + * still worth keeping. + * + * @return the live byte count + */ + public long getVlogLiveBytes() { + return vlogLiveBytes; } /** - * Gets the lease epoch this primary currently holds (object-store single-writer fencing). - * Returns 0 when this node is not a primary or holds no lease. A promotion that takes - * effect bumps this value. + * Returns the value-log segment files currently open, the one taking appends + * included. * - * @return the current primary lease epoch, or 0 if not a primary + * @return the segment count */ - public long getPrimaryEpoch() { - return primaryEpoch; + public long getVlogSegmentCount() { + return vlogSegmentCount; } /** - * Gets the highest lease epoch this node has observed (object-store single-writer fencing). - * A fenced primary sees {@link #isReplicaMode()} flip back to true once a newer epoch is seen. + * Returns the value-log bytes ever appended, reclamation's own rewrites + * included. On a store that separates its values most of the writing happens + * here. * - * @return the highest observed lease epoch + * @return the appended byte count */ - public long getSeenEpoch() { - return seenEpoch; + public long getVlogBytesWritten() { + return vlogBytesWritten; } /** - * Gets the framed bytes appended to the shared unified WAL (lifetime since open). - * Returns 0 when unified memtable mode is off. + * Returns the value-log bytes beyond what the live values account for, the + * space a reclaim could recover. * - * @return unified WAL bytes written + * @return the dead byte count */ - public long getUwalBytesWritten() { - return uwalBytesWritten; + public long getVlogDeadBytes() { + return vlogDeadBytes; } /** - * Gets the per-column-family WAL bytes summed across all column families - * (lifetime since open). + * Returns the value-log reclaims attempted, lifetime. Read beside + * {@link #getVlogReclaimPasses()} this separates a reclaim that never runs + * from one that runs and finds nothing worth draining. * - * @return WAL bytes written across all CFs + * @return the reclaim call count */ - public long getWalBytesWritten() { - return walBytesWritten; + public long getVlogReclaimCalls() { + return vlogReclaimCalls; } /** - * Gets the flush output bytes summed across all column families (lifetime since open). + * Returns the value-log reclaim passes that drained a segment, since this + * handle opened. * - * @return flush output bytes written across all CFs + * @return the reclaim pass count */ - public long getFlushBytesWritten() { - return flushBytesWritten; + public long getVlogReclaimPasses() { + return vlogReclaimPasses; } /** - * Gets the compaction output bytes summed across all column families (lifetime since open). + * Returns the value-log segment files a reclaim has unlinked. * - * @return compaction output bytes written across all CFs + * @return the retired segment count */ - public long getCompactionBytesWritten() { - return compactionBytesWritten; + public long getVlogSegmentsRetired() { + return vlogSegmentsRetired; } /** - * Gets the compaction input bytes summed across all column families (lifetime since open). + * Returns the sealed segments holding so little live data that rewriting the + * tables referencing them would free most of a file. Read against + * {@link #getVlogDeadBytes()} this says how much of the garbage is currently + * actionable, and a figure that stays high is reclamation falling behind. * - * @return compaction input bytes read across all CFs + * @return the drainable segment count */ - public long getCompactionBytesRead() { - return compactionBytesRead; + public long getVlogSegmentsDrainable() { + return vlogSegmentsDrainable; } /** - * Gets the logical committed bytes summed across all column families (lifetime since open). - * This is the database-wide write-amplification denominator: - * {@code (uwal + wal + flush + compaction) / userBytesWritten}. + * Returns the commits the L0 admission policy made dwell before admitting. * - * @return user bytes written across all CFs + * @return the throttled write count */ - public long getUserBytesWritten() { - return userBytesWritten; + public long getWritesThrottled() { + return writesThrottled; } /** - * Gets the number of flushed SSTables summed across all column families - * (lifetime since open). + * Returns the commits the L0 admission policy made wait for the flush queue + * to drain. * - * @return flush count across all CFs + * @return the blocked write count */ - public long getFlushCount() { - return flushCount; + public long getWritesBlocked() { + return writesBlocked; } /** - * Gets the number of compaction output SSTables summed across all column families - * (lifetime since open). + * Returns the total microseconds commits were held in L0 admission, dwell + * plus wait. * - * @return compaction count across all CFs + * @return the write stall time in microseconds */ - public long getCompactionCount() { - return compactionCount; + public long getWriteStallUs() { + return writeStallUs; + } + + /** + * Returns the commits admitted only because the admission wait ceiling + * expired. Any of these means flush did not keep up with ingestion. + * + * @return the ceiling hit count + */ + public long getWriteStallCeilingHits() { + return writeStallCeilingHits; } @Override public String toString() { return "DbStats{" + "numColumnFamilies=" + numColumnFamilies + - ", totalMemory=" + totalMemory + - ", availableMemory=" + availableMemory + - ", resolvedMemoryLimit=" + resolvedMemoryLimit + - ", memoryPressureLevel=" + memoryPressureLevel + - ", flushPendingCount=" + flushPendingCount + - ", totalMemtableBytes=" + totalMemtableBytes + - ", totalImmutableCount=" + totalImmutableCount + + ", immutableMemtableCount=" + immutableMemtableCount + + ", compactionPendingCount=" + compactionPendingCount + ", totalSstableCount=" + totalSstableCount + ", totalDataSizeBytes=" + totalDataSizeBytes + ", numOpenSstables=" + numOpenSstables + ", globalSeq=" + globalSeq + + ", minSnapshotSeq=" + minSnapshotSeq + + ", activeTxnCount=" + activeTxnCount + ", txnMemoryBytes=" + txnMemoryBytes + - ", compactionQueueSize=" + compactionQueueSize + - ", flushQueueSize=" + flushQueueSize + - ", unifiedMemtableEnabled=" + unifiedMemtableEnabled + - ", unifiedMemtableBytes=" + unifiedMemtableBytes + - ", unifiedImmutableCount=" + unifiedImmutableCount + - ", unifiedIsFlushing=" + unifiedIsFlushing + - ", unifiedNextCfIndex=" + unifiedNextCfIndex + - ", unifiedWalGeneration=" + unifiedWalGeneration + - ", objectStoreEnabled=" + objectStoreEnabled + - ", objectStoreConnector='" + objectStoreConnector + '\'' + - ", localCacheBytesUsed=" + localCacheBytesUsed + - ", localCacheBytesMax=" + localCacheBytesMax + - ", localCacheNumFiles=" + localCacheNumFiles + - ", lastUploadedGeneration=" + lastUploadedGeneration + - ", uploadQueueDepth=" + uploadQueueDepth + - ", totalUploads=" + totalUploads + - ", totalUploadFailures=" + totalUploadFailures + - ", replicaMode=" + replicaMode + - ", primaryEpoch=" + primaryEpoch + - ", seenEpoch=" + seenEpoch + - ", uwalBytesWritten=" + uwalBytesWritten + - ", walBytesWritten=" + walBytesWritten + + ", memtableBytes=" + memtableBytes + + ", flushing=" + flushing + + ", nextCfIndex=" + nextCfIndex + + ", walGeneration=" + walGeneration + + ", flushCount=" + flushCount + + ", compactionCount=" + compactionCount + ", flushBytesWritten=" + flushBytesWritten + ", compactionBytesWritten=" + compactionBytesWritten + ", compactionBytesRead=" + compactionBytesRead + + ", walBytesWritten=" + walBytesWritten + ", userBytesWritten=" + userBytesWritten + - ", flushCount=" + flushCount + - ", compactionCount=" + compactionCount + + ", vlogFileSize=" + vlogFileSize + + ", vlogValueCount=" + vlogValueCount + + ", vlogUsedBytes=" + vlogUsedBytes + + ", vlogStoredBytes=" + vlogStoredBytes + + ", vlogLiveBytes=" + vlogLiveBytes + + ", vlogSegmentCount=" + vlogSegmentCount + + ", vlogBytesWritten=" + vlogBytesWritten + + ", vlogDeadBytes=" + vlogDeadBytes + + ", vlogReclaimCalls=" + vlogReclaimCalls + + ", vlogReclaimPasses=" + vlogReclaimPasses + + ", vlogSegmentsRetired=" + vlogSegmentsRetired + + ", vlogSegmentsDrainable=" + vlogSegmentsDrainable + + ", writesThrottled=" + writesThrottled + + ", writesBlocked=" + writesBlocked + + ", writeStallUs=" + writeStallUs + + ", writeStallCeilingHits=" + writeStallCeilingHits + '}'; } } diff --git a/src/main/java/com/tidesdb/EncodingStats.java b/src/main/java/com/tidesdb/EncodingStats.java new file mode 100644 index 0000000..83e12ce --- /dev/null +++ b/src/main/java/com/tidesdb/EncodingStats.java @@ -0,0 +1,125 @@ +/** + * + * Copyright (C) TidesDB + * + * Original Author: Alex Gaetano Padula + * + * Licensed under the Mozilla Public License, v. 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.mozilla.org/en-US/MPL/2.0/ + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.tidesdb; + +import java.util.Arrays; + +/** + * What one encoding chain achieved on the data it wrote, returned from + * {@link TidesDB#getKlogEncodingStats()} and + * {@link TidesDB#getVlogEncodingStats()}. + * + *

Reported per chain rather than per column family because a family can + * change its codec, and compaction rewrites data under whichever pipeline is + * merging it, so a single figure for a family would average across settings + * that no longer apply and describe none of them. + */ +public class EncodingStats { + + /** + * The most encoding chains reported separately, for either the key logs or + * the value log. A collector never returns more than this many entries. + */ + public static final int MAX_CHAINS = 16; + + private final int[] ids; + private final long logicalBytes; + private final long storedBytes; + private final long itemCount; + + /** + * Creates a new {@code EncodingStats}. Typically called by the JNI bridge + * rather than application code. + * + * @param ids the codec ids in the order applied, empty when the data was + * stored verbatim + * @param logicalBytes what the data amounts to before encoding + * @param storedBytes what it occupies on disk + * @param itemCount values for the value log, SSTables for the key logs + */ + public EncodingStats(int[] ids, long logicalBytes, long storedBytes, long itemCount) { + this.ids = ids == null ? new int[0] : ids.clone(); + this.logicalBytes = logicalBytes; + this.storedBytes = storedBytes; + this.itemCount = itemCount; + } + + /** + * Returns the codec ids in the order applied, empty when the data was + * stored verbatim. Ids in the range of {@link CompressionAlgorithm} name a + * built-in compression codec. + * + * @return a copy of the codec ids + */ + public int[] getIds() { + return ids.clone(); + } + + /** + * Returns what the data amounts to before encoding. + * + * @return the logical byte count + */ + public long getLogicalBytes() { + return logicalBytes; + } + + /** + * Returns what the data occupies on disk. + * + * @return the stored byte count + */ + public long getStoredBytes() { + return storedBytes; + } + + /** + * Returns the number of items this chain wrote: values for the value log, + * SSTables for the key logs. + * + * @return the item count + */ + public long getItemCount() { + return itemCount; + } + + /** + * Returns the realised compression ratio, logical over stored, so a value + * above 1.0 means the data shrank. + * + * @return the ratio, or 0.0 when nothing has been stored + */ + public double getRatio() { + if (storedBytes == 0) { + return 0.0; + } + return (double) logicalBytes / (double) storedBytes; + } + + @Override + public String toString() { + return "EncodingStats{" + + "ids=" + Arrays.toString(ids) + + ", logicalBytes=" + logicalBytes + + ", storedBytes=" + storedBytes + + ", itemCount=" + itemCount + + ", ratio=" + getRatio() + + '}'; + } +} diff --git a/src/main/java/com/tidesdb/IoClass.java b/src/main/java/com/tidesdb/IoClass.java new file mode 100644 index 0000000..ab6f7b6 --- /dev/null +++ b/src/main/java/com/tidesdb/IoClass.java @@ -0,0 +1,96 @@ +/** + * + * Copyright (C) TidesDB + * + * Original Author: Alex Gaetano Padula + * + * Licensed under the Mozilla Public License, v. 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.mozilla.org/en-US/MPL/2.0/ + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.tidesdb; + +/** + * The kinds of file the engine writes, so device time can be attributed rather + * than pooled. + * + *

The ordinal of each constant is its index into {@link IoStats}. + */ +public enum IoClass { + + /** + * Key logs, written by flush and compaction. + */ + SSTABLE(0), + + /** + * The write-ahead log, written by its own single flush thread. + */ + WAL(1), + + /** + * Value log segments, written by a commit that separates a value and by the + * reclaim that copies live values forward. This is the write cost of + * key/value separation, so it is counted apart from the key logs whose size + * that separation is what keeps down. + */ + VLOG(2); + + static { + NativeLibrary.load(); + } + + private final int value; + + IoClass(int value) { + this.value = value; + } + + /** + * Returns the JNI numeric mapping for this I/O class, which is also its + * index into {@link IoStats}. + * + * @return the integer value used by the native library + */ + public int getValue() { + return value; + } + + /** + * Returns the native library's stable short name for this class, suitable + * for a log line or a stats table's row label. + * + * @return the name, never {@code null} + */ + public String getNativeName() { + return nativeName(value); + } + + private static native String nativeName(int cls); + + /** + * Returns the {@link IoClass} constant matching the given JNI integer + * value. + * + * @param value the JNI integer value + * @return the matching constant + * @throws IllegalArgumentException if {@code value} does not map to any + * known constant + */ + public static IoClass fromValue(int value) { + for (IoClass cls : values()) { + if (cls.value == value) { + return cls; + } + } + throw new IllegalArgumentException("Unknown I/O class value: " + value); + } +} diff --git a/src/main/java/com/tidesdb/IoStat.java b/src/main/java/com/tidesdb/IoStat.java new file mode 100644 index 0000000..fe8dcf3 --- /dev/null +++ b/src/main/java/com/tidesdb/IoStat.java @@ -0,0 +1,107 @@ +/** + * + * Copyright (C) TidesDB + * + * Original Author: Alex Gaetano Padula + * + * Licensed under the Mozilla Public License, v. 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.mozilla.org/en-US/MPL/2.0/ + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.tidesdb; + +/** + * What one {@link IoClass} asked of the device. Created by the native library + * and returned inside {@link IoStats}. + */ +public class IoStat { + + private final long ops; + private final long bytes; + private final long totalUs; + private final long maxUs; + + /** + * Creates a new {@code IoStat}. Typically called by the JNI bridge rather + * than application code. + * + * @param ops writes issued + * @param bytes bytes written + * @param totalUs the summed time inside those writes, in microseconds + * @param maxUs the slowest single write, in microseconds + */ + public IoStat(long ops, long bytes, long totalUs, long maxUs) { + this.ops = ops; + this.bytes = bytes; + this.totalUs = totalUs; + this.maxUs = maxUs; + } + + /** + * Returns the number of writes issued. + * + * @return the write count + */ + public long getOps() { + return ops; + } + + /** + * Returns the number of bytes written. + * + * @return the byte count + */ + public long getBytes() { + return bytes; + } + + /** + * Returns the summed time inside those writes. + * + * @return the total time in microseconds + */ + public long getTotalUs() { + return totalUs; + } + + /** + * Returns the slowest single write. + * + * @return the slowest write in microseconds + */ + public long getMaxUs() { + return maxUs; + } + + /** + * Returns the throughput this class actually achieved, in bytes per second. + * Compare it against what the storage can sustain, because a saturated + * device and a stalled engine look identical from the application. + * + * @return bytes per second, or 0.0 when no time has been spent writing + */ + public double getBytesPerSecond() { + if (totalUs == 0) { + return 0.0; + } + return (double) bytes * 1_000_000.0 / (double) totalUs; + } + + @Override + public String toString() { + return "IoStat{" + + "ops=" + ops + + ", bytes=" + bytes + + ", totalUs=" + totalUs + + ", maxUs=" + maxUs + + '}'; + } +} diff --git a/src/main/java/com/tidesdb/IoStats.java b/src/main/java/com/tidesdb/IoStats.java new file mode 100644 index 0000000..ab8c699 --- /dev/null +++ b/src/main/java/com/tidesdb/IoStats.java @@ -0,0 +1,99 @@ +/** + * + * Copyright (C) TidesDB + * + * Original Author: Alex Gaetano Padula + * + * Licensed under the Mozilla Public License, v. 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.mozilla.org/en-US/MPL/2.0/ + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.tidesdb; + +/** + * What each class of file asked of the device, one entry per {@link IoClass}, + * returned from {@link TidesDB#getIoStats()}. + * + *

This is the other half of {@link TidesDB#getStallStats()}: that says + * writers waited on the log, this says whether the device was the reason. + * + *

Only handles the engine opens through its descriptor manager are counted, + * which is every key log and every write-ahead log. The value log and the + * manifest are not, so this measures the two classes that compete for the + * device under load rather than every byte the database writes. + */ +public class IoStats { + + private final IoStat[] classes; + + /** + * Creates a new {@code IoStats}. Typically called by the JNI bridge rather + * than application code. + * + * @param classes the per-class totals, indexed by {@link IoClass#getValue()}; + * must not be {@code null} and must hold one entry per class + */ + public IoStats(IoStat[] classes) { + if (classes == null || classes.length != IoClass.values().length) { + throw new IllegalArgumentException( + "classes must hold exactly " + IoClass.values().length + " entries"); + } + this.classes = classes.clone(); + } + + /** + * Returns the totals for one I/O class. + * + * @param cls the class; must not be {@code null} + * @return the totals for that class, never {@code null} + */ + public IoStat get(IoClass cls) { + if (cls == null) { + throw new IllegalArgumentException("I/O class cannot be null"); + } + return classes[cls.getValue()]; + } + + /** + * Returns the totals for every I/O class, indexed by + * {@link IoClass#getValue()}. + * + * @return a copy of the per-class totals + */ + public IoStat[] getClasses() { + return classes.clone(); + } + + /** + * Returns the bytes written across every class. + * + * @return the total byte count + */ + public long getTotalBytes() { + long total = 0; + for (IoStat stat : classes) { + total += stat.getBytes(); + } + return total; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("IoStats{"); + for (IoClass cls : IoClass.values()) { + if (cls.getValue() > 0) { + sb.append(", "); + } + sb.append(cls.name()).append('=').append(classes[cls.getValue()]); + } + return sb.append('}').toString(); + } +} diff --git a/src/main/java/com/tidesdb/LogLevel.java b/src/main/java/com/tidesdb/LogLevel.java index c25bfee..650836a 100644 --- a/src/main/java/com/tidesdb/LogLevel.java +++ b/src/main/java/com/tidesdb/LogLevel.java @@ -19,47 +19,45 @@ package com.tidesdb; /** - * Logging level for the native TidesDB library. Each constant maps to an - * integer used by the JNI bridge. + * Logging level for the native TidesDB library, used both as a message severity + * and as the sink threshold a message must meet or exceed to be emitted. A larger + * value is more severe, so a higher threshold emits fewer lines. + * + *

Each constant maps to an integer used by the JNI bridge. */ public enum LogLevel { /** - * Verbose debug output. + * No logging. */ - DEBUG(0), + NONE(0), /** - * Informational messages (default). + * Low severity, highly detailed messages for technical debugging. */ - INFO(1), + TRACE(1), /** - * Warning messages. + * Standard information describing engine status or operations. */ - WARN(2), + INFO(2), /** - * Error messages. + * Non-imminent errors that require awareness. */ - ERROR(3), + WARN(3), /** - * Fatal messages. + * An operation failed. */ - FATAL(4), + ERROR(4); - /** - * Logging disabled. - */ - NONE(99); - private final int value; - + LogLevel(int value) { this.value = value; } - + /** * Returns the JNI numeric mapping for this log level. * @@ -68,7 +66,7 @@ public enum LogLevel { public int getValue() { return value; } - + /** * Returns the {@link LogLevel} constant matching the given JNI integer * value. diff --git a/src/main/java/com/tidesdb/ObjectStoreConfig.java b/src/main/java/com/tidesdb/ObjectStoreConfig.java deleted file mode 100644 index 2dff8fd..0000000 --- a/src/main/java/com/tidesdb/ObjectStoreConfig.java +++ /dev/null @@ -1,227 +0,0 @@ -/** - * - * Copyright (C) TidesDB - * - * Original Author: Alex Gaetano Padula - * - * Licensed under the Mozilla Public License, v. 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.mozilla.org/en-US/MPL/2.0/ - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.tidesdb; - -/** - * Configuration for object store mode behavior. - */ -public class ObjectStoreConfig { - - private String localCachePath; - private long localCacheMaxBytes; - private boolean cacheOnRead; - private boolean cacheOnWrite; - private int maxConcurrentUploads; - private int maxConcurrentDownloads; - private long multipartThreshold; - private long multipartPartSize; - private boolean syncManifestToObject; - private boolean replicateWal; - private boolean walUploadSync; - private long walSyncThresholdBytes; - private boolean walSyncOnCommit; - private boolean replicaMode; - private long replicaSyncIntervalUs; - private boolean replicaReplayWal; - - private ObjectStoreConfig(Builder builder) { - this.localCachePath = builder.localCachePath; - this.localCacheMaxBytes = builder.localCacheMaxBytes; - this.cacheOnRead = builder.cacheOnRead; - this.cacheOnWrite = builder.cacheOnWrite; - this.maxConcurrentUploads = builder.maxConcurrentUploads; - this.maxConcurrentDownloads = builder.maxConcurrentDownloads; - this.multipartThreshold = builder.multipartThreshold; - this.multipartPartSize = builder.multipartPartSize; - this.syncManifestToObject = builder.syncManifestToObject; - this.replicateWal = builder.replicateWal; - this.walUploadSync = builder.walUploadSync; - this.walSyncThresholdBytes = builder.walSyncThresholdBytes; - this.walSyncOnCommit = builder.walSyncOnCommit; - this.replicaMode = builder.replicaMode; - this.replicaSyncIntervalUs = builder.replicaSyncIntervalUs; - this.replicaReplayWal = builder.replicaReplayWal; - } - - /** - * Creates a default object store configuration matching tidesdb_objstore_default_config(). - * - * @return a new ObjectStoreConfig with default values - */ - public static ObjectStoreConfig defaultConfig() { - return new Builder().build(); - } - - /** - * Creates a new builder for ObjectStoreConfig. - * - * @return a new Builder - */ - public static Builder builder() { - return new Builder(); - } - - public String getLocalCachePath() { return localCachePath; } - public long getLocalCacheMaxBytes() { return localCacheMaxBytes; } - public boolean isCacheOnRead() { return cacheOnRead; } - public boolean isCacheOnWrite() { return cacheOnWrite; } - public int getMaxConcurrentUploads() { return maxConcurrentUploads; } - public int getMaxConcurrentDownloads() { return maxConcurrentDownloads; } - public long getMultipartThreshold() { return multipartThreshold; } - public long getMultipartPartSize() { return multipartPartSize; } - public boolean isSyncManifestToObject() { return syncManifestToObject; } - public boolean isReplicateWal() { return replicateWal; } - public boolean isWalUploadSync() { return walUploadSync; } - public long getWalSyncThresholdBytes() { return walSyncThresholdBytes; } - public boolean isWalSyncOnCommit() { return walSyncOnCommit; } - public boolean isReplicaMode() { return replicaMode; } - public long getReplicaSyncIntervalUs() { return replicaSyncIntervalUs; } - public boolean isReplicaReplayWal() { return replicaReplayWal; } - - /** - * Builder for ObjectStoreConfig. - */ - public static class Builder { - private String localCachePath = null; - private long localCacheMaxBytes = 0; - private boolean cacheOnRead = true; - private boolean cacheOnWrite = true; - private int maxConcurrentUploads = 4; - private int maxConcurrentDownloads = 8; - private long multipartThreshold = 64 * 1024 * 1024; - private long multipartPartSize = 8 * 1024 * 1024; - private boolean syncManifestToObject = true; - private boolean replicateWal = true; - private boolean walUploadSync = false; - private long walSyncThresholdBytes = 1048576; - private boolean walSyncOnCommit = false; - private boolean replicaMode = false; - private long replicaSyncIntervalUs = 5000000; - private boolean replicaReplayWal = true; - - public Builder localCachePath(String localCachePath) { - this.localCachePath = localCachePath; - return this; - } - - public Builder localCacheMaxBytes(long localCacheMaxBytes) { - this.localCacheMaxBytes = localCacheMaxBytes; - return this; - } - - public Builder cacheOnRead(boolean cacheOnRead) { - this.cacheOnRead = cacheOnRead; - return this; - } - - public Builder cacheOnWrite(boolean cacheOnWrite) { - this.cacheOnWrite = cacheOnWrite; - return this; - } - - public Builder maxConcurrentUploads(int maxConcurrentUploads) { - this.maxConcurrentUploads = maxConcurrentUploads; - return this; - } - - public Builder maxConcurrentDownloads(int maxConcurrentDownloads) { - this.maxConcurrentDownloads = maxConcurrentDownloads; - return this; - } - - public Builder multipartThreshold(long multipartThreshold) { - this.multipartThreshold = multipartThreshold; - return this; - } - - public Builder multipartPartSize(long multipartPartSize) { - this.multipartPartSize = multipartPartSize; - return this; - } - - public Builder syncManifestToObject(boolean syncManifestToObject) { - this.syncManifestToObject = syncManifestToObject; - return this; - } - - public Builder replicateWal(boolean replicateWal) { - this.replicateWal = replicateWal; - return this; - } - - public Builder walUploadSync(boolean walUploadSync) { - this.walUploadSync = walUploadSync; - return this; - } - - public Builder walSyncThresholdBytes(long walSyncThresholdBytes) { - this.walSyncThresholdBytes = walSyncThresholdBytes; - return this; - } - - public Builder walSyncOnCommit(boolean walSyncOnCommit) { - this.walSyncOnCommit = walSyncOnCommit; - return this; - } - - public Builder replicaMode(boolean replicaMode) { - this.replicaMode = replicaMode; - return this; - } - - public Builder replicaSyncIntervalUs(long replicaSyncIntervalUs) { - this.replicaSyncIntervalUs = replicaSyncIntervalUs; - return this; - } - - public Builder replicaReplayWal(boolean replicaReplayWal) { - this.replicaReplayWal = replicaReplayWal; - return this; - } - - public ObjectStoreConfig build() { - validate(); - return new ObjectStoreConfig(this); - } - - private void validate() { - if (localCacheMaxBytes < 0) { - throw new IllegalArgumentException("localCacheMaxBytes must not be negative, was: " + localCacheMaxBytes); - } - if (maxConcurrentUploads <= 0) { - throw new IllegalArgumentException("maxConcurrentUploads must be positive, was: " + maxConcurrentUploads); - } - if (maxConcurrentDownloads <= 0) { - throw new IllegalArgumentException("maxConcurrentDownloads must be positive, was: " + maxConcurrentDownloads); - } - if (multipartThreshold < 0) { - throw new IllegalArgumentException("multipartThreshold must not be negative, was: " + multipartThreshold); - } - if (multipartPartSize < 0) { - throw new IllegalArgumentException("multipartPartSize must not be negative, was: " + multipartPartSize); - } - if (walSyncThresholdBytes < 0) { - throw new IllegalArgumentException("walSyncThresholdBytes must not be negative, was: " + walSyncThresholdBytes); - } - if (replicaSyncIntervalUs < 0) { - throw new IllegalArgumentException("replicaSyncIntervalUs must not be negative, was: " + replicaSyncIntervalUs); - } - } - } -} diff --git a/src/main/java/com/tidesdb/PreparedTransaction.java b/src/main/java/com/tidesdb/PreparedTransaction.java new file mode 100644 index 0000000..7a167c1 --- /dev/null +++ b/src/main/java/com/tidesdb/PreparedTransaction.java @@ -0,0 +1,71 @@ +/** + * + * Copyright (C) TidesDB + * + * Original Author: Alex Gaetano Padula + * + * Licensed under the Mozilla Public License, v. 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.mozilla.org/en-US/MPL/2.0/ + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.tidesdb; + +/** + * One transaction that was durably prepared before a restart and has no decision + * recorded after it, returned from {@link TidesDB#recoverPrepared()}. + * + *

Resolve it with {@link Transaction#commitPrepared()} or + * {@link Transaction#rollbackPrepared()}, then free the transaction like any + * other. A transaction that was decided in the log is settled during open and + * never appears here. + */ +public class PreparedTransaction { + + private final Transaction transaction; + private final byte[] xid; + + /** + * Creates a new {@code PreparedTransaction}. Typically called by the JNI + * bridge rather than application code. + * + * @param transaction a handle in the {@link TransactionState#PREPARED} state + * @param xid the transaction id the coordinator prepared it under + */ + public PreparedTransaction(Transaction transaction, byte[] xid) { + this.transaction = transaction; + this.xid = xid == null ? new byte[0] : xid.clone(); + } + + /** + * Returns the in-doubt transaction, in the {@link TransactionState#PREPARED} + * state. The caller owns it and must free it once resolved. + * + * @return the transaction handle + */ + public Transaction getTransaction() { + return transaction; + } + + /** + * Returns the transaction id the coordinator prepared this transaction + * under. + * + * @return a copy of the xid bytes + */ + public byte[] getXid() { + return xid.clone(); + } + + @Override + public String toString() { + return "PreparedTransaction{xidSize=" + xid.length + '}'; + } +} diff --git a/src/main/java/com/tidesdb/RangeStats.java b/src/main/java/com/tidesdb/RangeStats.java new file mode 100644 index 0000000..d1b4d6e --- /dev/null +++ b/src/main/java/com/tidesdb/RangeStats.java @@ -0,0 +1,86 @@ +/** + * + * Copyright (C) TidesDB + * + * Original Author: Alex Gaetano Padula + * + * Licensed under the Mozilla Public License, v. 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.mozilla.org/en-US/MPL/2.0/ + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.tidesdb; + +/** + * What a query planner needs to know about a key range, returned from + * {@link ColumnFamily#rangeStats(byte[], byte[])}. Both figures are taken from + * one layout snapshot, so they describe the same instant. + */ +public class RangeStats { + + private final long sstablesOverlapping; + private final long estimatedKeys; + private final boolean keysExact; + + /** + * Creates a new {@code RangeStats}. Typically called by the JNI bridge + * rather than application code. + * + * @param sstablesOverlapping sorted runs a scan of the range would merge + * @param estimatedKeys live keys the range holds + * @param keysExact whether {@code estimatedKeys} was counted rather than + * estimated from metadata + */ + public RangeStats(long sstablesOverlapping, long estimatedKeys, boolean keysExact) { + this.sstablesOverlapping = sstablesOverlapping; + this.estimatedKeys = estimatedKeys; + this.keysExact = keysExact; + } + + /** + * Returns the number of sorted runs a scan of the range would merge, which + * is the shape of its cost. + * + * @return the overlapping SSTable count + */ + public long getSstablesOverlapping() { + return sstablesOverlapping; + } + + /** + * Returns the live keys the range holds, with tombstoned and superseded + * versions excluded. + * + * @return the key count + */ + public long getEstimatedKeys() { + return estimatedKeys; + } + + /** + * Returns whether {@link #getEstimatedKeys()} was counted rather than + * estimated from metadata, so a planner can trust it outright instead of + * hedging. + * + * @return {@code true} when the count is exact + */ + public boolean isKeysExact() { + return keysExact; + } + + @Override + public String toString() { + return "RangeStats{" + + "sstablesOverlapping=" + sstablesOverlapping + + ", estimatedKeys=" + estimatedKeys + + ", keysExact=" + keysExact + + '}'; + } +} diff --git a/src/main/java/com/tidesdb/S3Config.java b/src/main/java/com/tidesdb/S3Config.java deleted file mode 100644 index 4b070da..0000000 --- a/src/main/java/com/tidesdb/S3Config.java +++ /dev/null @@ -1,169 +0,0 @@ -/** - * - * Copyright (C) TidesDB - * - * Original Author: Alex Gaetano Padula - * - * Licensed under the Mozilla Public License, v. 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.mozilla.org/en-US/MPL/2.0/ - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.tidesdb; - -/** - * Configuration for an S3-compatible object store connector (AWS S3, MinIO, etc.). - * - *

Mirrors {@code tidesdb_objstore_s3_config_t}. Set it on a {@link Config} via - * {@link Config.Builder#objectStoreS3Config(S3Config)} to back a database with object storage. - * Combine it with an {@link ObjectStoreConfig} (cache, multipart, WAL replication, replica mode) - * for full control over object store behavior. - * - *

Requires the native TidesDB library to have been built with {@code TIDESDB_WITH_S3=ON}; - * otherwise opening the database throws a {@link TidesDBException}. Use - * {@link TidesDB#isS3Available()} to probe support at runtime. - */ -public class S3Config { - - private final String endpoint; - private final String bucket; - private final String prefix; - private final String accessKey; - private final String secretKey; - private final String region; - private final boolean useSsl; - private final boolean usePathStyle; - private final String tlsCaPath; - private final boolean tlsInsecureSkipVerify; - private final long multipartThreshold; - private final long multipartPartSize; - - private S3Config(Builder builder) { - this.endpoint = builder.endpoint; - this.bucket = builder.bucket; - this.prefix = builder.prefix; - this.accessKey = builder.accessKey; - this.secretKey = builder.secretKey; - this.region = builder.region; - this.useSsl = builder.useSsl; - this.usePathStyle = builder.usePathStyle; - this.tlsCaPath = builder.tlsCaPath; - this.tlsInsecureSkipVerify = builder.tlsInsecureSkipVerify; - this.multipartThreshold = builder.multipartThreshold; - this.multipartPartSize = builder.multipartPartSize; - } - - public static Builder builder() { - return new Builder(); - } - - public String getEndpoint() { return endpoint; } - public String getBucket() { return bucket; } - public String getPrefix() { return prefix; } - public String getAccessKey() { return accessKey; } - public String getSecretKey() { return secretKey; } - public String getRegion() { return region; } - public boolean isUseSsl() { return useSsl; } - public boolean isUsePathStyle() { return usePathStyle; } - public String getTlsCaPath() { return tlsCaPath; } - public boolean isTlsInsecureSkipVerify() { return tlsInsecureSkipVerify; } - public long getMultipartThreshold() { return multipartThreshold; } - public long getMultipartPartSize() { return multipartPartSize; } - - /** - * Builder for {@link S3Config}. {@code endpoint}, {@code bucket}, {@code accessKey}, and - * {@code secretKey} are required; the rest default to secure, AWS-friendly values - * (HTTPS on, virtual-hosted URLs, TLS verification enabled, built-in multipart sizing). - */ - public static class Builder { - private String endpoint; - private String bucket; - private String prefix = null; - private String accessKey; - private String secretKey; - private String region = null; - private boolean useSsl = true; - private boolean usePathStyle = false; - private String tlsCaPath = null; - private boolean tlsInsecureSkipVerify = false; - private long multipartThreshold = 0; // 0 = library default - private long multipartPartSize = 0; // 0 = library default - - /** S3 endpoint, e.g. "s3.amazonaws.com" or "minio.local:9000" (required). */ - public Builder endpoint(String endpoint) { this.endpoint = endpoint; return this; } - - /** Bucket name (required). */ - public Builder bucket(String bucket) { this.bucket = bucket; return this; } - - /** Key prefix, e.g. "production/db1/" (optional). */ - public Builder prefix(String prefix) { this.prefix = prefix; return this; } - - /** AWS access key ID (required). */ - public Builder accessKey(String accessKey) { this.accessKey = accessKey; return this; } - - /** AWS secret access key (required). */ - public Builder secretKey(String secretKey) { this.secretKey = secretKey; return this; } - - /** AWS region, e.g. "us-east-1"; null for MinIO/default. */ - public Builder region(String region) { this.region = region; return this; } - - /** 1 for HTTPS (default), 0 for HTTP. */ - public Builder useSsl(boolean useSsl) { this.useSsl = useSsl; return this; } - - /** Path-style URLs (MinIO) when true; virtual-hosted (AWS) when false (default). */ - public Builder usePathStyle(boolean usePathStyle) { this.usePathStyle = usePathStyle; return this; } - - /** Custom CA bundle file path, or null for the system bundle. */ - public Builder tlsCaPath(String tlsCaPath) { this.tlsCaPath = tlsCaPath; return this; } - - /** - * Disable TLS peer and host verification when true (test only, insecure). Default false - * keeps verification on. - */ - public Builder tlsInsecureSkipVerify(boolean tlsInsecureSkipVerify) { - this.tlsInsecureSkipVerify = tlsInsecureSkipVerify; - return this; - } - - /** Object size at/above which multipart upload is used; 0 uses the library default. */ - public Builder multipartThreshold(long multipartThreshold) { - this.multipartThreshold = multipartThreshold; - return this; - } - - /** Multipart chunk size in bytes; 0 uses the library default. */ - public Builder multipartPartSize(long multipartPartSize) { - this.multipartPartSize = multipartPartSize; - return this; - } - - public S3Config build() { - if (endpoint == null || endpoint.isEmpty()) { - throw new IllegalArgumentException("S3 endpoint is required"); - } - if (bucket == null || bucket.isEmpty()) { - throw new IllegalArgumentException("S3 bucket is required"); - } - if (accessKey == null || accessKey.isEmpty()) { - throw new IllegalArgumentException("S3 access key is required"); - } - if (secretKey == null || secretKey.isEmpty()) { - throw new IllegalArgumentException("S3 secret key is required"); - } - if (multipartThreshold < 0) { - throw new IllegalArgumentException("multipartThreshold must not be negative, was: " + multipartThreshold); - } - if (multipartPartSize < 0) { - throw new IllegalArgumentException("multipartPartSize must not be negative, was: " + multipartPartSize); - } - return new S3Config(this); - } - } -} diff --git a/src/main/java/com/tidesdb/Snapshot.java b/src/main/java/com/tidesdb/Snapshot.java new file mode 100644 index 0000000..719514a --- /dev/null +++ b/src/main/java/com/tidesdb/Snapshot.java @@ -0,0 +1,112 @@ +/** + * + * Copyright (C) TidesDB + * + * Original Author: Alex Gaetano Padula + * + * Licensed under the Mozilla Public License, v. 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.mozilla.org/en-US/MPL/2.0/ + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.tidesdb; + +import java.io.Closeable; + +/** + * Names the database as it stood at a point in time, so it can be read again + * later through {@link TidesDB#beginTransactionAtSnapshot(Snapshot)}. + * + *

A snapshot holds the reclamation floor at its own sequence for as long as + * it lives, which is what keeps the versions it resolves to from being compacted + * away, and is also its cost: the same cost a long-running transaction has. + * Release it as soon as the point in time is no longer wanted. + * + *

{@code Snapshot} implements {@link java.io.Closeable} for use with + * try-with-resources. Every {@link Transaction} opened against a snapshot must + * be freed before the snapshot is released, since they read versions only it + * keeps alive. + * + *

This class is not guaranteed to be thread-safe. + */ +public class Snapshot implements Closeable { + + static { + NativeLibrary.load(); + } + + private long nativeHandle; + private final long seq; + private boolean released = false; + + Snapshot(long nativeHandle) { + this.nativeHandle = nativeHandle; + this.seq = nativeSeq(nativeHandle); + } + + /** + * Returns the sequence this snapshot reads at, for reporting and for + * comparing against {@link DbStats#getMinSnapshotSeq()}. + * + *

The sequence is read once at creation and stays available after the + * snapshot is released. + * + * @return the snapshot sequence + */ + public long getSeq() { + return seq; + } + + /** + * Releases the snapshot and the reclamation floor it was holding. + * + *

This method is idempotent; subsequent calls are no-ops. + */ + public void release() { + if (!released && nativeHandle != 0) { + nativeRelease(nativeHandle); + nativeHandle = 0; + released = true; + } + } + + /** + * Releases this snapshot. Equivalent to {@link #release()}. + */ + @Override + public void close() { + release(); + } + + /** + * Reports whether this snapshot has been released. + * + * @return {@code true} once released + */ + public boolean isReleased() { + return released; + } + + long getNativeHandle() { + if (released) { + throw new IllegalStateException("Snapshot has been released"); + } + return nativeHandle; + } + + @Override + public String toString() { + return "Snapshot{seq=" + seq + ", released=" + released + '}'; + } + + private static native long nativeSeq(long handle); + + private static native void nativeRelease(long handle); +} diff --git a/src/main/java/com/tidesdb/StallReason.java b/src/main/java/com/tidesdb/StallReason.java new file mode 100644 index 0000000..782555f --- /dev/null +++ b/src/main/java/com/tidesdb/StallReason.java @@ -0,0 +1,109 @@ +/** + * + * Copyright (C) TidesDB + * + * Original Author: Alex Gaetano Padula + * + * Licensed under the Mozilla Public License, v. 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.mozilla.org/en-US/MPL/2.0/ + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.tidesdb; + +/** + * The places a caller's own thread can be made to wait inside a write. A commit + * that took far longer than its peers was held at exactly one of these, and + * knowing which is the difference between a device that cannot keep up and an + * engine that is not letting it. + * + *

The ordinal of each constant is its index into {@link StallStats}. + */ +public enum StallReason { + + /** + * Waiting on the write-ahead log, either for staging-ring space or, under a + * syncing mode, for the record to reach the file. The two are one figure + * because both are the same wait on the same single writer. + */ + WAL_APPEND(0), + + /** + * Waiting to take the rotation lock, so another committer was rotating. + */ + ROTATE_LOCK(1), + + /** + * Performing the rotation, which this thread pays on everyone's behalf. + */ + ROTATE_WORK(2), + + /** + * Held by write admission because the unflushed backlog was too deep. + */ + ADMISSION(3), + + /** + * Inside a manifest commit, which every flush install, every compaction + * install and every DDL serialises through, so a database making no + * progress is often waiting here. + */ + MANIFEST_COMMIT(4); + + static { + NativeLibrary.load(); + } + + private final int value; + + StallReason(int value) { + this.value = value; + } + + /** + * Returns the JNI numeric mapping for this stall reason, which is also its + * index into {@link StallStats}. + * + * @return the integer value used by the native library + */ + public int getValue() { + return value; + } + + /** + * Returns the native library's stable short name for this reason, suitable + * for a log line or a stats table's row label. + * + * @return the name, never {@code null} + */ + public String getNativeName() { + return nativeName(value); + } + + private static native String nativeName(int reason); + + /** + * Returns the {@link StallReason} constant matching the given JNI integer + * value. + * + * @param value the JNI integer value + * @return the matching constant + * @throws IllegalArgumentException if {@code value} does not map to any + * known constant + */ + public static StallReason fromValue(int value) { + for (StallReason reason : values()) { + if (reason.value == value) { + return reason; + } + } + throw new IllegalArgumentException("Unknown stall reason value: " + value); + } +} diff --git a/src/main/java/com/tidesdb/StallStat.java b/src/main/java/com/tidesdb/StallStat.java new file mode 100644 index 0000000..e469321 --- /dev/null +++ b/src/main/java/com/tidesdb/StallStat.java @@ -0,0 +1,81 @@ +/** + * + * Copyright (C) TidesDB + * + * Original Author: Alex Gaetano Padula + * + * Licensed under the Mozilla Public License, v. 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.mozilla.org/en-US/MPL/2.0/ + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.tidesdb; + +/** + * How much waiting one {@link StallReason} accounted for since the database + * opened. Created by the native library and returned inside {@link StallStats}. + */ +public class StallStat { + + private final long count; + private final long totalUs; + private final long maxUs; + + /** + * Creates a new {@code StallStat}. Typically called by the JNI bridge + * rather than application code. + * + * @param count how many times a thread waited here + * @param totalUs the summed wait in microseconds + * @param maxUs the longest single wait in microseconds + */ + public StallStat(long count, long totalUs, long maxUs) { + this.count = count; + this.totalUs = totalUs; + this.maxUs = maxUs; + } + + /** + * Returns how many times a thread waited here. + * + * @return the wait count + */ + public long getCount() { + return count; + } + + /** + * Returns the summed wait, so a reason's share of all waiting is + * comparable. + * + * @return the total wait in microseconds + */ + public long getTotalUs() { + return totalUs; + } + + /** + * Returns the longest single wait, which is what a latency tail is made of. + * + * @return the longest wait in microseconds + */ + public long getMaxUs() { + return maxUs; + } + + @Override + public String toString() { + return "StallStat{" + + "count=" + count + + ", totalUs=" + totalUs + + ", maxUs=" + maxUs + + '}'; + } +} diff --git a/src/main/java/com/tidesdb/StallStats.java b/src/main/java/com/tidesdb/StallStats.java new file mode 100644 index 0000000..e1dab4c --- /dev/null +++ b/src/main/java/com/tidesdb/StallStats.java @@ -0,0 +1,95 @@ +/** + * + * Copyright (C) TidesDB + * + * Original Author: Alex Gaetano Padula + * + * Licensed under the Mozilla Public License, v. 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.mozilla.org/en-US/MPL/2.0/ + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.tidesdb; + +/** + * Where writers have been made to wait, one entry per {@link StallReason}, + * returned from {@link TidesDB#getStallStats()}. + * + *

A write latency tail is answerable from this alone: compare each reason's + * {@link StallStat#getMaxUs()} against the tail you measured, and its + * {@link StallStat#getTotalUs()} against the others. + */ +public class StallStats { + + private final StallStat[] reasons; + + /** + * Creates a new {@code StallStats}. Typically called by the JNI bridge + * rather than application code. + * + * @param reasons the per-reason totals, indexed by {@link StallReason#getValue()}; + * must not be {@code null} and must hold one entry per reason + */ + public StallStats(StallStat[] reasons) { + if (reasons == null || reasons.length != StallReason.values().length) { + throw new IllegalArgumentException( + "reasons must hold exactly " + StallReason.values().length + " entries"); + } + this.reasons = reasons.clone(); + } + + /** + * Returns the totals for one wait reason. + * + * @param reason the reason; must not be {@code null} + * @return the totals for that reason, never {@code null} + */ + public StallStat get(StallReason reason) { + if (reason == null) { + throw new IllegalArgumentException("Stall reason cannot be null"); + } + return reasons[reason.getValue()]; + } + + /** + * Returns the totals for every wait reason, indexed by + * {@link StallReason#getValue()}. + * + * @return a copy of the per-reason totals + */ + public StallStat[] getReasons() { + return reasons.clone(); + } + + /** + * Returns the summed wait across every reason. + * + * @return the total wait in microseconds + */ + public long getTotalUs() { + long total = 0; + for (StallStat stat : reasons) { + total += stat.getTotalUs(); + } + return total; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("StallStats{"); + for (StallReason reason : StallReason.values()) { + if (reason.getValue() > 0) { + sb.append(", "); + } + sb.append(reason.name()).append('=').append(reasons[reason.getValue()]); + } + return sb.append('}').toString(); + } +} diff --git a/src/main/java/com/tidesdb/Stats.java b/src/main/java/com/tidesdb/Stats.java deleted file mode 100644 index 5122b1c..0000000 --- a/src/main/java/com/tidesdb/Stats.java +++ /dev/null @@ -1,470 +0,0 @@ -/** - * - * Copyright (C) TidesDB - * - * Original Author: Alex Gaetano Padula - * - * Licensed under the Mozilla Public License, v. 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.mozilla.org/en-US/MPL/2.0/ - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.tidesdb; - -/** - * Statistics about a column family, including level structure, memtable size, - * and the active configuration. Created by the native library and returned - * from {@link ColumnFamily#getStats()}. - * - *

The arrays returned by {@link #getLevelSizes()} and - * {@link #getLevelNumSSTables()} are the internal stored references, not - * defensive copies. Callers must not modify them. - */ -public class Stats { - - private final int numLevels; - private final long memtableSize; - private final long[] levelSizes; - private final int[] levelNumSSTables; - private final ColumnFamilyConfig config; - private final long totalKeys; - private final long totalDataSize; - private final double avgKeySize; - private final double avgValueSize; - private final long[] levelKeyCounts; - private final double readAmp; - private final double hitRate; - private final boolean useBtree; - private final long btreeTotalNodes; - private final int btreeMaxHeight; - private final double btreeAvgHeight; - private final long totalTombstones; - private final double tombstoneRatio; - private final long[] levelTombstoneCounts; - private final double maxSstDensity; - private final int maxSstDensityLevel; - private final long walBytesWritten; - private final long flushBytesWritten; - private final long compactionBytesWritten; - private final long compactionBytesRead; - private final long userBytesWritten; - private final long flushCount; - private final long compactionCount; - - /** - * Creates a new {@code Stats} instance. Typically called by the JNI - * bridge rather than application code. - * - * @param numLevels the number of LSM levels - * @param memtableSize the current memtable size in bytes - * @param levelSizes the size in bytes for each level (stored by reference) - * @param levelNumSSTables the SSTable count for each level (stored by reference) - * @param config the column family configuration - * @param totalKeys total number of keys across memtable and all SSTables - * @param totalDataSize total data size (klog + vlog) across all SSTables - * @param avgKeySize average key size in bytes - * @param avgValueSize average value size in bytes - * @param levelKeyCounts number of keys per level - * @param readAmp read amplification (point lookup cost multiplier) - * @param hitRate cache hit rate (0.0 to 1.0) - * @param useBtree whether this column family uses B+tree format - * @param btreeTotalNodes total number of B+tree nodes across all SSTables - * @param btreeMaxHeight maximum B+tree height across all SSTables - * @param btreeAvgHeight average B+tree height across all SSTables - * @param totalTombstones total number of tombstones across every SSTable - * @param tombstoneRatio tombstone ratio (totalTombstones / totalKeys) - * @param levelTombstoneCounts per-level tombstone counts - * @param maxSstDensity worst per-SSTable tombstone density - * @param maxSstDensityLevel 1-based level index where worst density was observed - * @param walBytesWritten framed bytes appended to WAL (lifetime since open) - * @param flushBytesWritten on-disk bytes flushes wrote to L0 SSTables - * @param compactionBytesWritten on-disk bytes compactions wrote - * @param compactionBytesRead on-disk bytes compactions read as input - * @param userBytesWritten logical key+value bytes committed - * @param flushCount number of flushed SSTables produced - * @param compactionCount number of compaction output SSTables produced - */ - public Stats(int numLevels, long memtableSize, long[] levelSizes, int[] levelNumSSTables, - ColumnFamilyConfig config, long totalKeys, long totalDataSize, - double avgKeySize, double avgValueSize, long[] levelKeyCounts, - double readAmp, double hitRate, boolean useBtree, long btreeTotalNodes, - int btreeMaxHeight, double btreeAvgHeight, - long totalTombstones, double tombstoneRatio, long[] levelTombstoneCounts, - double maxSstDensity, int maxSstDensityLevel, - long walBytesWritten, long flushBytesWritten, long compactionBytesWritten, - long compactionBytesRead, long userBytesWritten, long flushCount, - long compactionCount) { - this.numLevels = numLevels; - this.memtableSize = memtableSize; - this.levelSizes = levelSizes; - this.levelNumSSTables = levelNumSSTables; - this.config = config; - this.totalKeys = totalKeys; - this.totalDataSize = totalDataSize; - this.avgKeySize = avgKeySize; - this.avgValueSize = avgValueSize; - this.levelKeyCounts = levelKeyCounts; - this.readAmp = readAmp; - this.hitRate = hitRate; - this.useBtree = useBtree; - this.btreeTotalNodes = btreeTotalNodes; - this.btreeMaxHeight = btreeMaxHeight; - this.btreeAvgHeight = btreeAvgHeight; - this.totalTombstones = totalTombstones; - this.tombstoneRatio = tombstoneRatio; - this.levelTombstoneCounts = levelTombstoneCounts; - this.maxSstDensity = maxSstDensity; - this.maxSstDensityLevel = maxSstDensityLevel; - this.walBytesWritten = walBytesWritten; - this.flushBytesWritten = flushBytesWritten; - this.compactionBytesWritten = compactionBytesWritten; - this.compactionBytesRead = compactionBytesRead; - this.userBytesWritten = userBytesWritten; - this.flushCount = flushCount; - this.compactionCount = compactionCount; - } - - /** - * Returns the number of LSM levels. - * - * @return the number of levels - */ - public int getNumLevels() { - return numLevels; - } - - /** - * Returns the current memtable size in bytes. - * - * @return the memtable size in bytes - */ - public long getMemtableSize() { - return memtableSize; - } - - /** - * Returns the sizes of each LSM level in bytes. - * - *

The returned array is the stored internal reference, not a - * defensive copy. Callers must not modify it. - * - * @return the level sizes in bytes (internal reference) - */ - public long[] getLevelSizes() { - return levelSizes; - } - - /** - * Returns the number of SSTables at each LSM level. - * - *

The returned array is the stored internal reference, not a - * defensive copy. Callers must not modify it. - * - * @return the SSTable counts per level (internal reference) - */ - public int[] getLevelNumSSTables() { - return levelNumSSTables; - } - - /** - * Returns the column family configuration active when these statistics - * were captured. - * - * @return the configuration, never {@code null} - */ - public ColumnFamilyConfig getConfig() { - return config; - } - - /** - * Gets the total number of keys across memtable and all SSTables. - * - * @return total key count - */ - public long getTotalKeys() { - return totalKeys; - } - - /** - * Gets the total data size (klog + vlog) across all SSTables. - * - * @return total data size in bytes - */ - public long getTotalDataSize() { - return totalDataSize; - } - - /** - * Gets the average key size in bytes. - * - * @return average key size - */ - public double getAvgKeySize() { - return avgKeySize; - } - - /** - * Gets the average value size in bytes. - * - * @return average value size - */ - public double getAvgValueSize() { - return avgValueSize; - } - - /** - * Gets the number of keys per level. - * - * @return array of key counts per level - */ - public long[] getLevelKeyCounts() { - return levelKeyCounts; - } - - /** - * Gets the read amplification (point lookup cost multiplier). - * - * @return read amplification factor - */ - public double getReadAmp() { - return readAmp; - } - - /** - * Gets the cache hit rate for this column family. - * - * @return hit rate (0.0 to 1.0), or 0.0 if cache is disabled - */ - public double getHitRate() { - return hitRate; - } - - /** - * Returns whether this column family uses B+tree format. - * - * @return true if B+tree format is used - */ - public boolean isUseBtree() { - return useBtree; - } - - /** - * Gets the total number of B+tree nodes across all SSTables. - * Only populated when useBtree is true. - * - * @return total B+tree nodes - */ - public long getBtreeTotalNodes() { - return btreeTotalNodes; - } - - /** - * Gets the maximum B+tree height across all SSTables. - * Only populated when useBtree is true. - * - * @return maximum tree height - */ - public int getBtreeMaxHeight() { - return btreeMaxHeight; - } - - /** - * Gets the average B+tree height across all SSTables. - * Only populated when useBtree is true. - * - * @return average tree height - */ - public double getBtreeAvgHeight() { - return btreeAvgHeight; - } - - /** - * Gets the total number of tombstones across every SSTable in the column family. - * - * @return total tombstone count - */ - public long getTotalTombstones() { - return totalTombstones; - } - - /** - * Gets the tombstone ratio (totalTombstones / totalKeys). - * Returns 0.0 when totalKeys is 0. Always within [0.0, 1.0]. - * - * @return tombstone ratio - */ - public double getTombstoneRatio() { - return tombstoneRatio; - } - - /** - * Gets the per-level tombstone counts. Length matches numLevels and parallels - * {@link #getLevelKeyCounts()}. - * - * @return per-level tombstone counts - */ - public long[] getLevelTombstoneCounts() { - return levelTombstoneCounts; - } - - /** - * Gets the worst per-SSTable tombstone density (tombstone_count / num_entries) - * observed in this column family. Always within [0.0, 1.0]. - * - * @return max per-SSTable tombstone density - */ - public double getMaxSstDensity() { - return maxSstDensity; - } - - /** - * Gets the 1-based level index where the worst per-SSTable tombstone density - * was observed. Returns 0 if no SSTable contributed to the measurement. - * - * @return 1-based level index of the worst SSTable, or 0 if none - */ - public int getMaxSstDensityLevel() { - return maxSstDensityLevel; - } - - /** - * Gets the framed bytes appended to this column family's WAL (lifetime since open). - * Always 0 in unified memtable mode, where the shared WAL volume is reported db-wide - * via {@link DbStats#getUwalBytesWritten()}. - * - * @return WAL bytes written - */ - public long getWalBytesWritten() { - return walBytesWritten; - } - - /** - * Gets the on-disk bytes this column family's flushes wrote to L0 SSTables - * (lifetime since open). - * - * @return flush output bytes written - */ - public long getFlushBytesWritten() { - return flushBytesWritten; - } - - /** - * Gets the on-disk bytes this column family's compactions wrote (lifetime since open). - * - * @return compaction output bytes written - */ - public long getCompactionBytesWritten() { - return compactionBytesWritten; - } - - /** - * Gets the on-disk bytes this column family's compactions read as input - * (lifetime since open). - * - * @return compaction input bytes read - */ - public long getCompactionBytesRead() { - return compactionBytesRead; - } - - /** - * Gets the logical key+value bytes committed to this column family (lifetime since open). - * This is the write-amplification denominator: divide the WAL, flush, and compaction - * write totals by this value to compute write amplification. - * - * @return user bytes written - */ - public long getUserBytesWritten() { - return userBytesWritten; - } - - /** - * Gets the number of flushed SSTables produced by this column family (lifetime since open). - * - * @return flush count - */ - public long getFlushCount() { - return flushCount; - } - - /** - * Gets the number of compaction output SSTables produced by this column family - * (lifetime since open). - * - * @return compaction count - */ - public long getCompactionCount() { - return compactionCount; - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("Stats{numLevels=").append(numLevels); - sb.append(", memtableSize=").append(memtableSize); - sb.append(", totalKeys=").append(totalKeys); - sb.append(", totalDataSize=").append(totalDataSize); - sb.append(", avgKeySize=").append(avgKeySize); - sb.append(", avgValueSize=").append(avgValueSize); - sb.append(", readAmp=").append(readAmp); - sb.append(", hitRate=").append(hitRate); - sb.append(", useBtree=").append(useBtree); - if (useBtree) { - sb.append(", btreeTotalNodes=").append(btreeTotalNodes); - sb.append(", btreeMaxHeight=").append(btreeMaxHeight); - sb.append(", btreeAvgHeight=").append(btreeAvgHeight); - } - sb.append(", totalTombstones=").append(totalTombstones); - sb.append(", tombstoneRatio=").append(tombstoneRatio); - sb.append(", maxSstDensity=").append(maxSstDensity); - sb.append(", maxSstDensityLevel=").append(maxSstDensityLevel); - sb.append(", walBytesWritten=").append(walBytesWritten); - sb.append(", flushBytesWritten=").append(flushBytesWritten); - sb.append(", compactionBytesWritten=").append(compactionBytesWritten); - sb.append(", compactionBytesRead=").append(compactionBytesRead); - sb.append(", userBytesWritten=").append(userBytesWritten); - sb.append(", flushCount=").append(flushCount); - sb.append(", compactionCount=").append(compactionCount); - if (levelSizes != null) { - sb.append(", levelSizes=["); - for (int i = 0; i < levelSizes.length; i++) { - if (i > 0) sb.append(", "); - sb.append(levelSizes[i]); - } - sb.append("]"); - } - if (levelNumSSTables != null) { - sb.append(", levelNumSSTables=["); - for (int i = 0; i < levelNumSSTables.length; i++) { - if (i > 0) sb.append(", "); - sb.append(levelNumSSTables[i]); - } - sb.append("]"); - } - if (levelKeyCounts != null) { - sb.append(", levelKeyCounts=["); - for (int i = 0; i < levelKeyCounts.length; i++) { - if (i > 0) sb.append(", "); - sb.append(levelKeyCounts[i]); - } - sb.append("]"); - } - if (levelTombstoneCounts != null) { - sb.append(", levelTombstoneCounts=["); - for (int i = 0; i < levelTombstoneCounts.length; i++) { - if (i > 0) sb.append(", "); - sb.append(levelTombstoneCounts[i]); - } - sb.append("]"); - } - sb.append("}"); - return sb.toString(); - } -} diff --git a/src/main/java/com/tidesdb/TidesDB.java b/src/main/java/com/tidesdb/TidesDB.java index 3017e3e..419fc89 100644 --- a/src/main/java/com/tidesdb/TidesDB.java +++ b/src/main/java/com/tidesdb/TidesDB.java @@ -29,36 +29,44 @@ * library through JNI and owns the underlying database handle. * *

{@code TidesDB} implements {@link java.io.Closeable} and should be used with - * try-with-resources. Callers must close any {@link TidesDBIterator} and - * {@link Transaction} instances before closing the database. Closing an already-closed - * instance is a no-op. + * try-with-resources. Callers must close any {@link TidesDBIterator}, + * {@link Transaction}, and {@link Snapshot} derived from it before closing the + * database. Closing an already-closed instance is a no-op. * *

Operations on a closed instance throw {@link IllegalStateException}. * + *

The memtable, write-ahead log, block cache, and value log are + * database-level and shared by every column family, so the operations that act + * on them — {@link #flushMemtable()}, {@link #syncWal()}, {@link #checkpoint()} — + * live here rather than on {@link ColumnFamily}. + * *

This class is not guaranteed to be thread-safe. */ public class TidesDB implements Closeable { - + static { NativeLibrary.load(); } - + private long nativeHandle; private boolean closed = false; private final Set hookBearingCFs = new HashSet<>(); - + private TidesDB(long nativeHandle) { this.nativeHandle = nativeHandle; } - + /** - * Opens a TidesDB instance with the given configuration. + * Opens or creates a database at the configured path, recovering the + * manifest, SSTables, and write-ahead log, and starting the worker pool. * * @param config the database configuration; must not be {@code null} * @return a new TidesDB instance * @throws IllegalArgumentException if {@code config} is {@code null} or its * {@code dbPath} is {@code null} or empty - * @throws TidesDBException if the native database cannot be opened + * @throws TidesDBException if the native database cannot be opened, which + * includes {@link TidesDBException#ERR_LOCKED} when another handle + * already holds the directory, from this process or any other */ public static TidesDB open(Config config) throws TidesDBException { if (config == null) { @@ -67,30 +75,6 @@ public static TidesDB open(Config config) throws TidesDBException { if (config.getDbPath() == null || config.getDbPath().isEmpty()) { throw new IllegalArgumentException("Database path cannot be null or empty"); } - - ObjectStoreConfig osc = config.getObjectStoreConfig(); - - // Build an S3 connector up front, if requested. The native handle is owned by the - // database after a successful open (released by close), mirroring the filesystem - // connector path. Creation throws if the library was built without S3 support. - S3Config s3 = config.getObjectStoreS3Config(); - long objStoreHandle = 0; - if (s3 != null) { - objStoreHandle = nativeObjstoreS3Create( - s3.getEndpoint(), - s3.getBucket(), - s3.getPrefix(), - s3.getAccessKey(), - s3.getSecretKey(), - s3.getRegion(), - s3.isUseSsl(), - s3.isUsePathStyle(), - s3.getTlsCaPath(), - s3.isTlsInsecureSkipVerify(), - s3.getMultipartThreshold(), - s3.getMultipartPartSize() - ); - } long handle = nativeOpen( config.getDbPath(), @@ -101,64 +85,90 @@ public static TidesDB open(Config config) throws TidesDBException { config.getMaxOpenSSTables(), config.isLogToFile(), config.getLogTruncationAt(), - config.getMaxMemoryUsage(), - config.isUnifiedMemtable(), - config.getUnifiedMemtableWriteBufferSize(), - config.getUnifiedMemtableSkipListMaxLevel(), - config.getUnifiedMemtableSkipListProbability(), - config.getUnifiedMemtableSyncMode(), - config.getUnifiedMemtableSyncIntervalUs(), - config.getObjectStoreFsPath(), - osc != null ? osc.getLocalCachePath() : null, - osc != null ? osc.getLocalCacheMaxBytes() : 0, - osc != null ? osc.isCacheOnRead() : true, - osc != null ? osc.isCacheOnWrite() : true, - osc != null ? osc.getMaxConcurrentUploads() : 4, - osc != null ? osc.getMaxConcurrentDownloads() : 8, - osc != null ? osc.getMultipartThreshold() : 64 * 1024 * 1024, - osc != null ? osc.getMultipartPartSize() : 8 * 1024 * 1024, - osc != null ? osc.isSyncManifestToObject() : true, - osc != null ? osc.isReplicateWal() : true, - osc != null ? osc.isWalUploadSync() : false, - osc != null ? osc.getWalSyncThresholdBytes() : 1048576, - osc != null ? osc.isWalSyncOnCommit() : false, - osc != null ? osc.isReplicaMode() : false, - osc != null ? osc.getReplicaSyncIntervalUs() : 5000000, - osc != null ? osc.isReplicaReplayWal() : true, - config.getMaxConcurrentFlushes(), - config.isFinishCompactionsOnClose(), - objStoreHandle + config.getMemtableWriteBufferSize(), + config.getMemtableSkipListMaxLevel(), + config.getMemtableSkipListProbability(), + config.getMemtableSyncMode().getValue(), + config.getMemtableSyncIntervalUs(), + config.getValueSeparationThreshold(), + config.getVlogSegmentSize(), + config.getMemtableL0QueueStallThreshold(), + config.getMemtableIdleFlushSeconds(), + config.getTxnTimeoutSeconds() ); return new TidesDB(handle); } /** - * Reports whether the native TidesDB library was built with S3 object store support - * ({@code TIDESDB_WITH_S3=ON}). When false, configuring a {@link S3Config} and opening the - * database throws a {@link TidesDBException}. + * Reports whether this build of the native library can actually use a + * compression algorithm. Every algorithm is always named by the enum, but a + * backend is only linked in when its build option was set, so a caller + * choosing one for a column family's encoding pipeline asks here first + * rather than discovering it when a node fails to decode. + * + *

{@link CompressionAlgorithm#NONE} is always available. + * + * @param algorithm the algorithm to query; must not be {@code null} + * @return {@code true} if this build can compress and decompress with it + */ + public static boolean isCompressionAvailable(CompressionAlgorithm algorithm) { + if (algorithm == null) { + throw new IllegalArgumentException("Compression algorithm cannot be null"); + } + return nativeCompressionAvailable(algorithm.getValue()); + } + + /** + * Returns the native library's short description of a result code. An + * unrecognised code describes itself as unknown rather than failing. + * + * @param code any {@code ERR_*} value from {@link TidesDBException} + * @return the description, never {@code null} + */ + public static String strerror(int code) { + return nativeStrerror(code); + } + + /** + * Raises this process's open-file ceiling toward {@code desired} descriptors + * so a database can keep more SSTables open. The engine sizes + * {@link Config#getMaxOpenSSTables()} to fit this at open time, so call it + * before {@link #open(Config)}. This is an explicit, opt-in + * operator action; TidesDB never raises the limit itself. + * + *

On POSIX systems this raises the {@code RLIMIT_NOFILE} soft limit toward + * the hard limit; on Windows it raises the CRT stdio cap (max 8192). A failed + * or partial raise is non-fatal. * - * @return true if an S3-compatible connector can be created, false otherwise + * @param desired target descriptor count; values ≤ 0 just report the + * current ceiling + * @return the open-file ceiling in effect after the attempt */ - public static boolean isS3Available() { - return nativeS3Available(); + public static long raiseOpenFileLimit(long desired) { + return nativeRaiseOpenFileLimit(desired); } - + /** - * Closes the database and releases all native resources. + * Flushes, quiesces the workers, and releases all native resources. * *

This method is idempotent; subsequent calls are no-ops. After closing, * all other operations on this instance throw {@link IllegalStateException}. * - *

Callers should close all {@link TidesDBIterator} and {@link Transaction} - * instances before calling this method. + *

Callers should close all {@link TidesDBIterator}, {@link Transaction}, + * and {@link Snapshot} instances before calling this method. + * + *

The shutdown path reports no status, so an I/O error while writing the + * last of the data is logged rather than thrown. A caller that needs its data + * on the device asks for that before closing, with {@link #syncWal()}, + * {@link #checkpoint()}, or by running under {@link SyncMode#SYNC_FULL}. */ @Override public void close() { if (!closed && nativeHandle != 0) { closed = true; - // Drain all registered hooks. Loop because a concurrent setCommitHook + // Drain all registered hooks. Loop because a concurrent setCommitHook // may have passed checkOwnerOpen before closed=true but hasn't // registered yet; its registration is rejected (closed is true), but a // retry loop guarantees the set is empty before calling nativeClose. @@ -178,12 +188,17 @@ public void close() { nativeHandle = 0; } } - + + /* ===== column families ===== */ + /** - * Creates a new column family with the given configuration. + * Creates a column family and registers it in the manifest. * - * @param name the column family name; must not be {@code null} or empty - * @param config the column family configuration; must not be {@code null} + * @param name the column family name; must not be {@code null} or empty, and + * must be shorter than {@link ColumnFamilyConfig#MAX_NAME_LENGTH} + * @param config the column family configuration; must not be {@code null}. + * Its {@code name} field is ignored, since the {@code name} argument + * is authoritative * @throws IllegalArgumentException if {@code name} is {@code null} or empty, * or {@code config} is {@code null} * @throws IllegalStateException if this database is closed @@ -197,38 +212,30 @@ public void createColumnFamily(String name, ColumnFamilyConfig config) throws Ti if (config == null) { throw new IllegalArgumentException("Column family config cannot be null"); } - + nativeCreateColumnFamily(nativeHandle, name, - config.getWriteBufferSize(), config.getLevelSizeRatio(), config.getMinLevels(), config.getDividingLevelOffset(), - config.getKlogValueThreshold(), - config.getCompressionAlgorithm().getValue(), + config.isKeepValuesInline(), + config.getBtreeKlogBlockSize(), + config.getEncodingPipeline(), config.isEnableBloomFilter(), - config.getBloomFPR(), - config.isEnableBlockIndexes(), - config.getIndexSampleRatio(), - config.getBlockIndexPrefixLen(), - config.getSyncMode().getValue(), - config.getSyncIntervalUs(), - config.getComparatorName(), - config.getSkipListMaxLevel(), - config.getSkipListProbability(), + config.getBloomFpr(), config.getDefaultIsolationLevel().getValue(), - config.getMinDiskSpace(), config.getL1FileCountTrigger(), - config.getL0QueueStallThreshold(), config.getTombstoneDensityTrigger(), - config.getTombstoneDensityMinEntries(), - config.isUseBtree(), - config.isObjectLazyCompaction(), - config.isObjectPrefetchCompaction() + config.getTombstoneDensityMinEntries() ); } - + /** - * Drops a column family and all associated data. + * Drops a column family by name, deleting its SSTables and manifest records. + * Waits out any compaction already running on the family, so a heavily + * compacting family takes longer to drop. + * + *

This destroys data and cannot be undone. Any {@link ColumnFamily} handle + * held for the family is invalid afterwards. * * @param name the column family name; must not be {@code null} or empty * @throws IllegalArgumentException if {@code name} is {@code null} or empty @@ -242,16 +249,67 @@ public void dropColumnFamily(String name) throws TidesDBException { } nativeDropColumnFamily(nativeHandle, name); } - + + /** + * Atomically renames a column family. The family is claimed against the + * compaction scheduler, the whole database memtable is flushed — it is + * shared, so this is not just this family's data — and new flushes are frozen + * while the directory moves and the family reloads. + * + * @param oldName the current column family name + * @param newName the new column family name + * @throws IllegalArgumentException if either name is {@code null} or empty + * @throws IllegalStateException if this database is closed + * @throws TidesDBException if the rename fails, including + * {@link TidesDBException#ERR_LOCKED} when the family stayed under + * compaction for the whole quiesce window + */ + public void renameColumnFamily(String oldName, String newName) throws TidesDBException { + checkNotClosed(); + if (oldName == null || oldName.isEmpty()) { + throw new IllegalArgumentException("Old column family name cannot be null or empty"); + } + if (newName == null || newName.isEmpty()) { + throw new IllegalArgumentException("New column family name cannot be null or empty"); + } + nativeRenameColumnFamily(nativeHandle, oldName, newName); + } + /** - * Retrieves a column family by name. + * Clones a column family to a new name, copying its SSTables. The source is + * claimed against the compaction scheduler and the whole database memtable is + * flushed first, so the copy carries everything written before the call + * rather than only what had already reached disk. + * + *

The result is a point-in-time copy: later writes to the source do not + * appear in it. + * + * @param sourceName the source column family name + * @param destName the new cloned column family name + * @throws IllegalArgumentException if either name is {@code null} or empty + * @throws IllegalStateException if this database is closed + * @throws TidesDBException if the clone fails + */ + public void cloneColumnFamily(String sourceName, String destName) throws TidesDBException { + checkNotClosed(); + if (sourceName == null || sourceName.isEmpty()) { + throw new IllegalArgumentException("Source column family name cannot be null or empty"); + } + if (destName == null || destName.isEmpty()) { + throw new IllegalArgumentException("Destination column family name cannot be null or empty"); + } + nativeCloneColumnFamily(nativeHandle, sourceName, destName); + } + + /** + * Looks up a column family handle by name. * * @param name the column family name; must not be {@code null} or empty - * @return the column family handle + * @return the column family handle, never {@code null} * @throws IllegalArgumentException if {@code name} is {@code null} or empty * @throws IllegalStateException if this database is closed - * @throws TidesDBException if the column family is not found or a native - * error occurs + * @throws TidesDBException with {@link TidesDBException#ERR_NOT_FOUND} if no + * such column family exists */ public ColumnFamily getColumnFamily(String name) throws TidesDBException { checkNotClosed(); @@ -261,11 +319,12 @@ public ColumnFamily getColumnFamily(String name) throws TidesDBException { long cfHandle = nativeGetColumnFamily(nativeHandle, name); return new ColumnFamily(cfHandle, name, this); } - + /** - * Lists all column families in the database. + * Lists the names of every column family. * - * @return array of column family names, never {@code null} + * @return the column family names, never {@code null}; empty when there are + * none * @throws IllegalStateException if this database is closed * @throws TidesDBException if the native list operation fails */ @@ -273,9 +332,11 @@ public String[] listColumnFamilies() throws TidesDBException { checkNotClosed(); return nativeListColumnFamilies(nativeHandle); } - + + /* ===== transactions ===== */ + /** - * Begins a new transaction with the default isolation level. + * Begins a transaction at the database default isolation level. * *

Close the returned transaction before closing this database. * @@ -285,12 +346,11 @@ public String[] listColumnFamilies() throws TidesDBException { */ public Transaction beginTransaction() throws TidesDBException { checkNotClosed(); - long txnHandle = nativeBeginTransaction(nativeHandle); - return new Transaction(txnHandle); + return new Transaction(nativeBeginTransaction(nativeHandle)); } - + /** - * Begins a new transaction with the specified isolation level. + * Begins a transaction at an explicit isolation level. * *

Close the returned transaction before closing this database. * @@ -305,150 +365,206 @@ public Transaction beginTransaction(IsolationLevel isolationLevel) throws TidesD if (isolationLevel == null) { throw new IllegalArgumentException("Isolation level cannot be null"); } - long txnHandle = nativeBeginTransactionWithIsolation(nativeHandle, isolationLevel.getValue()); - return new Transaction(txnHandle); + return new Transaction( + nativeBeginTransactionWithIsolation(nativeHandle, isolationLevel.getValue())); } - + /** - * Retrieves statistics about the block cache. + * Begins a transaction at the given column family's default isolation level. * - * @return cache statistics, never {@code null} + * @param cf the column family whose default isolation to use; must not be + * {@code null} + * @return a new transaction + * @throws IllegalArgumentException if {@code cf} is {@code null} * @throws IllegalStateException if this database is closed - * @throws TidesDBException if the native stats retrieval fails + * @throws TidesDBException if the native transaction cannot be started */ - public CacheStats getCacheStats() throws TidesDBException { + public Transaction beginTransaction(ColumnFamily cf) throws TidesDBException { checkNotClosed(); - return nativeGetCacheStats(nativeHandle); + if (cf == null) { + throw new IllegalArgumentException("Column family cannot be null"); + } + return new Transaction(nativeBeginTransactionCf(nativeHandle, cf.getNativeHandle())); } - + /** - * Registers a custom comparator with the database. + * Names the database as it stands now, so it can be read again later. * - * @param name the comparator name; must not be {@code null} or empty - * @param context optional context string, may be {@code null} - * @throws IllegalArgumentException if {@code name} is {@code null} or empty + *

The snapshot holds the reclamation floor at its own sequence for as long + * as it lives. Release it as soon as the point in time is no longer wanted. + * + * @return a new snapshot * @throws IllegalStateException if this database is closed - * @throws TidesDBException if the native comparator registration fails + * @throws TidesDBException if the native snapshot cannot be created */ - public void registerComparator(String name, String context) throws TidesDBException { + public Snapshot createSnapshot() throws TidesDBException { checkNotClosed(); - if (name == null || name.isEmpty()) { - throw new IllegalArgumentException("Comparator name cannot be null or empty"); - } - nativeRegisterComparator(nativeHandle, name, context); + return new Snapshot(nativeSnapshotCreate(nativeHandle)); } - + /** - * Creates an on-disk snapshot of the database without blocking normal reads/writes. + * Begins a transaction whose reads resolve as of a snapshot rather than as of + * now: the same keys, the same families, answered as they stood when the + * snapshot was taken. * - * @param dir the backup directory (must be non-existent or empty) - * @throws TidesDBException if the backup fails + *

The snapshot must outlive the transaction, because it is what holds the + * floor under the versions being read. + * + * @param snapshot the snapshot to read at; must not be {@code null} + * @return a new transaction + * @throws IllegalArgumentException if {@code snapshot} is {@code null} + * @throws IllegalStateException if this database is closed or the snapshot + * has been released + * @throws TidesDBException if the native transaction cannot be started */ - public void backup(String dir) throws TidesDBException { + public Transaction beginTransactionAtSnapshot(Snapshot snapshot) throws TidesDBException { checkNotClosed(); - if (dir == null || dir.isEmpty()) { - throw new IllegalArgumentException("Backup directory cannot be null or empty"); + if (snapshot == null) { + throw new IllegalArgumentException("Snapshot cannot be null"); } - nativeBackup(nativeHandle, dir); + return new Transaction( + nativeBeginTransactionAtSnapshot(nativeHandle, snapshot.getNativeHandle())); } - + /** - * Creates a lightweight, near-instant snapshot of an open database using hard links - * instead of copying SSTable data. + * Begins a transaction reading as of an explicit sequence, for a point in + * time no snapshot was taken at — a sequence read back from + * {@link Transaction#getReadSnapshot()}, or one recorded elsewhere. + * + *

It refuses rather than approximates. A sequence is readable only while + * something holds the reclamation floor under it: an open transaction, or a + * snapshot taken in advance. Once a collection has run past that sequence the + * call fails with {@link TidesDBException#ERR_TOO_OLD} and reads nothing. * - * @param dir the checkpoint directory (must be non-existent or empty) - * @throws TidesDBException if the checkpoint fails + * @param seq the sequence to read at + * @return a new transaction + * @throws IllegalStateException if this database is closed + * @throws TidesDBException with {@link TidesDBException#ERR_TOO_OLD} when a + * collection has already run below {@code seq} */ - public void checkpoint(String dir) throws TidesDBException { + public Transaction beginTransactionAtSeq(long seq) throws TidesDBException { checkNotClosed(); - if (dir == null || dir.isEmpty()) { - throw new IllegalArgumentException("Checkpoint directory cannot be null or empty"); - } - nativeCheckpoint(nativeHandle, dir); + return new Transaction(nativeBeginTransactionAtSeq(nativeHandle, seq)); } - + /** - * Atomically renames a column family and its underlying directory. - * The operation waits for any in-progress flush or compaction to complete before renaming. + * Returns the oldest sequence {@link #beginTransactionAtSeq(long)} will still + * accept, which is the highest reclamation floor any collection has taken. It + * only ever rises, and a snapshot or an open transaction is what keeps it + * from rising past a point still wanted. * - * @param oldName the current column family name - * @param newName the new column family name - * @throws TidesDBException if the rename fails + * @return the oldest readable sequence + * @throws IllegalStateException if this database is closed */ - public void renameColumnFamily(String oldName, String newName) throws TidesDBException { + public long getOldestReadableSeq() { checkNotClosed(); - if (oldName == null || oldName.isEmpty()) { - throw new IllegalArgumentException("Old column family name cannot be null or empty"); - } - if (newName == null || newName.isEmpty()) { - throw new IllegalArgumentException("New column family name cannot be null or empty"); - } - nativeRenameColumnFamily(nativeHandle, oldName, newName); + return nativeOldestReadableSeq(nativeHandle); } - + /** - * Creates a complete copy of an existing column family with a new name. - * The clone contains all the data from the source at the time of cloning. - * The clone is completely independent - modifications to one do not affect the other. + * Lists the transactions that were durably prepared before the last shutdown + * and never committed or rolled back, so a coordinator can finish deciding + * them. One that was decided in the log is settled during open and never + * appears here. * - * @param sourceName the source column family name - * @param destName the destination column family name - * @throws TidesDBException if the clone fails + *

The caller owns every returned {@link Transaction} and must resolve it + * with {@link Transaction#commitPrepared()} or + * {@link Transaction#rollbackPrepared()}, then free it. + * + * @return the in-doubt transactions, never {@code null} + * @throws IllegalStateException if this database is closed + * @throws TidesDBException if the recovery scan fails */ - public void cloneColumnFamily(String sourceName, String destName) throws TidesDBException { + public PreparedTransaction[] recoverPrepared() throws TidesDBException { checkNotClosed(); - if (sourceName == null || sourceName.isEmpty()) { - throw new IllegalArgumentException("Source column family name cannot be null or empty"); - } - if (destName == null || destName.isEmpty()) { - throw new IllegalArgumentException("Destination column family name cannot be null or empty"); - } - nativeCloneColumnFamily(nativeHandle, sourceName, destName); + return nativeRecoverPrepared(nativeHandle); + } + + /* ===== maintenance ===== */ + + /** + * Synchronously rotates and flushes the shared memtable to SSTables, waiting + * a bounded time for the immutable queue to drain. + * + * @throws IllegalStateException if this database is closed + * @throws TidesDBException with {@link TidesDBException#ERR_LOCKED} if the + * queue had not drained when the wait expired, which says flush is + * not keeping up rather than that anything is wrong with the call + */ + public void flushMemtable() throws TidesDBException { + checkNotClosed(); + nativeFlushMemtable(nativeHandle); } - + /** - * Forces a synchronous flush and aggressive compaction for all column families, - * then drains both the global flush and compaction queues. - * This blocks until all work is complete. + * Reports whether the memtable is currently flushing or rotating. * - * @throws TidesDBException if the purge fails + * @return {@code true} if flushing + * @throws IllegalStateException if this database is closed */ - public void purge() throws TidesDBException { + public boolean isFlushing() { checkNotClosed(); - nativePurge(nativeHandle); + return nativeIsFlushing(nativeHandle); } - + /** - * Deletes a column family using its handle. - * This is an alternative to {@link #dropColumnFamily(String)} that takes a column family - * object instead of a name. + * Forces an fsync of the write-ahead log. Useful for explicit durability + * control under {@link SyncMode#SYNC_NONE} or {@link SyncMode#SYNC_INTERVAL}. * - * @param cf the column family to delete - * @throws TidesDBException if the column family cannot be deleted + * @throws IllegalStateException if this database is closed + * @throws TidesDBException if the sync fails */ - public void deleteColumnFamily(ColumnFamily cf) throws TidesDBException { + public void syncWal() throws TidesDBException { checkNotClosed(); - if (cf == null) { - throw new IllegalArgumentException("Column family cannot be null"); + nativeSyncWal(nativeHandle); + } + + /** + * Writes a consistent, directly-openable copy of the database into + * {@code dir}: flushes the memtable, then copies the manifest, the shared + * value log, and every SSTable it references at a single manifest snapshot + * while compaction is held off, so the copy references no file that a merge + * could delete mid-copy. + * + * @param dir the destination directory, created if absent; must not be + * {@code null} or empty + * @throws IllegalArgumentException if {@code dir} is {@code null} or empty + * @throws IllegalStateException if this database is closed + * @throws TidesDBException if the backup fails, including + * {@link TidesDBException#ERR_LOCKED} if a family stayed under + * compaction for the whole freeze window + */ + public void backup(String dir) throws TidesDBException { + checkNotClosed(); + if (dir == null || dir.isEmpty()) { + throw new IllegalArgumentException("Backup directory cannot be null or empty"); } - nativeDeleteColumnFamily(nativeHandle, cf.getNativeHandle()); + nativeBackup(nativeHandle, dir); } /** - * Switches a read-only replica database to primary mode. + * Establishes a durability barrier in the live database: flushes the memtable + * to L1, then forces the value log, the write-ahead log, and the manifest to + * disk regardless of the configured sync mode. * - * @throws TidesDBException if not in replica mode or promotion fails + * @throws IllegalStateException if this database is closed + * @throws TidesDBException if the checkpoint fails, including + * {@link TidesDBException#ERR_LOCKED} if the flush it begins with + * could not drain the immutable queue */ - public void promoteToPrimary() throws TidesDBException { + public void checkpoint() throws TidesDBException { checkNotClosed(); - nativePromoteToPrimary(nativeHandle); + nativeCheckpoint(nativeHandle); } + /* ===== statistics ===== */ + /** - * Retrieves aggregate statistics across the entire database instance. + * Collects database-level statistics. * - * @return database-level statistics - * @throws TidesDBException if the stats cannot be retrieved + * @return the statistics, never {@code null} + * @throws IllegalStateException if this database is closed + * @throws TidesDBException if the native stats retrieval fails */ public DbStats getDbStats() throws TidesDBException { checkNotClosed(); @@ -456,36 +572,70 @@ public DbStats getDbStats() throws TidesDBException { } /** - * Cancels background compaction database-wide. In-flight merges bail safely at their - * next checkpoint (their uncommitted output is discarded, inputs are left intact, so no - * data is lost) and any queued compaction is skipped. Flushes are unaffected, so - * durability is preserved. Blocks (bounded) until compaction is idle. + * Collects block-cache statistics. * - *

The cancellation is sticky for the session and is reset on the next open. It is - * intended to be called immediately before {@link #close()} for a fast shutdown.

+ * @return the statistics, never {@code null} + * @throws IllegalStateException if this database is closed + * @throws TidesDBException if the native stats retrieval fails + */ + public CacheStats getCacheStats() throws TidesDBException { + checkNotClosed(); + return nativeGetCacheStats(nativeHandle); + } + + /** + * Collects where writers have been made to wait. A write latency tail is + * answerable from this alone: compare each reason's maximum against the tail + * you measured, and its total against the others. * - * @throws TidesDBException if the operation fails + * @return the statistics, never {@code null} + * @throws IllegalStateException if this database is closed + * @throws TidesDBException if the native stats retrieval fails */ - public void cancelBackgroundWork() throws TidesDBException { + public StallStats getStallStats() throws TidesDBException { checkNotClosed(); - nativeCancelBackgroundWork(nativeHandle); + return nativeGetStallStats(nativeHandle); } /** - * Raises this process's open-file ceiling toward {@code desired} descriptors so a database - * can keep more SSTables open. The engine sizes {@code maxOpenSSTables} to fit this at open - * time, so call it before {@link #open(Config)}. This is an explicit, opt-in - * operator action; TidesDB never raises the limit itself. + * Collects what each class of file asked of the device. This is the other + * half of {@link #getStallStats()}: that says writers waited on the log, this + * says whether the device was the reason. * - *

On POSIX systems (Linux, macOS, the BSDs, illumos) this raises the {@code RLIMIT_NOFILE} - * soft limit toward the hard limit; on Windows it raises the CRT stdio cap (max 8192). A - * failed or partial raise is non-fatal.

+ * @return the statistics, never {@code null} + * @throws IllegalStateException if this database is closed + * @throws TidesDBException if the native stats retrieval fails + */ + public IoStats getIoStats() throws TidesDBException { + checkNotClosed(); + return nativeGetIoStats(nativeHandle); + } + + /** + * Reports what each encoding chain achieved on the key logs, one entry per + * chain the live SSTables were written with. A table written before a family + * changed its codec keeps reporting what its own pipeline achieved. * - * @param desired target descriptor count; values ≤ 0 just report the current ceiling - * @return the open-file ceiling in effect after the attempt + * @return the per-chain statistics, never {@code null} + * @throws IllegalStateException if this database is closed + * @throws TidesDBException if the native stats retrieval fails */ - public static long raiseOpenFileLimit(long desired) { - return nativeRaiseOpenFileLimit(desired); + public EncodingStats[] getKlogEncodingStats() throws TidesDBException { + checkNotClosed(); + return nativeGetKlogEncodingStats(nativeHandle); + } + + /** + * Reports what each encoding chain achieved on the separated values, read + * back from the chain each value records with itself. + * + * @return the per-chain statistics, never {@code null} + * @throws IllegalStateException if this database is closed + * @throws TidesDBException if the native stats retrieval fails + */ + public EncodingStats[] getVlogEncodingStats() throws TidesDBException { + checkNotClosed(); + return nativeGetVlogEncodingStats(nativeHandle); } private void checkNotClosed() { @@ -493,14 +643,14 @@ private void checkNotClosed() { throw new IllegalStateException("TidesDB instance is closed"); } } - + /** * Reports whether this database instance has been closed. */ boolean isClosed() { return closed; } - + /** * Registers a column family that has an installed commit hook. */ @@ -509,7 +659,7 @@ void registerHookColumnFamily(ColumnFamily cf) { if (!closed) hookBearingCFs.add(cf); } } - + /** * Unregisters a column family whose commit hook has been cleared. */ @@ -518,88 +668,83 @@ void unregisterHookColumnFamily(ColumnFamily cf) { hookBearingCFs.remove(cf); } } - + long getNativeHandle() { return nativeHandle; } - - private static native long nativeOpen(String dbPath, int numFlushThreads, int numCompactionThreads, - int logLevel, long blockCacheSize, long maxOpenSSTables, + + private static native long nativeOpen(String dbPath, int numFlushThreads, + int numCompactionThreads, int logLevel, + long blockCacheSize, long maxOpenSSTables, boolean logToFile, long logTruncationAt, - long maxMemoryUsage, boolean unifiedMemtable, - long unifiedMemtableWriteBufferSize, - int unifiedMemtableSkipListMaxLevel, - float unifiedMemtableSkipListProbability, - int unifiedMemtableSyncMode, - long unifiedMemtableSyncIntervalUs, - String objectStoreFsPath, - String oscLocalCachePath, long oscLocalCacheMaxBytes, - boolean oscCacheOnRead, boolean oscCacheOnWrite, - int oscMaxConcurrentUploads, int oscMaxConcurrentDownloads, - long oscMultipartThreshold, long oscMultipartPartSize, - boolean oscSyncManifestToObject, boolean oscReplicateWal, - boolean oscWalUploadSync, long oscWalSyncThresholdBytes, - boolean oscWalSyncOnCommit, boolean oscReplicaMode, - long oscReplicaSyncIntervalUs, - boolean oscReplicaReplayWal, - int maxConcurrentFlushes, - boolean finishCompactionsOnClose, - long objStoreHandle) throws TidesDBException; - - private static native long nativeObjstoreS3Create(String endpoint, String bucket, String prefix, - String accessKey, String secretKey, String region, - boolean useSsl, boolean usePathStyle, - String tlsCaPath, boolean tlsInsecureSkipVerify, - long multipartThreshold, long multipartPartSize) - throws TidesDBException; - - private static native boolean nativeS3Available(); + long memtableWriteBufferSize, + int memtableSkipListMaxLevel, + float memtableSkipListProbability, + int memtableSyncMode, long memtableSyncIntervalUs, + long valueSeparationThreshold, long vlogSegmentSize, + int memtableL0QueueStallThreshold, + int memtableIdleFlushSeconds, + long txnTimeoutSeconds) throws TidesDBException; private static native void nativeClose(long handle); - + + private static native boolean nativeCompressionAvailable(int algorithm); + + private static native String nativeStrerror(int code); + + private static native long nativeRaiseOpenFileLimit(long desired); + private static native void nativeCreateColumnFamily(long handle, String name, - long writeBufferSize, long levelSizeRatio, int minLevels, int dividingLevelOffset, - long klogValueThreshold, int compressionAlgorithm, boolean enableBloomFilter, - double bloomFPR, boolean enableBlockIndexes, int indexSampleRatio, int blockIndexPrefixLen, - int syncMode, long syncIntervalUs, String comparatorName, int skipListMaxLevel, - float skipListProbability, int defaultIsolationLevel, long minDiskSpace, - int l1FileCountTrigger, int l0QueueStallThreshold, - double tombstoneDensityTrigger, long tombstoneDensityMinEntries, - boolean useBtree, - boolean objectLazyCompaction, - boolean objectPrefetchCompaction) throws TidesDBException; - + long levelSizeRatio, int minLevels, int dividingLevelOffset, boolean keepValuesInline, + long btreeKlogBlockSize, int[] encodingPipeline, boolean enableBloomFilter, double bloomFpr, + int defaultIsolationLevel, int l1FileCountTrigger, double tombstoneDensityTrigger, + long tombstoneDensityMinEntries) throws TidesDBException; + private static native void nativeDropColumnFamily(long handle, String name) throws TidesDBException; - + + private static native void nativeRenameColumnFamily(long handle, String oldName, String newName) throws TidesDBException; + + private static native void nativeCloneColumnFamily(long handle, String sourceName, String destName) throws TidesDBException; + private static native long nativeGetColumnFamily(long handle, String name) throws TidesDBException; - + private static native String[] nativeListColumnFamilies(long handle) throws TidesDBException; - + private static native long nativeBeginTransaction(long handle) throws TidesDBException; - + private static native long nativeBeginTransactionWithIsolation(long handle, int isolationLevel) throws TidesDBException; - - private static native CacheStats nativeGetCacheStats(long handle) throws TidesDBException; - - private static native void nativeRegisterComparator(long handle, String name, String context) throws TidesDBException; - + + private static native long nativeBeginTransactionCf(long handle, long cfHandle) throws TidesDBException; + + private static native long nativeSnapshotCreate(long handle) throws TidesDBException; + + private static native long nativeBeginTransactionAtSnapshot(long handle, long snapshotHandle) throws TidesDBException; + + private static native long nativeBeginTransactionAtSeq(long handle, long seq) throws TidesDBException; + + private static native long nativeOldestReadableSeq(long handle); + + private static native PreparedTransaction[] nativeRecoverPrepared(long handle) throws TidesDBException; + + private static native void nativeFlushMemtable(long handle) throws TidesDBException; + + private static native boolean nativeIsFlushing(long handle); + + private static native void nativeSyncWal(long handle) throws TidesDBException; + private static native void nativeBackup(long handle, String dir) throws TidesDBException; - - private static native void nativeCheckpoint(long handle, String dir) throws TidesDBException; - - private static native void nativeRenameColumnFamily(long handle, String oldName, String newName) throws TidesDBException; - - private static native void nativeCloneColumnFamily(long handle, String sourceName, String destName) throws TidesDBException; - - private static native void nativePurge(long handle) throws TidesDBException; + + private static native void nativeCheckpoint(long handle) throws TidesDBException; private static native DbStats nativeGetDbStats(long handle) throws TidesDBException; - private static native void nativeDeleteColumnFamily(long handle, long cfHandle) throws TidesDBException; + private static native CacheStats nativeGetCacheStats(long handle) throws TidesDBException; - private static native void nativePromoteToPrimary(long handle) throws TidesDBException; + private static native StallStats nativeGetStallStats(long handle) throws TidesDBException; - private static native void nativeCancelBackgroundWork(long handle) throws TidesDBException; + private static native IoStats nativeGetIoStats(long handle) throws TidesDBException; - private static native long nativeRaiseOpenFileLimit(long desired); + private static native EncodingStats[] nativeGetKlogEncodingStats(long handle) throws TidesDBException; + + private static native EncodingStats[] nativeGetVlogEncodingStats(long handle) throws TidesDBException; } diff --git a/src/main/java/com/tidesdb/TidesDBException.java b/src/main/java/com/tidesdb/TidesDBException.java index 3cfaf03..2128861 100644 --- a/src/main/java/com/tidesdb/TidesDBException.java +++ b/src/main/java/com/tidesdb/TidesDBException.java @@ -23,14 +23,21 @@ * an integer error code corresponding to a TidesDB status. Use {@link #getErrorCode()} * to retrieve the numeric code and {@link #getErrorMessage()} for a human-readable * description. + * + *

Two codes describe conditions a caller is expected to handle rather than + * report: {@link #ERR_LOCKED} is transient contention and the remedy is always + * to retry, and {@link #ERR_CONFLICT} is the engine's first-committer-wins + * verdict on a transaction. {@link #isRetryable()} identifies the former. */ public class TidesDBException extends Exception { - + + private static final long serialVersionUID = 1L; + /** * The error code. */ private final int errorCode; - + /** * Error codes returned by the native TidesDB library. */ @@ -67,12 +74,14 @@ public class TidesDBException extends Exception { public static final int ERR_EXISTS = -6; /** - * Transaction conflict. + * Transaction conflict. The engine's own first-committer-wins verdict: + * another transaction committed a conflicting write first. */ public static final int ERR_CONFLICT = -7; /** - * Key or value exceeds size limits. + * A value does not fit the space that has to hold it, such as a caller's + * buffer too small for what was asked of it. */ public static final int ERR_TOO_LARGE = -8; @@ -82,7 +91,7 @@ public class TidesDBException extends Exception { public static final int ERR_MEMORY_LIMIT = -9; /** - * Invalid database handle. + * Invalid database handle, or the database is closing. */ public static final int ERR_INVALID_DB = -10; @@ -92,10 +101,45 @@ public class TidesDBException extends Exception { public static final int ERR_UNKNOWN = -11; /** - * Database is locked by another process. + * Transient contention: something else held what the call needed, nothing + * was written, and the remedy is always to try again. It is not confined to + * operations that take a column family exclusively, since a read has to open + * SSTables and walk sources a compaction may be moving. Treat it as retry, + * never as absence and never as an error to surface. */ public static final int ERR_LOCKED = -12; - + + /** + * The database is read-only. + */ + public static final int ERR_READONLY = -13; + + /** + * The transaction's timeout has passed; the next operation on it expired it. + */ + public static final int ERR_TXN_EXPIRED = -14; + + /** + * The device or filesystem is out of space. Distinct from {@link #ERR_IO} + * because the data is intact and the operation succeeds once space is freed. + */ + public static final int ERR_NO_SPACE = -15; + + /** + * The transaction was aborted from outside through + * {@link Transaction#requestAbort()}. Distinct from {@link #ERR_CONFLICT}: + * a conflict is the engine's own verdict, where this says an outside + * authority decided the transaction loses and the engine never got a say. + */ + public static final int ERR_TXN_ABORTED = -16; + + /** + * The point in time asked for is no longer reconstructable: a collection has + * already run below that sequence, so the versions it would resolve to are + * gone. + */ + public static final int ERR_TOO_OLD = -17; + /** * Creates an exception with a message and an unknown error code. * @@ -105,7 +149,7 @@ public TidesDBException(String message) { super(message); this.errorCode = ERR_UNKNOWN; } - + /** * Creates an exception with a message and a specific error code. * @@ -116,7 +160,7 @@ public TidesDBException(String message, int errorCode) { super(message); this.errorCode = errorCode; } - + /** * Creates an exception with a message and a cause, using an unknown * error code. @@ -128,7 +172,7 @@ public TidesDBException(String message, Throwable cause) { super(message, cause); this.errorCode = ERR_UNKNOWN; } - + /** * Creates an exception with a message, error code, and cause. * @@ -140,7 +184,7 @@ public TidesDBException(String message, int errorCode, Throwable cause) { super(message, cause); this.errorCode = errorCode; } - + /** * Returns the TidesDB error code. * @@ -149,7 +193,18 @@ public TidesDBException(String message, int errorCode, Throwable cause) { public int getErrorCode() { return errorCode; } - + + /** + * Returns whether the operation failed to transient contention and should + * simply be retried. Nothing was written and nothing is wrong with the + * database. + * + * @return {@code true} when the error code is {@link #ERR_LOCKED} + */ + public boolean isRetryable() { + return errorCode == ERR_LOCKED; + } + /** * Returns a human-readable description of the error code. The mapping is * local to this class and does not invoke any native method. @@ -175,13 +230,23 @@ public String getErrorMessage() { case ERR_CONFLICT: return "transaction conflict"; case ERR_TOO_LARGE: - return "key or value too large"; + return "value too large for the space that must hold it"; case ERR_MEMORY_LIMIT: return "memory limit exceeded"; case ERR_INVALID_DB: return "invalid database handle"; case ERR_LOCKED: - return "database is locked"; + return "resource is locked, retry"; + case ERR_READONLY: + return "database is read-only"; + case ERR_TXN_EXPIRED: + return "transaction expired"; + case ERR_NO_SPACE: + return "no space left on device"; + case ERR_TXN_ABORTED: + return "transaction aborted by request"; + case ERR_TOO_OLD: + return "sequence is no longer reconstructable"; default: return "unknown error"; } diff --git a/src/main/java/com/tidesdb/TidesDBIterator.java b/src/main/java/com/tidesdb/TidesDBIterator.java index b750dfb..09ff6ce 100644 --- a/src/main/java/com/tidesdb/TidesDBIterator.java +++ b/src/main/java/com/tidesdb/TidesDBIterator.java @@ -30,6 +30,12 @@ * and {@code close()} throw {@link IllegalStateException}. The {@link #isValid()} * method returns {@code false} on a freed iterator. * + *

A scan reaches the same SSTables a point read does, so every method here + * can fail with {@link TidesDBException#ERR_LOCKED} for the same reason and with + * the same remedy: the position did not move, nothing is wrong with the + * iterator, and the step should be retried. That is not the end of the range, + * which is what {@link #isValid()} reports. + * *

This class is not guaranteed to be thread-safe. */ public class TidesDBIterator implements Closeable { diff --git a/src/main/java/com/tidesdb/Transaction.java b/src/main/java/com/tidesdb/Transaction.java index 58bfff8..60494e4 100644 --- a/src/main/java/com/tidesdb/Transaction.java +++ b/src/main/java/com/tidesdb/Transaction.java @@ -21,62 +21,77 @@ import java.io.Closeable; /** - * Represents a transaction in TidesDB. Transactions provide atomic - * operations on the database and implement {@link java.io.Closeable} for - * use with try-with-resources. + * A transaction in TidesDB. Every read and write goes through one, and + * {@code Transaction} implements {@link java.io.Closeable} for use with + * try-with-resources. * - *

Close transactions before closing the owning {@link TidesDB} instance. - * After this transaction is freed, all operations except {@code close()} throw - * {@link IllegalStateException}. + *

Close transactions before closing the owning {@link TidesDB} instance, and + * before releasing any {@link Snapshot} they were opened against. After this + * transaction is freed, all operations except {@link #requestAbort()} and + * {@code close()} throw {@link IllegalStateException}. * - *

This class is not guaranteed to be thread-safe. + *

This class is not guaranteed to be thread-safe. The one exception is + * {@link #requestAbort()}, which may be called from a thread other than the one + * running the transaction. */ public class Transaction implements Closeable { - + static { NativeLibrary.load(); } - + + /** + * The longest bound {@link #deleteRange(ColumnFamily, byte[], byte[])} and + * {@link #deletePrefix(ColumnFamily, byte[])} accept. An interval delete + * holds its bounds in a fixed slot for the length of its commit, which is + * what stops a concurrent write landing inside the range; a range too wide to + * name in one call is expressed as several. + */ + public static final int MAX_RANGE_BOUND_SIZE = 256; + private long nativeHandle; - private boolean freed = false; - + private volatile boolean freed = false; + Transaction(long nativeHandle) { this.nativeHandle = nativeHandle; } - + + /* ===== writes ===== */ + /** - * Adds a key-value pair to the transaction. + * Buffers a put into this transaction. * - * @param cf the column family; must not be {@code null} + * @param cf the target column family; must not be {@code null} * @param key the key; must not be {@code null} or empty - * @param value the value; must not be {@code null} - * @param ttl expiration as seconds since the Unix epoch, or {@code -1} for - * no expiration + * @param value the value; must not be {@code null}. It may be empty, which + * stores the key present carrying nothing — a distinct state from an + * absence, since a read returns it with a zero length rather than + * reporting the key missing + * @param ttlSeconds how long the entry lives, in seconds from + * now; zero or negative never expires. The engine converts it + * to an absolute deadline once, here at the boundary, so a long + * recovery cannot extend an entry's life * @throws IllegalArgumentException if {@code cf} is {@code null}, {@code key} * is {@code null} or empty, or {@code value} is {@code null} * @throws IllegalStateException if this transaction is freed * @throws TidesDBException if the native put fails */ - public void put(ColumnFamily cf, byte[] key, byte[] value, long ttl) throws TidesDBException { + public void put(ColumnFamily cf, byte[] key, byte[] value, long ttlSeconds) + throws TidesDBException { checkNotFreed(); - if (cf == null) { - throw new IllegalArgumentException("Column family cannot be null"); - } - if (key == null || key.length == 0) { - throw new IllegalArgumentException("Key cannot be null or empty"); - } + requireCf(cf); + requireKey(key); if (value == null) { throw new IllegalArgumentException("Value cannot be null"); } - nativePut(nativeHandle, cf.getNativeHandle(), key, value, ttl); + nativePut(nativeHandle, cf.getNativeHandle(), key, value, ttlSeconds); } - + /** - * Adds a key-value pair to the transaction with no expiration. + * Buffers a put with no expiration. Equivalent to + * {@code put(cf, key, value, 0)}. * - *

Equivalent to {@code put(cf, key, value, -1)}. - * - * @param cf the column family; must not be {@code null} + * @param cf the target column family; must not be {@code null} * @param key the key; must not be {@code null} or empty * @param value the value; must not be {@code null} * @throws IllegalArgumentException if {@code cf} is {@code null}, {@code key} @@ -85,154 +100,211 @@ public void put(ColumnFamily cf, byte[] key, byte[] value, long ttl) throws Tide * @throws TidesDBException if the native put fails */ public void put(ColumnFamily cf, byte[] key, byte[] value) throws TidesDBException { - put(cf, key, value, -1); + put(cf, key, value, 0); } - + /** - * Retrieves a value from the transaction. + * Buffers a delete (tombstone) into this transaction. * - * @param cf the column family; must not be {@code null} + * @param cf the target column family; must not be {@code null} * @param key the key; must not be {@code null} or empty - * @return the value, or {@code null} if not found * @throws IllegalArgumentException if {@code cf} is {@code null}, or * {@code key} is {@code null} or empty * @throws IllegalStateException if this transaction is freed - * @throws TidesDBException if the native get fails + * @throws TidesDBException if the native delete fails */ - public byte[] get(ColumnFamily cf, byte[] key) throws TidesDBException { + public void delete(ColumnFamily cf, byte[] key) throws TidesDBException { checkNotFreed(); - if (cf == null) { - throw new IllegalArgumentException("Column family cannot be null"); - } - if (key == null || key.length == 0) { - throw new IllegalArgumentException("Key cannot be null or empty"); - } - return nativeGet(nativeHandle, cf.getNativeHandle(), key); + requireCf(cf); + requireKey(key); + nativeDelete(nativeHandle, cf.getNativeHandle(), key); } - + /** - * Removes a key-value pair from the transaction. + * Buffers a single-delete: a tombstone the caller promises supersedes at most + * one put, so the two can be reaped together at compaction. * - * @param cf the column family; must not be {@code null} + *

Caller contract: between any two single-deletes on the same key, and + * from the start of the key's history to its first single-delete, the key has + * been put at most once. The engine cannot verify this; violating it can + * leave older puts visible after the single-delete. When in doubt, prefer + * {@link #delete(ColumnFamily, byte[])}. + * + * @param cf the target column family; must not be {@code null} * @param key the key; must not be {@code null} or empty * @throws IllegalArgumentException if {@code cf} is {@code null}, or * {@code key} is {@code null} or empty * @throws IllegalStateException if this transaction is freed - * @throws TidesDBException if the native delete fails + * @throws TidesDBException if the native single-delete fails */ - public void delete(ColumnFamily cf, byte[] key) throws TidesDBException { + public void singleDelete(ColumnFamily cf, byte[] key) throws TidesDBException { checkNotFreed(); - if (cf == null) { - throw new IllegalArgumentException("Column family cannot be null"); - } - if (key == null || key.length == 0) { - throw new IllegalArgumentException("Key cannot be null"); - } - nativeDelete(nativeHandle, cf.getNativeHandle(), key); + requireCf(cf); + requireKey(key); + nativeSingleDelete(nativeHandle, cf.getNativeHandle(), key); } /** - * Writes a single-delete tombstone for a key. Has the same read semantics as - * {@link #delete}, but lets compaction drop the put and tombstone together as - * soon as both appear in the same merge input. - * - * Caller contract: between any two single-deletes on the same key (and from - * the start of the key's history to its first single-delete) the key has been - * put at most once. The engine cannot verify this; violating it can leave - * older puts visible after the single-delete. Use only for workloads that - * insert each key once and delete it once. When in doubt, prefer {@link #delete}. - * - * @param cf the column family - * @param key the key - * @throws TidesDBException if the single-delete fails + * Buffers a delete of every key in the column family from {@code lo} + * (inclusive) to {@code hi} (exclusive). + * + *

It costs one entry however many keys it covers, and it deletes keys + * written before it as well as keys written after it — so it is not the same + * as deleting the keys that happen to be there when you call it. A write to a + * key inside the range survives it when it is newer: buffered after it in the + * same transaction, or committed at a later sequence. + * + *

The delete is O(1) to write but not to reclaim: the keys it covers stay + * on disk until a compaction rewrites the range. + * + *

At {@link IsolationLevel#SNAPSHOT} and above the commit is refused with + * {@link TidesDBException#ERR_CONFLICT} when any key in the range was written + * after this transaction drew its snapshot. + * + * @param cf the target column family; must not be {@code null} + * @param lo the inclusive lower bound; must not be {@code null} or empty, and + * at most {@link #MAX_RANGE_BOUND_SIZE} bytes. Keys are never empty, so + * a single zero byte is a lower bound below every key there can be + * @param hi the exclusive upper bound, or {@code null}/empty to run to the end + * of the family; at most {@link #MAX_RANGE_BOUND_SIZE} bytes + * @throws IllegalArgumentException if {@code cf} is {@code null}, {@code lo} + * is {@code null} or empty, or either bound is too long + * @throws IllegalStateException if this transaction is freed + * @throws TidesDBException if the native delete fails */ - public void singleDelete(ColumnFamily cf, byte[] key) throws TidesDBException { + public void deleteRange(ColumnFamily cf, byte[] lo, byte[] hi) throws TidesDBException { checkNotFreed(); - if (cf == null) { - throw new IllegalArgumentException("Column family cannot be null"); + requireCf(cf); + if (lo == null || lo.length == 0) { + throw new IllegalArgumentException("Lower bound cannot be null or empty"); } - if (key == null || key.length == 0) { - throw new IllegalArgumentException("Key cannot be null"); + requireBoundSize(lo, "Lower bound"); + if (hi != null) { + requireBoundSize(hi, "Upper bound"); } - nativeSingleDelete(nativeHandle, cf.getNativeHandle(), key); + nativeDeleteRange(nativeHandle, cf.getNativeHandle(), lo, hi); } /** - * Commits the transaction, making all pending operations durable. + * Buffers a delete of every key in the column family that begins with + * {@code prefix}. This is the one-bound form of + * {@link #deleteRange(ColumnFamily, byte[], byte[])} and carries the same + * semantics and costs. * + *

A two-phase transaction holds its prefix from the prepare until phase + * two resolves it, so a write under it is refused for as long as it stays in + * doubt. + * + * @param cf the target column family; must not be {@code null} + * @param prefix the prefix to delete under; must not be {@code null} or empty, + * and at most {@link #MAX_RANGE_BOUND_SIZE} bytes. A whole family is + * dropped with {@link TidesDB#dropColumnFamily(String)} rather than + * deleted a prefix at a time + * @throws IllegalArgumentException if {@code cf} is {@code null}, or + * {@code prefix} is {@code null}, empty, or too long * @throws IllegalStateException if this transaction is freed - * @throws TidesDBException if the native commit fails + * @throws TidesDBException if the native delete fails */ - public void commit() throws TidesDBException { + public void deletePrefix(ColumnFamily cf, byte[] prefix) throws TidesDBException { checkNotFreed(); - nativeCommit(nativeHandle); + requireCf(cf); + if (prefix == null || prefix.length == 0) { + throw new IllegalArgumentException("Prefix cannot be null or empty"); + } + requireBoundSize(prefix, "Prefix"); + nativeDeletePrefix(nativeHandle, cf.getNativeHandle(), prefix); } - + + /* ===== reads ===== */ + /** - * Rolls back all operations in the transaction. + * Reads a key at the transaction snapshot, recording the read into the + * conflict footprint. * + * @param cf the target column family; must not be {@code null} + * @param key the key; must not be {@code null} or empty + * @return the value, or {@code null} if not found + * @throws IllegalArgumentException if {@code cf} is {@code null}, or + * {@code key} is {@code null} or empty * @throws IllegalStateException if this transaction is freed - * @throws TidesDBException if the native rollback fails + * @throws TidesDBException if the native read fails, including + * {@link TidesDBException#ERR_LOCKED} when contention left the read + * unservable and it should be retried */ - public void rollback() throws TidesDBException { + public byte[] get(ColumnFamily cf, byte[] key) throws TidesDBException { checkNotFreed(); - nativeRollback(nativeHandle); + requireCf(cf); + requireKey(key); + return nativeGet(nativeHandle, cf.getNativeHandle(), key); } - + /** - * Creates a named savepoint within the transaction. + * Reads a key at the transaction snapshot without recording it into + * the conflict footprint, for existence probes — such as primary-key + * uniqueness — that should not pollute the write-write base. * - * @param name the savepoint name; must not be {@code null} or empty - * @throws IllegalArgumentException if {@code name} is {@code null} or empty + * @param cf the target column family; must not be {@code null} + * @param key the key; must not be {@code null} or empty + * @return the value, or {@code null} if not found + * @throws IllegalArgumentException if {@code cf} is {@code null}, or + * {@code key} is {@code null} or empty * @throws IllegalStateException if this transaction is freed - * @throws TidesDBException if the native savepoint creation fails + * @throws TidesDBException if the native read fails */ - public void savepoint(String name) throws TidesDBException { + public byte[] getNoTrack(ColumnFamily cf, byte[] key) throws TidesDBException { checkNotFreed(); - if (name == null || name.isEmpty()) { - throw new IllegalArgumentException("Savepoint name cannot be null or empty"); - } - nativeSavepoint(nativeHandle, name); + requireCf(cf); + requireKey(key); + return nativeGetNoTrack(nativeHandle, cf.getNativeHandle(), key); } - + /** - * Rolls back the transaction to a named savepoint. + * Non-tracking existence check at the transaction snapshot. * - * @param name the savepoint name; must not be {@code null} or empty - * @throws IllegalArgumentException if {@code name} is {@code null} or empty + * @param cf the target column family; must not be {@code null} + * @param key the key; must not be {@code null} or empty + * @return {@code true} if the key is present + * @throws IllegalArgumentException if {@code cf} is {@code null}, or + * {@code key} is {@code null} or empty * @throws IllegalStateException if this transaction is freed - * @throws TidesDBException if the native rollback fails + * @throws TidesDBException if the native check fails, including + * {@link TidesDBException#ERR_LOCKED} when contention left the read + * unservable and it should be retried */ - public void rollbackToSavepoint(String name) throws TidesDBException { + public boolean contains(ColumnFamily cf, byte[] key) throws TidesDBException { checkNotFreed(); - if (name == null || name.isEmpty()) { - throw new IllegalArgumentException("Savepoint name cannot be null or empty"); - } - nativeRollbackToSavepoint(nativeHandle, name); + requireCf(cf); + requireKey(key); + return nativeContains(nativeHandle, cf.getNativeHandle(), key); } - + /** - * Releases a named savepoint without rolling back. + * Returns the sequence ceiling this transaction's reads filter at, so a + * caller can reason about which committed versions the transaction can and + * cannot see. * - * @param name the savepoint name; must not be {@code null} or empty - * @throws IllegalArgumentException if {@code name} is {@code null} or empty + *

The engine's sequence is an unsigned 64-bit value, so a ceiling of + * {@code UINT64_MAX} arrives here as {@code -1}. Compare sequences with + * {@link Long#compareUnsigned(long, long)} rather than {@code <}. + * + * @return {@code -1} (an unsigned {@code UINT64_MAX}) for read-uncommitted, + * the current sequence for read-committed, and the sequence frozen at + * begin for repeatable-read and stronger * @throws IllegalStateException if this transaction is freed - * @throws TidesDBException if the native release fails */ - public void releaseSavepoint(String name) throws TidesDBException { + public long getReadSnapshot() { checkNotFreed(); - if (name == null || name.isEmpty()) { - throw new IllegalArgumentException("Savepoint name cannot be null or empty"); - } - nativeReleaseSavepoint(nativeHandle, name); + return nativeReadSnapshot(nativeHandle); } - + + /* ===== iterators ===== */ + /** - * Creates a new iterator for the given column family within this transaction. + * Creates an iterator over a column family at this transaction's snapshot. * *

Close the returned iterator before freeing this transaction. * - * @param cf the column family; must not be {@code null} + * @param cf the column family to iterate; must not be {@code null} * @return a new iterator * @throws IllegalArgumentException if {@code cf} is {@code null} * @throws IllegalStateException if this transaction is freed @@ -240,19 +312,137 @@ public void releaseSavepoint(String name) throws TidesDBException { */ public TidesDBIterator newIterator(ColumnFamily cf) throws TidesDBException { checkNotFreed(); - if (cf == null) { - throw new IllegalArgumentException("Column family cannot be null"); + requireCf(cf); + return new TidesDBIterator(nativeNewIterator(nativeHandle, cf.getNativeHandle())); + } + + /** + * Creates an iterator over the part of a column family a scan will actually + * read, at this transaction's snapshot. + * + *

An iterator holds one open cursor per SSTable that could answer it. + * Telling it the range up front lets it leave out the SSTables whose own key + * range cannot meet it, so a scan of a narrow band costs what that band costs + * rather than what the whole column family costs. + * + *

The range is a promise about what will be read, not a fence the iterator + * enforces. Results are defined only inside it, because the SSTables that + * could answer outside it were never opened: seeking or stepping past either + * end may report absent a key that exists. Use + * {@link #newIterator(ColumnFamily)} for a scan whose extent is not known in + * advance. + * + * @param cf the column family to iterate; must not be {@code null} + * @param lower the range start, inclusive; must not be {@code null} or empty + * @param upper the range end, inclusive for the purpose of choosing SSTables, + * so an exclusive end may be passed unchanged; must not be {@code null} + * or empty + * @return a new iterator + * @throws IllegalArgumentException if {@code cf} is {@code null}, or either + * bound is {@code null} or empty + * @throws IllegalStateException if this transaction is freed + * @throws TidesDBException if the native iterator cannot be created + */ + public TidesDBIterator newRangeIterator(ColumnFamily cf, byte[] lower, byte[] upper) + throws TidesDBException { + checkNotFreed(); + requireCf(cf); + if (lower == null || lower.length == 0) { + throw new IllegalArgumentException("Lower bound cannot be null or empty"); + } + if (upper == null || upper.length == 0) { + throw new IllegalArgumentException("Upper bound cannot be null or empty"); } - long iterHandle = nativeNewIterator(nativeHandle, cf.getNativeHandle()); - return new TidesDBIterator(iterHandle); + return new TidesDBIterator( + nativeNewRangeIterator(nativeHandle, cf.getNativeHandle(), lower, upper)); } - + + /* ===== lifecycle ===== */ + /** - * Resets a committed or aborted transaction for reuse with a new isolation level. - * This avoids the overhead of freeing and reallocating transaction resources in hot loops. + * Commits the transaction, writing its batch to the write-ahead log and + * memtable. * - * @param isolation the new isolation level for the reset transaction - * @throws TidesDBException if the reset fails (e.g., transaction is still active) + * @throws IllegalStateException if this transaction is freed + * @throws TidesDBException with {@link TidesDBException#ERR_CONFLICT} when + * another transaction committed a conflicting write first, or + * {@link TidesDBException#ERR_TXN_ABORTED} when an outside authority + * aborted it through {@link #requestAbort()} + */ + public void commit() throws TidesDBException { + checkNotFreed(); + nativeCommit(nativeHandle); + } + + /** + * Discards the transaction's buffered writes without committing. + * + * @throws IllegalStateException if this transaction is freed + * @throws TidesDBException if the native rollback fails + */ + public void rollback() throws TidesDBException { + checkNotFreed(); + nativeRollback(nativeHandle); + } + + /** + * Bounds how long this transaction may stay active, overriding the database's + * {@link Config#getTxnTimeoutSeconds()} for this one. The deadline is + * measured from the moment of this call, so calling it again on a live + * transaction extends it. + * + *

The transaction is not aborted in the background when the deadline + * passes: the next operation on it notices, aborts it, and fails with + * {@link TidesDBException#ERR_TXN_EXPIRED}. + * + * @param seconds seconds from now at which it expires, or ≤ 0 to clear any + * timeout + * @throws IllegalStateException if this transaction is freed + * @throws TidesDBException if the transaction is already resolved + */ + public void setTimeout(long seconds) throws TidesDBException { + checkNotFreed(); + nativeSetTimeout(nativeHandle, seconds); + } + + /** + * Aborts a transaction that another thread is running, so its next operation + * fails with {@link TidesDBException#ERR_TXN_ABORTED} rather than whatever it + * would otherwise have done. This is the one method here that may be called + * on a transaction owned by a different thread. + * + *

It exists for a caller that is itself the authority on whether a + * transaction may proceed. The call stores a flag and does nothing else: the + * transaction is not rolled back here, and the thread running it observes the + * flag when it next enters an operation and frees the transaction as it + * normally would. + * + *

A request that lands just after the running thread has checked lets the + * operation in progress finish and stops the one after it. + * + *

A transaction that has already prepared is not affected: it has voted, + * and only the coordinator's decision may resolve it. + * + *

Calling this on a freed transaction is a no-op. + */ + public void requestAbort() { + long handle = nativeHandle; + if (freed || handle == 0) { + return; + } + nativeRequestAbort(handle); + } + + /** + * Resets this transaction for reuse at a new isolation level, discarding its + * buffered state. This avoids the cost of freeing and reallocating + * transaction resources in hot loops. + * + * @param isolation the isolation level for the reset transaction; must not be + * {@code null} + * @throws IllegalArgumentException if {@code isolation} is {@code null} + * @throws IllegalStateException if this transaction is freed + * @throws TidesDBException if the reset fails */ public void reset(IsolationLevel isolation) throws TidesDBException { checkNotFreed(); @@ -261,21 +451,138 @@ public void reset(IsolationLevel isolation) throws TidesDBException { } nativeReset(nativeHandle, isolation.getValue()); } - + + /** + * Reports this transaction's lifecycle state, so a coordinator can tell a + * prepared transaction from a resolved one. + * + * @return the current state + * @throws IllegalStateException if this transaction is freed + * @throws TidesDBException if the native call fails + */ + public TransactionState state() throws TidesDBException { + checkNotFreed(); + return TransactionState.fromValue(nativeState(nativeHandle)); + } + + /* ===== two-phase commit ===== */ + + /** + * Two-phase-commit phase one: runs the same conflict checks as + * {@link #commit()} and durably logs the write batch under the given + * transaction id, but leaves the writes invisible and unapplied so a + * coordinator can gather votes from every participant before deciding. + * + *

On success the transaction moves to {@link TransactionState#PREPARED} + * and holds its snapshot and reservations until resolved with + * {@link #commitPrepared()} or {@link #rollbackPrepared()}. A read-only + * transaction prepares with nothing durable and needs no phase two. The xid + * is copied. + * + * @param xid the transaction id to record durably; must not be {@code null} + * or empty + * @throws IllegalArgumentException if {@code xid} is {@code null} or empty + * @throws IllegalStateException if this transaction is freed + * @throws TidesDBException if the prepare fails + */ + public void prepare(byte[] xid) throws TidesDBException { + checkNotFreed(); + if (xid == null || xid.length == 0) { + throw new IllegalArgumentException("Transaction id cannot be null or empty"); + } + nativePrepare(nativeHandle, xid); + } + + /** + * Two-phase-commit phase two: durably logs the decision to commit the + * prepared transaction, then applies its batch and makes it visible. Valid + * only on a prepared transaction. A transient I/O failure leaves it prepared + * so the coordinator can retry. + * + * @throws IllegalStateException if this transaction is freed + * @throws TidesDBException with {@link TidesDBException#ERR_INVALID_ARGS} if + * the transaction is not prepared + */ + public void commitPrepared() throws TidesDBException { + checkNotFreed(); + nativeCommitPrepared(nativeHandle); + } + /** - * Frees the transaction and releases all native resources. + * Two-phase-commit phase two: durably logs the decision to roll back the + * prepared transaction and releases its reservations. Nothing was applied, so + * nothing is undone. Valid only on a prepared transaction. * - *

This method is idempotent; subsequent calls are no-ops. After freeing, - * all other operations on this instance throw {@link IllegalStateException}. + * @throws IllegalStateException if this transaction is freed + * @throws TidesDBException with {@link TidesDBException#ERR_INVALID_ARGS} if + * the transaction is not prepared + */ + public void rollbackPrepared() throws TidesDBException { + checkNotFreed(); + nativeRollbackPrepared(nativeHandle); + } + + /* ===== savepoints ===== */ + + /** + * Marks a named savepoint in this transaction to roll back to later. + * + * @param name the savepoint name; must not be {@code null} or empty + * @throws IllegalArgumentException if {@code name} is {@code null} or empty + * @throws IllegalStateException if this transaction is freed + * @throws TidesDBException if the native savepoint creation fails + */ + public void savepoint(String name) throws TidesDBException { + checkNotFreed(); + requireName(name); + nativeSavepoint(nativeHandle, name); + } + + /** + * Discards writes buffered since a named savepoint. + * + * @param name the savepoint name; must not be {@code null} or empty + * @throws IllegalArgumentException if {@code name} is {@code null} or empty + * @throws IllegalStateException if this transaction is freed + * @throws TidesDBException with {@link TidesDBException#ERR_NOT_FOUND} when + * no savepoint carries that name + */ + public void rollbackToSavepoint(String name) throws TidesDBException { + checkNotFreed(); + requireName(name); + nativeRollbackToSavepoint(nativeHandle, name); + } + + /** + * Releases a named savepoint without rolling back. + * + * @param name the savepoint name; must not be {@code null} or empty + * @throws IllegalArgumentException if {@code name} is {@code null} or empty + * @throws IllegalStateException if this transaction is freed + * @throws TidesDBException with {@link TidesDBException#ERR_NOT_FOUND} when + * no savepoint carries that name + */ + public void releaseSavepoint(String name) throws TidesDBException { + checkNotFreed(); + requireName(name); + nativeReleaseSavepoint(nativeHandle, name); + } + + /** + * Frees the transaction, rolling it back if still open, and releases all + * native resources. + * + *

This method is idempotent; subsequent calls are no-ops. */ public void free() { if (!freed && nativeHandle != 0) { - nativeFree(nativeHandle); - nativeHandle = 0; + long handle = nativeHandle; freed = true; + nativeHandle = 0; + nativeFree(handle); } } - + /** * Closes this transaction. Equivalent to {@link #free()}. */ @@ -283,27 +590,90 @@ public void free() { public void close() { free(); } - + private void checkNotFreed() { if (freed) { throw new IllegalStateException("Transaction has been freed"); } } - + + private static void requireCf(ColumnFamily cf) { + if (cf == null) { + throw new IllegalArgumentException("Column family cannot be null"); + } + cf.checkOwnerOpen(); + } + + private static void requireKey(byte[] key) { + if (key == null || key.length == 0) { + throw new IllegalArgumentException("Key cannot be null or empty"); + } + } + + private static void requireName(String name) { + if (name == null || name.isEmpty()) { + throw new IllegalArgumentException("Savepoint name cannot be null or empty"); + } + } + + private static void requireBoundSize(byte[] bound, String what) { + if (bound.length > MAX_RANGE_BOUND_SIZE) { + throw new IllegalArgumentException( + what + " must be at most " + MAX_RANGE_BOUND_SIZE + " bytes, was: " + bound.length); + } + } + long getNativeHandle() { return nativeHandle; } - - private static native void nativePut(long handle, long cfHandle, byte[] key, byte[] value, long ttl) throws TidesDBException; + + private static native void nativePut(long handle, long cfHandle, byte[] key, byte[] value, + long ttlSeconds) throws TidesDBException; + private static native byte[] nativeGet(long handle, long cfHandle, byte[] key) throws TidesDBException; + + private static native byte[] nativeGetNoTrack(long handle, long cfHandle, byte[] key) throws TidesDBException; + + private static native boolean nativeContains(long handle, long cfHandle, byte[] key) throws TidesDBException; + + private static native long nativeReadSnapshot(long handle); + private static native void nativeDelete(long handle, long cfHandle, byte[] key) throws TidesDBException; + private static native void nativeSingleDelete(long handle, long cfHandle, byte[] key) throws TidesDBException; + + private static native void nativeDeleteRange(long handle, long cfHandle, byte[] lo, byte[] hi) throws TidesDBException; + + private static native void nativeDeletePrefix(long handle, long cfHandle, byte[] prefix) throws TidesDBException; + + private static native long nativeNewIterator(long handle, long cfHandle) throws TidesDBException; + + private static native long nativeNewRangeIterator(long handle, long cfHandle, byte[] lower, + byte[] upper) throws TidesDBException; + private static native void nativeCommit(long handle) throws TidesDBException; + private static native void nativeRollback(long handle) throws TidesDBException; + + private static native void nativeSetTimeout(long handle, long seconds) throws TidesDBException; + + private static native void nativeRequestAbort(long handle); + + private static native void nativeReset(long handle, int isolationLevel) throws TidesDBException; + + private static native int nativeState(long handle) throws TidesDBException; + + private static native void nativePrepare(long handle, byte[] xid) throws TidesDBException; + + private static native void nativeCommitPrepared(long handle) throws TidesDBException; + + private static native void nativeRollbackPrepared(long handle) throws TidesDBException; + private static native void nativeSavepoint(long handle, String name) throws TidesDBException; + private static native void nativeRollbackToSavepoint(long handle, String name) throws TidesDBException; + private static native void nativeReleaseSavepoint(long handle, String name) throws TidesDBException; - private static native long nativeNewIterator(long handle, long cfHandle) throws TidesDBException; - private static native void nativeReset(long handle, int isolationLevel) throws TidesDBException; + private static native void nativeFree(long handle); } diff --git a/src/main/java/com/tidesdb/TransactionState.java b/src/main/java/com/tidesdb/TransactionState.java new file mode 100644 index 0000000..1c04541 --- /dev/null +++ b/src/main/java/com/tidesdb/TransactionState.java @@ -0,0 +1,80 @@ +/** + * + * Copyright (C) TidesDB + * + * Original Author: Alex Gaetano Padula + * + * Licensed under the Mozilla Public License, v. 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.mozilla.org/en-US/MPL/2.0/ + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.tidesdb; + +/** + * Transaction lifecycle state, reported by {@link Transaction#state()} for + * two-phase-commit coordination. Each constant maps to an integer used by the + * JNI bridge. + */ +public enum TransactionState { + + /** + * Buffering writes, not yet resolved. + */ + ACTIVE(0), + + /** + * Durably prepared under an xid, awaiting commit or rollback. + */ + PREPARED(1), + + /** + * Committed and applied. + */ + COMMITTED(2), + + /** + * Rolled back or expired. + */ + ABORTED(3); + + private final int value; + + TransactionState(int value) { + this.value = value; + } + + /** + * Returns the JNI numeric mapping for this transaction state. + * + * @return the integer value passed to the native library + */ + public int getValue() { + return value; + } + + /** + * Returns the {@link TransactionState} constant matching the given JNI + * integer value. + * + * @param value the JNI integer value + * @return the matching constant + * @throws IllegalArgumentException if {@code value} does not map to any + * known constant + */ + public static TransactionState fromValue(int value) { + for (TransactionState state : values()) { + if (state.value == value) { + return state; + } + } + throw new IllegalArgumentException("Unknown transaction state value: " + value); + } +} diff --git a/src/test/java/com/tidesdb/PojoAndEnumTest.java b/src/test/java/com/tidesdb/PojoAndEnumTest.java index 289a17f..469f5c7 100644 --- a/src/test/java/com/tidesdb/PojoAndEnumTest.java +++ b/src/test/java/com/tidesdb/PojoAndEnumTest.java @@ -8,7 +8,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * https://www.mozilla.org.org/en-US/MPL/2.0/ + * https://www.mozilla.org/en-US/MPL/2.0/ * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,825 +18,629 @@ */ package com.tidesdb; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; -import java.nio.file.Path; +import java.nio.charset.StandardCharsets; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.*; /** - * Tests for pure-Java POJOs, enums, builders, and exception classes. - * Covers branches and constructors not exercised by the integration tests - * in {@link TidesDBTest}. + * Tests for the value types and enums, which need no open database. */ -class PojoAndEnumTest { +public class PojoAndEnumTest { + + @Nested + class Enums { + + @Test + void logLevelMapsToTheNativeOrdering() { + assertEquals(0, LogLevel.NONE.getValue()); + assertEquals(1, LogLevel.TRACE.getValue()); + assertEquals(2, LogLevel.INFO.getValue()); + assertEquals(3, LogLevel.WARN.getValue()); + assertEquals(4, LogLevel.ERROR.getValue()); + + for (LogLevel level : LogLevel.values()) { + assertEquals(level, LogLevel.fromValue(level.getValue())); + } + assertThrows(IllegalArgumentException.class, () -> LogLevel.fromValue(99)); + } - // ----------------------------------------------------------------------- - // TidesDBException - // ----------------------------------------------------------------------- + @Test + void isolationLevelRunsWeakestToStrongest() { + assertEquals(0, IsolationLevel.READ_UNCOMMITTED.getValue()); + assertEquals(1, IsolationLevel.READ_COMMITTED.getValue()); + assertEquals(2, IsolationLevel.REPEATABLE_READ.getValue()); + assertEquals(3, IsolationLevel.SNAPSHOT.getValue()); + assertEquals(4, IsolationLevel.SERIALIZABLE.getValue()); + + for (IsolationLevel level : IsolationLevel.values()) { + assertEquals(level, IsolationLevel.fromValue(level.getValue())); + } + assertThrows(IllegalArgumentException.class, () -> IsolationLevel.fromValue(5)); + } - @Test - void tidesDBException_messageConstructor_defaultsToUnknownErrorCode() { - TidesDBException ex = new TidesDBException("boom"); - assertEquals("boom", ex.getMessage()); - assertEquals(TidesDBException.ERR_UNKNOWN, ex.getErrorCode()); - } + @Test + void syncModeMapsToTheNativeValues() { + assertEquals(0, SyncMode.SYNC_NONE.getValue()); + assertEquals(1, SyncMode.SYNC_FULL.getValue()); + assertEquals(2, SyncMode.SYNC_INTERVAL.getValue()); - @Test - void tidesDBException_messageAndCodeConstructor() { - TidesDBException ex = new TidesDBException("not found", TidesDBException.ERR_NOT_FOUND); - assertEquals("not found", ex.getMessage()); - assertEquals(TidesDBException.ERR_NOT_FOUND, ex.getErrorCode()); - } + for (SyncMode mode : SyncMode.values()) { + assertEquals(mode, SyncMode.fromValue(mode.getValue())); + } + assertThrows(IllegalArgumentException.class, () -> SyncMode.fromValue(3)); + } - @Test - void tidesDBException_messageAndCauseConstructor() { - RuntimeException cause = new RuntimeException("root"); - TidesDBException ex = new TidesDBException("wrapper", cause); - assertEquals("wrapper", ex.getMessage()); - assertSame(cause, ex.getCause()); - assertEquals(TidesDBException.ERR_UNKNOWN, ex.getErrorCode()); - } + @Test + void compressionAlgorithmValuesAreTheEncodingIds() { + assertEquals(0, CompressionAlgorithm.NONE.getValue()); + assertEquals(1, CompressionAlgorithm.SNAPPY.getValue()); + assertEquals(2, CompressionAlgorithm.LZ4.getValue()); + assertEquals(3, CompressionAlgorithm.ZSTD.getValue()); + assertEquals(4, CompressionAlgorithm.LZ4_FAST.getValue()); + + for (CompressionAlgorithm algo : CompressionAlgorithm.values()) { + assertEquals(algo, CompressionAlgorithm.fromValue(algo.getValue())); + } + assertThrows(IllegalArgumentException.class, () -> CompressionAlgorithm.fromValue(5)); + } - @Test - void tidesDBException_messageCodeAndCauseConstructor() { - RuntimeException cause = new RuntimeException("root"); - TidesDBException ex = new TidesDBException("io err", TidesDBException.ERR_IO, cause); - assertEquals("io err", ex.getMessage()); - assertEquals(TidesDBException.ERR_IO, ex.getErrorCode()); - assertSame(cause, ex.getCause()); - } + @Test + void transactionStateCoversTheLifecycle() { + assertEquals(0, TransactionState.ACTIVE.getValue()); + assertEquals(1, TransactionState.PREPARED.getValue()); + assertEquals(2, TransactionState.COMMITTED.getValue()); + assertEquals(3, TransactionState.ABORTED.getValue()); + + for (TransactionState state : TransactionState.values()) { + assertEquals(state, TransactionState.fromValue(state.getValue())); + } + assertThrows(IllegalArgumentException.class, () -> TransactionState.fromValue(4)); + } - @Test - void tidesDBException_getErrorMessage_coversAllKnownCodes() { - assertErrorMessage(TidesDBException.ERR_SUCCESS, "success"); - assertErrorMessage(TidesDBException.ERR_MEMORY, "memory allocation failed"); - assertErrorMessage(TidesDBException.ERR_INVALID_ARGS, "invalid arguments"); - assertErrorMessage(TidesDBException.ERR_NOT_FOUND, "not found"); - assertErrorMessage(TidesDBException.ERR_IO, "I/O error"); - assertErrorMessage(TidesDBException.ERR_CORRUPTION, "data corruption"); - assertErrorMessage(TidesDBException.ERR_EXISTS, "already exists"); - assertErrorMessage(TidesDBException.ERR_CONFLICT, "transaction conflict"); - assertErrorMessage(TidesDBException.ERR_TOO_LARGE, "key or value too large"); - assertErrorMessage(TidesDBException.ERR_MEMORY_LIMIT, "memory limit exceeded"); - assertErrorMessage(TidesDBException.ERR_INVALID_DB, "invalid database handle"); - assertErrorMessage(TidesDBException.ERR_LOCKED, "database is locked"); - } + @Test + void stallReasonValuesAreContiguousIndices() { + StallReason[] reasons = StallReason.values(); + for (int i = 0; i < reasons.length; i++) { + assertEquals(i, reasons[i].getValue(), "the value is the index into StallStats"); + assertEquals(reasons[i], StallReason.fromValue(i)); + } + assertThrows(IllegalArgumentException.class, + () -> StallReason.fromValue(reasons.length)); + } - @Test - void tidesDBException_getErrorMessage_unknownCodeReturnsUnknownError() { - TidesDBException ex = new TidesDBException("msg", 9999); - assertEquals("unknown error", ex.getErrorMessage()); - } + @Test + void ioClassValuesAreContiguousIndices() { + IoClass[] classes = IoClass.values(); + for (int i = 0; i < classes.length; i++) { + assertEquals(i, classes[i].getValue(), "the value is the index into IoStats"); + assertEquals(classes[i], IoClass.fromValue(i)); + } + assertThrows(IllegalArgumentException.class, () -> IoClass.fromValue(classes.length)); + } - @Test - void tidesDBException_errorCodeConstants_areDistinct() { - int[] codes = { - TidesDBException.ERR_SUCCESS, - TidesDBException.ERR_MEMORY, - TidesDBException.ERR_INVALID_ARGS, - TidesDBException.ERR_NOT_FOUND, - TidesDBException.ERR_IO, - TidesDBException.ERR_CORRUPTION, - TidesDBException.ERR_EXISTS, - TidesDBException.ERR_CONFLICT, - TidesDBException.ERR_TOO_LARGE, - TidesDBException.ERR_MEMORY_LIMIT, - TidesDBException.ERR_INVALID_DB, - TidesDBException.ERR_UNKNOWN, - TidesDBException.ERR_LOCKED, - }; - assertThat(codes).doesNotHaveDuplicates(); + @Test + void stallReasonAndIoClassCarryNativeNames() { + for (StallReason reason : StallReason.values()) { + assertNotNull(reason.getNativeName()); + assertFalse(reason.getNativeName().isEmpty()); + } + for (IoClass cls : IoClass.values()) { + assertNotNull(cls.getNativeName()); + assertFalse(cls.getNativeName().isEmpty()); + } + } } - private void assertErrorMessage(int code, String expectedMessage) { - TidesDBException ex = new TidesDBException("irrelevant", code); - assertEquals(expectedMessage, ex.getErrorMessage(), - "Error code " + code + " should map to '" + expectedMessage + "'"); - } + @Nested + class DatabaseConfig { + + @Test + void carriesTheNativeDefaults() { + Config config = Config.defaultConfig("/tmp/tidesdb-config-test"); + + assertEquals("/tmp/tidesdb-config-test", config.getDbPath()); + assertTrue(config.getNumFlushThreads() > 0); + assertTrue(config.getNumCompactionThreads() > 0); + assertTrue(config.getBlockCacheSize() > 0); + assertTrue(config.getMaxOpenSSTables() > 0); + assertTrue(config.getMemtableWriteBufferSize() > 0); + assertTrue(config.getValueSeparationThreshold() > 0); + assertTrue(config.getVlogSegmentSize() > 0); + assertNotNull(config.getLogLevel()); + assertNotNull(config.getMemtableSyncMode()); + assertNotNull(config.toString()); + } - // ----------------------------------------------------------------------- - // KeyValue - // ----------------------------------------------------------------------- - - @Test - void keyValue_constructorAndGetters() { - byte[] k = {1, 2, 3}; - byte[] v = {4, 5}; - KeyValue kv = new KeyValue(k, v); - assertSame(k, kv.getKey()); - assertSame(v, kv.getValue()); - } + @Test + void roundTripsThroughItsBuilder() { + Config config = Config.builder("/tmp/db") + .numFlushThreads(3) + .numCompactionThreads(5) + .logLevel(LogLevel.WARN) + .blockCacheSize(1024) + .maxOpenSSTables(64) + .logToFile(true) + .logTruncationAt(2048) + .memtableWriteBufferSize(4096) + .memtableSkipListMaxLevel(16) + .memtableSkipListProbability(0.5f) + .memtableSyncMode(SyncMode.SYNC_FULL) + .memtableSyncIntervalUs(1000) + .valueSeparationThreshold(512) + .vlogSegmentSize(8192) + .memtableL0QueueStallThreshold(8) + .memtableIdleFlushSeconds(30) + .txnTimeoutSeconds(60) + .build(); + + assertEquals("/tmp/db", config.getDbPath()); + assertEquals(3, config.getNumFlushThreads()); + assertEquals(5, config.getNumCompactionThreads()); + assertEquals(LogLevel.WARN, config.getLogLevel()); + assertEquals(1024, config.getBlockCacheSize()); + assertEquals(64, config.getMaxOpenSSTables()); + assertTrue(config.isLogToFile()); + assertEquals(2048, config.getLogTruncationAt()); + assertEquals(4096, config.getMemtableWriteBufferSize()); + assertEquals(16, config.getMemtableSkipListMaxLevel()); + assertEquals(0.5f, config.getMemtableSkipListProbability()); + assertEquals(SyncMode.SYNC_FULL, config.getMemtableSyncMode()); + assertEquals(1000, config.getMemtableSyncIntervalUs()); + assertEquals(512, config.getValueSeparationThreshold()); + assertEquals(8192, config.getVlogSegmentSize()); + assertEquals(8, config.getMemtableL0QueueStallThreshold()); + assertEquals(30, config.getMemtableIdleFlushSeconds()); + assertEquals(60, config.getTxnTimeoutSeconds()); + + Config copy = config.toBuilder().build(); + assertEquals(config.toString(), copy.toString()); + } - @Test - void keyValue_nullKeyAndValue() { - KeyValue kv = new KeyValue(null, null); - assertNull(kv.getKey()); - assertNull(kv.getValue()); - } + @Test + void rejectsNegativeAndNullFields() { + assertThrows(IllegalArgumentException.class, + () -> Config.builder(null).build()); + assertThrows(IllegalArgumentException.class, + () -> Config.builder("/tmp/db").numFlushThreads(-1).build()); + assertThrows(IllegalArgumentException.class, + () -> Config.builder("/tmp/db").numCompactionThreads(-1).build()); + assertThrows(IllegalArgumentException.class, + () -> Config.builder("/tmp/db").blockCacheSize(-1).build()); + assertThrows(IllegalArgumentException.class, + () -> Config.builder("/tmp/db").maxOpenSSTables(-1).build()); + assertThrows(IllegalArgumentException.class, + () -> Config.builder("/tmp/db").logTruncationAt(-1).build()); + assertThrows(IllegalArgumentException.class, + () -> Config.builder("/tmp/db").memtableWriteBufferSize(-1).build()); + assertThrows(IllegalArgumentException.class, + () -> Config.builder("/tmp/db").valueSeparationThreshold(-1).build()); + assertThrows(IllegalArgumentException.class, + () -> Config.builder("/tmp/db").vlogSegmentSize(-1).build()); + assertThrows(IllegalArgumentException.class, + () -> Config.builder("/tmp/db").memtableL0QueueStallThreshold(-1).build()); + assertThrows(IllegalArgumentException.class, + () -> Config.builder("/tmp/db").memtableIdleFlushSeconds(-1).build()); + assertThrows(IllegalArgumentException.class, + () -> Config.builder("/tmp/db").txnTimeoutSeconds(-1).build()); + assertThrows(IllegalArgumentException.class, + () -> Config.builder("/tmp/db").logLevel(null).build()); + assertThrows(IllegalArgumentException.class, + () -> Config.builder("/tmp/db").memtableSyncMode(null).build()); + } - // ----------------------------------------------------------------------- - // CommitOp - // ----------------------------------------------------------------------- - - @Test - void commitOp_putOperation() { - byte[] key = {10}; - byte[] val = {20}; - CommitOp op = new CommitOp(key, val, 3600L, false); - assertSame(key, op.getKey()); - assertSame(val, op.getValue()); - assertEquals(3600L, op.getTtl()); - assertFalse(op.isDelete()); + @Test + void boundsSkipListProbability() { + assertThrows(IllegalArgumentException.class, + () -> Config.builder("/tmp/db").memtableSkipListProbability(-0.1f).build()); + assertThrows(IllegalArgumentException.class, + () -> Config.builder("/tmp/db").memtableSkipListProbability(1.1f).build()); + assertThrows(IllegalArgumentException.class, + () -> Config.builder("/tmp/db").memtableSkipListProbability(Float.NaN).build()); + assertDoesNotThrow( + () -> Config.builder("/tmp/db").memtableSkipListProbability(0.25f).build()); + } } - @Test - void commitOp_deleteOperation() { - byte[] key = {10}; - CommitOp op = new CommitOp(key, null, -1, true); - assertSame(key, op.getKey()); - assertNull(op.getValue()); - assertEquals(-1, op.getTtl()); - assertTrue(op.isDelete()); - } + @Nested + class FamilyConfig { - // ----------------------------------------------------------------------- - // CacheStats - // ----------------------------------------------------------------------- - - @Test - void cacheStats_constructorAndGetters() { - CacheStats cs = new CacheStats(true, 100, 4096, 80, 20, 0.8, 4); - assertTrue(cs.isEnabled()); - assertEquals(100, cs.getTotalEntries()); - assertEquals(4096, cs.getTotalBytes()); - assertEquals(80, cs.getHits()); - assertEquals(20, cs.getMisses()); - assertEquals(0.8, cs.getHitRate(), 1e-9); - assertEquals(4, cs.getNumPartitions()); - } + @Test + void carriesTheNativeDefaults() { + ColumnFamilyConfig config = ColumnFamilyConfig.defaultConfig(); - @Test - void cacheStats_toString() { - CacheStats cs = new CacheStats(false, 0, 0, 0, 0, 0.0, 1); - String str = cs.toString(); - assertThat(str).contains("enabled=false"); - assertThat(str).contains("totalEntries=0"); - assertThat(str).contains("totalBytes=0"); - assertThat(str).contains("hits=0"); - assertThat(str).contains("misses=0"); - assertThat(str).contains("hitRate=0.0"); - assertThat(str).contains("numPartitions=1"); - } + assertTrue(config.getLevelSizeRatio() > 0); + assertTrue(config.getBtreeKlogBlockSize() > 0); + assertNotNull(config.getEncodingPipeline()); + assertNotNull(config.getDefaultIsolationLevel()); + assertNotNull(config.getName()); + assertNotNull(config.toString()); + } - // ----------------------------------------------------------------------- - // DbStats - // ----------------------------------------------------------------------- - - @Test - void dbStats_constructorAndGetters() { - DbStats stats = new DbStats( - 5, 1000000, 500000, 2000000, 1, 3, - 8000, 2, 10, 50000, 8, 42, - 100, 0, 0, - false, 0, 0, false, 0, 0, - false, null, 0, 0, 0, 0, 0, 0, 0, - false, 0, 0, - 0, 5000, 3000, 4000, 2000, 6000, 10, 20); - - assertEquals(5, stats.getNumColumnFamilies()); - assertEquals(1000000, stats.getTotalMemory()); - assertEquals(500000, stats.getAvailableMemory()); - assertEquals(2000000, stats.getResolvedMemoryLimit()); - assertEquals(1, stats.getMemoryPressureLevel()); - assertEquals(3, stats.getFlushPendingCount()); - assertEquals(8000, stats.getTotalMemtableBytes()); - assertEquals(2, stats.getTotalImmutableCount()); - assertEquals(10, stats.getTotalSstableCount()); - assertEquals(50000, stats.getTotalDataSizeBytes()); - assertEquals(8, stats.getNumOpenSstables()); - assertEquals(42, stats.getGlobalSeq()); - assertEquals(100, stats.getTxnMemoryBytes()); - assertEquals(0, stats.getCompactionQueueSize()); - assertEquals(0, stats.getFlushQueueSize()); - assertFalse(stats.isUnifiedMemtableEnabled()); - assertEquals(0, stats.getUnifiedMemtableBytes()); - assertEquals(0, stats.getUnifiedImmutableCount()); - assertFalse(stats.isUnifiedIsFlushing()); - assertEquals(0, stats.getUnifiedNextCfIndex()); - assertEquals(0, stats.getUnifiedWalGeneration()); - assertFalse(stats.isObjectStoreEnabled()); - assertNull(stats.getObjectStoreConnector()); - assertEquals(0, stats.getLocalCacheBytesUsed()); - assertEquals(0, stats.getLocalCacheBytesMax()); - assertEquals(0, stats.getLocalCacheNumFiles()); - assertEquals(0, stats.getLastUploadedGeneration()); - assertEquals(0, stats.getUploadQueueDepth()); - assertEquals(0, stats.getTotalUploads()); - assertEquals(0, stats.getTotalUploadFailures()); - assertFalse(stats.isReplicaMode()); - assertEquals(0, stats.getPrimaryEpoch()); - assertEquals(0, stats.getSeenEpoch()); - assertEquals(0, stats.getUwalBytesWritten()); - assertEquals(5000, stats.getWalBytesWritten()); - assertEquals(3000, stats.getFlushBytesWritten()); - assertEquals(4000, stats.getCompactionBytesWritten()); - assertEquals(2000, stats.getCompactionBytesRead()); - assertEquals(6000, stats.getUserBytesWritten()); - assertEquals(10, stats.getFlushCount()); - assertEquals(20, stats.getCompactionCount()); - } + @Test + void roundTripsThroughItsBuilder() { + ColumnFamilyConfig config = ColumnFamilyConfig.builder() + .levelSizeRatio(12) + .minLevels(2) + .dividingLevelOffset(1) + .keepValuesInline(true) + .btreeKlogBlockSize(8192) + .encodingPipeline(CompressionAlgorithm.LZ4, CompressionAlgorithm.ZSTD) + .enableBloomFilter(true) + .bloomFpr(0.02) + .defaultIsolationLevel(IsolationLevel.SERIALIZABLE) + .l1FileCountTrigger(6) + .tombstoneDensityTrigger(0.4) + .tombstoneDensityMinEntries(1000) + .build(); + + assertEquals(12, config.getLevelSizeRatio()); + assertEquals(2, config.getMinLevels()); + assertEquals(1, config.getDividingLevelOffset()); + assertTrue(config.isKeepValuesInline()); + assertEquals(8192, config.getBtreeKlogBlockSize()); + assertArrayEquals(new int[]{2, 3}, config.getEncodingPipeline()); + assertTrue(config.isEnableBloomFilter()); + assertEquals(0.02, config.getBloomFpr(), 1e-9); + assertEquals(IsolationLevel.SERIALIZABLE, config.getDefaultIsolationLevel()); + assertEquals(6, config.getL1FileCountTrigger()); + assertEquals(0.4, config.getTombstoneDensityTrigger(), 1e-9); + assertEquals(1000, config.getTombstoneDensityMinEntries()); + + ColumnFamilyConfig copy = config.toBuilder().build(); + assertEquals(config.toString(), copy.toString()); + } - @Test - void dbStats_toString() { - DbStats stats = new DbStats( - 2, 100, 50, 200, 0, 0, - 30, 0, 5, 100, 3, 1, - 0, 0, 0, - false, 0, 0, false, 0, 0, - false, null, 0, 0, 0, 0, 0, 0, 0, - false, 0, 0, - 0, 100, 200, 300, 400, 500, 1, 2); - - String str = stats.toString(); - assertThat(str).contains("numColumnFamilies=2"); - assertThat(str).contains("totalMemory=100"); - assertThat(str).contains("replicaMode=false"); - } + @Test + void treatsNoCompressionAsAnEmptyPipeline() { + assertArrayEquals(new int[0], ColumnFamilyConfig.builder() + .compression(CompressionAlgorithm.NONE).build().getEncodingPipeline()); + assertArrayEquals(new int[]{CompressionAlgorithm.SNAPPY.getValue()}, + ColumnFamilyConfig.builder() + .compression(CompressionAlgorithm.SNAPPY).build().getEncodingPipeline()); + } - // ----------------------------------------------------------------------- - // CompressionAlgorithm enum - // ----------------------------------------------------------------------- - - @Test - void compressionAlgorithm_values() { - assertEquals(0, CompressionAlgorithm.NO_COMPRESSION.getValue()); - assertEquals(1, CompressionAlgorithm.SNAPPY_COMPRESSION.getValue()); - assertEquals(2, CompressionAlgorithm.LZ4_COMPRESSION.getValue()); - assertEquals(3, CompressionAlgorithm.ZSTD_COMPRESSION.getValue()); - assertEquals(4, CompressionAlgorithm.LZ4_FAST_COMPRESSION.getValue()); - } + @Test + void acceptsRawEncodingIds() { + ColumnFamilyConfig config = + ColumnFamilyConfig.builder().encodingPipelineIds(2, 3).build(); + assertArrayEquals(new int[]{2, 3}, config.getEncodingPipeline()); - @Test - void compressionAlgorithm_fromValue_roundTripsAllValues() { - for (CompressionAlgorithm algo : CompressionAlgorithm.values()) { - assertEquals(algo, CompressionAlgorithm.fromValue(algo.getValue()), - "fromValue should round-trip for " + algo.name()); + assertArrayEquals(new int[0], + ColumnFamilyConfig.builder().encodingPipelineIds().build().getEncodingPipeline()); } - } - @Test - void compressionAlgorithm_fromValue_invalidValueThrows() { - assertThatThrownBy(() -> CompressionAlgorithm.fromValue(99)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("99"); - } + @Test + void copiesThePipelineOnTheWayInAndOut() { + int[] source = {2, 3}; + ColumnFamilyConfig config = + ColumnFamilyConfig.builder().encodingPipelineIds(source).build(); - // ----------------------------------------------------------------------- - // IsolationLevel enum - // ----------------------------------------------------------------------- - - @Test - void isolationLevel_values() { - assertEquals(0, IsolationLevel.READ_UNCOMMITTED.getValue()); - assertEquals(1, IsolationLevel.READ_COMMITTED.getValue()); - assertEquals(2, IsolationLevel.REPEATABLE_READ.getValue()); - assertEquals(3, IsolationLevel.SNAPSHOT.getValue()); - assertEquals(4, IsolationLevel.SERIALIZABLE.getValue()); - } + source[0] = 99; + assertArrayEquals(new int[]{2, 3}, config.getEncodingPipeline(), + "the builder took a copy"); - @Test - void isolationLevel_fromValue_roundTripsAllValues() { - for (IsolationLevel level : IsolationLevel.values()) { - assertEquals(level, IsolationLevel.fromValue(level.getValue()), - "fromValue should round-trip for " + level.name()); + int[] returned = config.getEncodingPipeline(); + returned[0] = 99; + assertArrayEquals(new int[]{2, 3}, config.getEncodingPipeline(), + "the getter returned a copy"); } - } - @Test - void isolationLevel_fromValue_invalidValueThrows() { - assertThatThrownBy(() -> IsolationLevel.fromValue(-99)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("-99"); - } + @Test + void boundsThePipelineLength() { + int[] tooMany = new int[ColumnFamilyConfig.MAX_ENCODING_PIPELINE + 1]; + assertThrows(IllegalArgumentException.class, + () -> ColumnFamilyConfig.builder().encodingPipelineIds(tooMany).build()); + assertThrows(IllegalArgumentException.class, + () -> ColumnFamilyConfig.builder().encodingPipelineIds(256).build()); + assertThrows(IllegalArgumentException.class, + () -> ColumnFamilyConfig.builder().encodingPipelineIds(-1).build()); + } - // ----------------------------------------------------------------------- - // LogLevel enum - // ----------------------------------------------------------------------- - - @Test - void logLevel_values() { - assertEquals(0, LogLevel.DEBUG.getValue()); - assertEquals(1, LogLevel.INFO.getValue()); - assertEquals(2, LogLevel.WARN.getValue()); - assertEquals(3, LogLevel.ERROR.getValue()); - assertEquals(4, LogLevel.FATAL.getValue()); - assertEquals(99, LogLevel.NONE.getValue()); - } + @Test + void rejectsInvalidFields() { + assertThrows(IllegalArgumentException.class, + () -> ColumnFamilyConfig.builder().bloomFpr(-0.1).build()); + assertThrows(IllegalArgumentException.class, + () -> ColumnFamilyConfig.builder().bloomFpr(1.0).build()); + assertThrows(IllegalArgumentException.class, + () -> ColumnFamilyConfig.builder().bloomFpr(Double.NaN).build()); + assertThrows(IllegalArgumentException.class, + () -> ColumnFamilyConfig.builder().tombstoneDensityTrigger(1.5).build()); + assertThrows(IllegalArgumentException.class, + () -> ColumnFamilyConfig.builder().minLevels(-1).build()); + assertThrows(IllegalArgumentException.class, + () -> ColumnFamilyConfig.builder().l1FileCountTrigger(-1).build()); + assertThrows(IllegalArgumentException.class, + () -> ColumnFamilyConfig.builder().defaultIsolationLevel(null).build()); + assertThrows(IllegalArgumentException.class, + () -> ColumnFamilyConfig.builder().compression(null).build()); + } - @Test - void logLevel_fromValue_roundTripsAllValues() { - for (LogLevel level : LogLevel.values()) { - assertEquals(level, LogLevel.fromValue(level.getValue()), - "fromValue should round-trip for " + level.name()); + @Test + void boundsTheName() { + StringBuilder tooLong = new StringBuilder(); + for (int i = 0; i < ColumnFamilyConfig.MAX_NAME_LENGTH; i++) { + tooLong.append('x'); + } + assertThrows(IllegalArgumentException.class, + () -> ColumnFamilyConfig.builder().name(tooLong.toString()).build()); + assertEquals("", ColumnFamilyConfig.builder().name(null).build().getName()); } } - @Test - void logLevel_fromValue_invalidValueThrows() { - assertThatThrownBy(() -> LogLevel.fromValue(42)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("42"); - } + @Nested + class ValueTypes { - // ----------------------------------------------------------------------- - // SyncMode enum - // ----------------------------------------------------------------------- + @Test + void commitOpCarriesItsFields() { + byte[] key = "k".getBytes(StandardCharsets.UTF_8); + byte[] value = "v".getBytes(StandardCharsets.UTF_8); - @Test - void syncMode_values() { - assertEquals(0, SyncMode.SYNC_NONE.getValue()); - assertEquals(1, SyncMode.SYNC_FULL.getValue()); - assertEquals(2, SyncMode.SYNC_INTERVAL.getValue()); - } + CommitOp put = new CommitOp(key, value, 12345L, false); + assertArrayEquals(key, put.getKey()); + assertArrayEquals(value, put.getValue()); + assertEquals(12345L, put.getTtl()); + assertFalse(put.isDelete()); - @Test - void syncMode_fromValue_roundTripsAllValues() { - for (SyncMode mode : SyncMode.values()) { - assertEquals(mode, SyncMode.fromValue(mode.getValue()), - "fromValue should round-trip for " + mode.name()); + CommitOp delete = new CommitOp(key, null, -1L, true); + assertNull(delete.getValue()); + assertTrue(delete.isDelete()); } - } - @Test - void syncMode_fromValue_invalidValueThrows() { - assertThatThrownBy(() -> SyncMode.fromValue(-1)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("-1"); - } + @Test + void keyValueCarriesBothHalves() { + byte[] key = {1, 2}; + byte[] value = {3, 4}; + KeyValue kv = new KeyValue(key, value); + assertArrayEquals(key, kv.getKey()); + assertArrayEquals(value, kv.getValue()); + } + + @Test + void cacheStatsCarriesItsFields() { + CacheStats stats = new CacheStats(true, 10, 2048, 7, 3, 0.7, 8); + assertTrue(stats.isEnabled()); + assertEquals(10, stats.getTotalEntries()); + assertEquals(2048, stats.getTotalBytes()); + assertEquals(7, stats.getHits()); + assertEquals(3, stats.getMisses()); + assertEquals(0.7, stats.getHitRate(), 1e-9); + assertEquals(8, stats.getNumPartitions()); + assertNotNull(stats.toString()); + } - // ----------------------------------------------------------------------- - // Config.Builder validation - // ----------------------------------------------------------------------- + @Test + void stallStatsIsIndexedByReason() { + StallStat[] reasons = new StallStat[StallReason.values().length]; + for (int i = 0; i < reasons.length; i++) { + reasons[i] = new StallStat(i, i * 10L, i * 100L); + } + StallStats stats = new StallStats(reasons); + + assertEquals(0, stats.get(StallReason.WAL_APPEND).getCount()); + assertEquals(StallReason.MANIFEST_COMMIT.getValue(), + stats.get(StallReason.MANIFEST_COMMIT).getCount()); + assertEquals(reasons.length, stats.getReasons().length); + assertNotNull(stats.toString()); + + long expectedTotal = 0; + for (StallStat r : reasons) { + expectedTotal += r.getTotalUs(); + } + assertEquals(expectedTotal, stats.getTotalUs()); + } - @Test - void configBuilder_nullDbPathThrows() { - assertThatThrownBy(() -> Config.builder(null).build()) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("Database path"); - } + @Test + void stallStatsRejectsAMissizedArray() { + assertThrows(IllegalArgumentException.class, () -> new StallStats(null)); + assertThrows(IllegalArgumentException.class, + () -> new StallStats(new StallStat[]{new StallStat(0, 0, 0)})); + } - @Test - void configBuilder_numFlushThreadsZeroOrNegativeThrows(@TempDir Path dir) { - assertThatThrownBy(() -> Config.builder(dir.toString()).numFlushThreads(0).build()) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("flush threads"); - assertThatThrownBy(() -> Config.builder(dir.toString()).numFlushThreads(-1).build()) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("flush threads"); - } + @Test + void ioStatsIsIndexedByClass() { + IoStat[] classes = new IoStat[IoClass.values().length]; + for (int i = 0; i < classes.length; i++) { + classes[i] = new IoStat(i, i * 1000L, i * 10L, i); + } + IoStats stats = new IoStats(classes); + + assertEquals(0, stats.get(IoClass.SSTABLE).getOps()); + assertEquals(IoClass.VLOG.getValue(), stats.get(IoClass.VLOG).getOps()); + assertEquals(classes.length, stats.getClasses().length); + assertNotNull(stats.toString()); + } - @Test - void configBuilder_numCompactionThreadsZeroOrNegativeThrows(@TempDir Path dir) { - assertThatThrownBy(() -> Config.builder(dir.toString()).numCompactionThreads(0).build()) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("compaction threads"); - assertThatThrownBy(() -> Config.builder(dir.toString()).numCompactionThreads(-1).build()) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("compaction threads"); - } + @Test + void ioStatComputesThroughput() { + assertEquals(0.0, new IoStat(0, 0, 0, 0).getBytesPerSecond(), 1e-9); + assertEquals(1_000_000.0, new IoStat(1, 1_000_000, 1_000_000, 1).getBytesPerSecond(), + 1e-6); + } - @Test - void configBuilder_nullLogLevelThrows(@TempDir Path dir) { - assertThatThrownBy(() -> Config.builder(dir.toString()).logLevel(null).build()) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("Log level"); - } + @Test + void ioStatsRejectsAMissizedArray() { + assertThrows(IllegalArgumentException.class, () -> new IoStats(null)); + assertThrows(IllegalArgumentException.class, + () -> new IoStats(new IoStat[]{new IoStat(0, 0, 0, 0)})); + } - @Test - void configBuilder_negativeBlockCacheSizeThrows(@TempDir Path dir) { - assertThatThrownBy(() -> Config.builder(dir.toString()).blockCacheSize(-1).build()) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("Block cache size"); - } + @Test + void encodingStatsComputesItsRatio() { + EncodingStats stats = new EncodingStats(new int[]{2}, 1000, 250, 4); + assertArrayEquals(new int[]{2}, stats.getIds()); + assertEquals(1000, stats.getLogicalBytes()); + assertEquals(250, stats.getStoredBytes()); + assertEquals(4, stats.getItemCount()); + assertEquals(4.0, stats.getRatio(), 1e-9); + assertNotNull(stats.toString()); + + assertEquals(0.0, new EncodingStats(null, 100, 0, 0).getRatio(), 1e-9, + "nothing stored means no ratio to report"); + assertArrayEquals(new int[0], new EncodingStats(null, 0, 0, 0).getIds()); + } - @Test - void configBuilder_zeroOrNegativeMaxOpenSSTablesThrows(@TempDir Path dir) { - assertThatThrownBy(() -> Config.builder(dir.toString()).maxOpenSSTables(0).build()) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("Max open SSTables"); - assertThatThrownBy(() -> Config.builder(dir.toString()).maxOpenSSTables(-5).build()) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("Max open SSTables"); - } + @Test + void rangeStatsCarriesItsFields() { + RangeStats exact = new RangeStats(3, 500, true); + assertEquals(3, exact.getSstablesOverlapping()); + assertEquals(500, exact.getEstimatedKeys()); + assertTrue(exact.isKeysExact()); + assertNotNull(exact.toString()); - @Test - void configBuilder_negativeMaxConcurrentFlushesThrows(@TempDir Path dir) { - assertThatThrownBy(() -> Config.builder(dir.toString()).maxConcurrentFlushes(-1).build()) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("maxConcurrentFlushes"); - } + assertFalse(new RangeStats(9, 100_000, false).isKeysExact()); + } - @Test - void configBuilder_negativeUnifiedMemtableSkipListMaxLevelThrows(@TempDir Path dir) { - assertThatThrownBy(() -> Config.builder(dir.toString()).unifiedMemtableSkipListMaxLevel(-1).build()) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("unifiedMemtableSkipListMaxLevel"); - } + @Test + void preparedTransactionCopiesItsXid() { + byte[] xid = {1, 2, 3}; + PreparedTransaction prepared = new PreparedTransaction(null, xid); - @Test - void configBuilder_negativeUnifiedMemtableSyncModeThrows(@TempDir Path dir) { - assertThatThrownBy(() -> Config.builder(dir.toString()).unifiedMemtableSyncMode(-1).build()) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("unifiedMemtableSyncMode"); - } + xid[0] = 99; + assertArrayEquals(new byte[]{1, 2, 3}, prepared.getXid(), + "the constructor holds the array it was given, and the getter copies it"); - @Test - void configBuilder_unifiedMemtableSkipListProbabilityOutOfRangeThrows(@TempDir Path dir) { - assertThatThrownBy(() -> Config.builder(dir.toString()).unifiedMemtableSkipListProbability(-0.1f).build()) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("unifiedMemtableSkipListProbability"); - assertThatThrownBy(() -> Config.builder(dir.toString()).unifiedMemtableSkipListProbability(1.1f).build()) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("unifiedMemtableSkipListProbability"); - assertThatThrownBy(() -> Config.builder(dir.toString()).unifiedMemtableSkipListProbability(Float.NaN).build()) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("unifiedMemtableSkipListProbability"); - assertThatThrownBy(() -> Config.builder(dir.toString()).unifiedMemtableSkipListProbability(Float.POSITIVE_INFINITY).build()) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("unifiedMemtableSkipListProbability"); - } + byte[] returned = prepared.getXid(); + returned[0] = 99; + assertArrayEquals(new byte[]{1, 2, 3}, prepared.getXid()); - @Test - void configBuilder_validBoundaryValuesAccepted(@TempDir Path dir) { - assertDoesNotThrow(() -> Config.builder(dir.toString()) - .numFlushThreads(1) - .numCompactionThreads(1) - .maxOpenSSTables(1) - .blockCacheSize(0) - .logTruncationAt(0) - .maxMemoryUsage(0) - .maxConcurrentFlushes(0) - .unifiedMemtableSkipListProbability(0.0f) - .build()); - - assertDoesNotThrow(() -> Config.builder(dir.toString()) - .unifiedMemtableSkipListProbability(1.0f) - .build()); - } + assertNull(prepared.getTransaction()); + assertArrayEquals(new byte[0], new PreparedTransaction(null, null).getXid()); + assertNotNull(prepared.toString()); + } - @Test - void configBuilder_settersPreserveValues(@TempDir Path dir) { - Config config = Config.builder(dir.toString()) - .numFlushThreads(8) - .numCompactionThreads(4) - .logLevel(LogLevel.ERROR) - .blockCacheSize(1024) - .maxOpenSSTables(128) - .logToFile(true) - .logTruncationAt(1234) - .maxMemoryUsage(9999) - .unifiedMemtable(true) - .unifiedMemtableWriteBufferSize(512) - .unifiedMemtableSkipListMaxLevel(6) - .unifiedMemtableSkipListProbability(0.5f) - .unifiedMemtableSyncMode(1) - .unifiedMemtableSyncIntervalUs(500) - .objectStoreFsPath("/some/path") - .maxConcurrentFlushes(3) - .finishCompactionsOnClose(true) - .build(); - - assertEquals(8, config.getNumFlushThreads()); - assertEquals(4, config.getNumCompactionThreads()); - assertEquals(LogLevel.ERROR, config.getLogLevel()); - assertEquals(1024, config.getBlockCacheSize()); - assertEquals(128, config.getMaxOpenSSTables()); - assertTrue(config.isLogToFile()); - assertEquals(1234, config.getLogTruncationAt()); - assertEquals(9999, config.getMaxMemoryUsage()); - assertTrue(config.isUnifiedMemtable()); - assertEquals(512, config.getUnifiedMemtableWriteBufferSize()); - assertEquals(6, config.getUnifiedMemtableSkipListMaxLevel()); - assertEquals(0.5f, config.getUnifiedMemtableSkipListProbability(), 1e-6); - assertEquals(1, config.getUnifiedMemtableSyncMode()); - assertEquals(500, config.getUnifiedMemtableSyncIntervalUs()); - assertEquals("/some/path", config.getObjectStoreFsPath()); - assertEquals(3, config.getMaxConcurrentFlushes()); - assertTrue(config.isFinishCompactionsOnClose()); - } + @Test + void cfStatsCopiesItsLevelArrays() { + long[] sizes = new long[CfStats.MAX_LEVELS]; + sizes[0] = 100; + CfStats stats = new CfStats(1, null, sizes, new int[CfStats.MAX_LEVELS], + new long[CfStats.MAX_LEVELS], new long[CfStats.MAX_LEVELS], 10, 100, 1.0, 2.0, 1.0, + 5, 2, 2.0, 0, 0.0, 0.0, 0, 0, 0, 0, 0, 0, 0, 0, 0); - // ----------------------------------------------------------------------- - // ColumnFamilyConfig.Builder validation - // ----------------------------------------------------------------------- - - @Test - void columnFamilyConfigBuilder_settersPreserveValues() { - ColumnFamilyConfig config = ColumnFamilyConfig.builder() - .writeBufferSize(64 * 1024) - .levelSizeRatio(5) - .minLevels(3) - .dividingLevelOffset(1) - .klogValueThreshold(256) - .compressionAlgorithm(CompressionAlgorithm.ZSTD_COMPRESSION) - .enableBloomFilter(false) - .bloomFPR(0.05) - .enableBlockIndexes(false) - .indexSampleRatio(4) - .blockIndexPrefixLen(32) - .syncMode(SyncMode.SYNC_NONE) - .syncIntervalUs(5000) - .comparatorName("my_cmp") - .skipListMaxLevel(8) - .skipListProbability(0.5f) - .defaultIsolationLevel(IsolationLevel.SNAPSHOT) - .minDiskSpace(1024) - .l1FileCountTrigger(8) - .l0QueueStallThreshold(10) - .tombstoneDensityTrigger(0.3) - .tombstoneDensityMinEntries(512) - .useBtree(true) - .objectLazyCompaction(true) - .objectPrefetchCompaction(false) - .build(); - - assertEquals(64 * 1024, config.getWriteBufferSize()); - assertEquals(5, config.getLevelSizeRatio()); - assertEquals(3, config.getMinLevels()); - assertEquals(1, config.getDividingLevelOffset()); - assertEquals(256, config.getKlogValueThreshold()); - assertEquals(CompressionAlgorithm.ZSTD_COMPRESSION, config.getCompressionAlgorithm()); - assertFalse(config.isEnableBloomFilter()); - assertEquals(0.05, config.getBloomFPR(), 1e-9); - assertFalse(config.isEnableBlockIndexes()); - assertEquals(4, config.getIndexSampleRatio()); - assertEquals(32, config.getBlockIndexPrefixLen()); - assertEquals(SyncMode.SYNC_NONE, config.getSyncMode()); - assertEquals(5000, config.getSyncIntervalUs()); - assertEquals("my_cmp", config.getComparatorName()); - assertEquals(8, config.getSkipListMaxLevel()); - assertEquals(0.5f, config.getSkipListProbability(), 1e-6); - assertEquals(IsolationLevel.SNAPSHOT, config.getDefaultIsolationLevel()); - assertEquals(1024, config.getMinDiskSpace()); - assertEquals(8, config.getL1FileCountTrigger()); - assertEquals(10, config.getL0QueueStallThreshold()); - assertEquals(0.3, config.getTombstoneDensityTrigger(), 1e-9); - assertEquals(512, config.getTombstoneDensityMinEntries()); - assertTrue(config.isUseBtree()); - assertTrue(config.isObjectLazyCompaction()); - assertFalse(config.isObjectPrefetchCompaction()); - } + sizes[0] = 999; + assertEquals(100, stats.getLevelSizes()[0], "the constructor took a copy"); - // ----------------------------------------------------------------------- - // ObjectStoreConfig.Builder - // ----------------------------------------------------------------------- - - @Test - void objectStoreConfigBuilder_settersPreserveValues() { - ObjectStoreConfig config = ObjectStoreConfig.builder() - .localCachePath("/cache") - .localCacheMaxBytes(1024) - .cacheOnRead(false) - .cacheOnWrite(false) - .maxConcurrentUploads(2) - .maxConcurrentDownloads(16) - .multipartThreshold(1024 * 1024) - .multipartPartSize(256 * 1024) - .syncManifestToObject(false) - .replicateWal(false) - .walUploadSync(true) - .walSyncThresholdBytes(2048) - .walSyncOnCommit(true) - .replicaMode(true) - .replicaSyncIntervalUs(1000) - .replicaReplayWal(false) - .build(); - - assertEquals("/cache", config.getLocalCachePath()); - assertEquals(1024, config.getLocalCacheMaxBytes()); - assertFalse(config.isCacheOnRead()); - assertFalse(config.isCacheOnWrite()); - assertEquals(2, config.getMaxConcurrentUploads()); - assertEquals(16, config.getMaxConcurrentDownloads()); - assertEquals(1024 * 1024, config.getMultipartThreshold()); - assertEquals(256 * 1024, config.getMultipartPartSize()); - assertFalse(config.isSyncManifestToObject()); - assertFalse(config.isReplicateWal()); - assertTrue(config.isWalUploadSync()); - assertEquals(2048, config.getWalSyncThresholdBytes()); - assertTrue(config.isWalSyncOnCommit()); - assertTrue(config.isReplicaMode()); - assertEquals(1000, config.getReplicaSyncIntervalUs()); - assertFalse(config.isReplicaReplayWal()); - } + long[] returned = stats.getLevelSizes(); + returned[0] = 999; + assertEquals(100, stats.getLevelSizes()[0], "the getter returned a copy"); - @Test - void objectStoreConfigBuilder_defaultValues() { - ObjectStoreConfig config = ObjectStoreConfig.builder().build(); - assertNull(config.getLocalCachePath()); - assertEquals(0, config.getLocalCacheMaxBytes()); - assertTrue(config.isCacheOnRead()); - assertTrue(config.isCacheOnWrite()); - assertEquals(4, config.getMaxConcurrentUploads()); - assertEquals(8, config.getMaxConcurrentDownloads()); - assertEquals(64 * 1024 * 1024, config.getMultipartThreshold()); - assertEquals(8 * 1024 * 1024, config.getMultipartPartSize()); - assertTrue(config.isSyncManifestToObject()); - assertTrue(config.isReplicateWal()); - assertFalse(config.isWalUploadSync()); - assertEquals(1048576, config.getWalSyncThresholdBytes()); - assertFalse(config.isWalSyncOnCommit()); - assertFalse(config.isReplicaMode()); - assertEquals(5000000, config.getReplicaSyncIntervalUs()); - assertTrue(config.isReplicaReplayWal()); + assertEquals(1, stats.getNumLevels()); + assertEquals(10, stats.getTotalKeys()); + assertNotNull(stats.toString()); + } } - // ----------------------------------------------------------------------- - // S3Config.Builder - // ----------------------------------------------------------------------- - - @Test - void s3ConfigBuilder_emptyRequiredFieldsThrow() { - assertThatThrownBy(() -> S3Config.builder().endpoint("").bucket("b").accessKey("ak").secretKey("sk").build()) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("endpoint"); - assertThatThrownBy(() -> S3Config.builder().endpoint("e").bucket("").accessKey("ak").secretKey("sk").build()) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("bucket"); - assertThatThrownBy(() -> S3Config.builder().endpoint("e").bucket("b").accessKey("").secretKey("sk").build()) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("access key"); - assertThatThrownBy(() -> S3Config.builder().endpoint("e").bucket("b").accessKey("ak").secretKey("").build()) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("secret key"); - } + @Nested + class Exceptions { - @Test - void s3ConfigBuilder_defaultOptionalFields() { - S3Config config = S3Config.builder() - .endpoint("s3.example.com") - .bucket("my-bucket") - .accessKey("AK") - .secretKey("SK") - .build(); - - assertNull(config.getPrefix()); - assertNull(config.getRegion()); - assertTrue(config.isUseSsl()); - assertFalse(config.isUsePathStyle()); - assertNull(config.getTlsCaPath()); - assertFalse(config.isTlsInsecureSkipVerify()); - assertEquals(0, config.getMultipartThreshold()); - assertEquals(0, config.getMultipartPartSize()); - } + @Test + void carriesACodeAndADescription() { + TidesDBException e = new TidesDBException("boom", TidesDBException.ERR_IO); + assertEquals(TidesDBException.ERR_IO, e.getErrorCode()); + assertEquals("boom", e.getMessage()); + assertEquals("I/O error", e.getErrorMessage()); + } - @Test - void s3ConfigBuilder_allFieldsSet() { - S3Config config = S3Config.builder() - .endpoint("minio.local:9000") - .bucket("test-bucket") - .prefix("prefix/") - .accessKey("access") - .secretKey("secret") - .region("eu-west-1") - .useSsl(false) - .usePathStyle(true) - .tlsCaPath("/etc/ssl/ca.pem") - .tlsInsecureSkipVerify(true) - .multipartThreshold(5 * 1024 * 1024) - .multipartPartSize(1024 * 1024) - .build(); - - assertEquals("minio.local:9000", config.getEndpoint()); - assertEquals("test-bucket", config.getBucket()); - assertEquals("prefix/", config.getPrefix()); - assertEquals("access", config.getAccessKey()); - assertEquals("secret", config.getSecretKey()); - assertEquals("eu-west-1", config.getRegion()); - assertFalse(config.isUseSsl()); - assertTrue(config.isUsePathStyle()); - assertEquals("/etc/ssl/ca.pem", config.getTlsCaPath()); - assertTrue(config.isTlsInsecureSkipVerify()); - assertEquals(5 * 1024 * 1024, config.getMultipartThreshold()); - assertEquals(1024 * 1024, config.getMultipartPartSize()); - } + @Test + void defaultsToUnknown() { + assertEquals(TidesDBException.ERR_UNKNOWN, + new TidesDBException("boom").getErrorCode()); + assertEquals(TidesDBException.ERR_UNKNOWN, + new TidesDBException("boom", new RuntimeException()).getErrorCode()); + } - // ----------------------------------------------------------------------- - // Stats - // ----------------------------------------------------------------------- - - @Test - void stats_constructorAndGetters() { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - Stats stats = new Stats( - 3, 1024, - new long[]{100, 200, 300}, new int[]{1, 2, 3}, - cfConfig, - 500, 600, - 10.5, 20.5, new long[]{100, 200, 200}, - 1.5, 0.95, - false, 0, 0, 0.0, - 10, 0.02, new long[]{5, 3, 2}, - 0.15, 2, - 1000, 2000, 3000, 4000, 5000, 5, 10); - - assertEquals(3, stats.getNumLevels()); - assertEquals(1024, stats.getMemtableSize()); - assertArrayEquals(new long[]{100, 200, 300}, stats.getLevelSizes()); - assertArrayEquals(new int[]{1, 2, 3}, stats.getLevelNumSSTables()); - assertNotNull(stats.getConfig()); - assertEquals(500, stats.getTotalKeys()); - assertEquals(600, stats.getTotalDataSize()); - assertEquals(10.5, stats.getAvgKeySize(), 1e-9); - assertEquals(20.5, stats.getAvgValueSize(), 1e-9); - assertArrayEquals(new long[]{100, 200, 200}, stats.getLevelKeyCounts()); - assertEquals(1.5, stats.getReadAmp(), 1e-9); - assertEquals(0.95, stats.getHitRate(), 1e-9); - assertFalse(stats.isUseBtree()); - assertEquals(0, stats.getBtreeTotalNodes()); - assertEquals(0, stats.getBtreeMaxHeight()); - assertEquals(0.0, stats.getBtreeAvgHeight(), 1e-9); - assertEquals(10, stats.getTotalTombstones()); - assertEquals(0.02, stats.getTombstoneRatio(), 1e-9); - assertArrayEquals(new long[]{5, 3, 2}, stats.getLevelTombstoneCounts()); - assertEquals(0.15, stats.getMaxSstDensity(), 1e-9); - assertEquals(2, stats.getMaxSstDensityLevel()); - assertEquals(1000, stats.getWalBytesWritten()); - assertEquals(2000, stats.getFlushBytesWritten()); - assertEquals(3000, stats.getCompactionBytesWritten()); - assertEquals(4000, stats.getCompactionBytesRead()); - assertEquals(5000, stats.getUserBytesWritten()); - assertEquals(5, stats.getFlushCount()); - assertEquals(10, stats.getCompactionCount()); - } + @Test + void keepsItsCause() { + RuntimeException cause = new RuntimeException("root"); + TidesDBException e = new TidesDBException("boom", TidesDBException.ERR_IO, cause); + assertSame(cause, e.getCause()); + assertEquals(TidesDBException.ERR_IO, e.getErrorCode()); + } - @Test - void stats_toString_containsAllFields() { - Stats stats = new Stats( - 2, 512, - new long[]{100, 200}, new int[]{1, 2}, - ColumnFamilyConfig.defaultConfig(), - 300, 400, - 8.0, 16.0, new long[]{150, 150}, - 1.0, 0.9, - false, 0, 0, 0.0, - 5, 0.01, new long[]{3, 2}, - 0.1, 1, - 500, 1000, 1500, 2000, 2500, 3, 7); - - String str = stats.toString(); - assertThat(str).contains("numLevels=2"); - assertThat(str).contains("memtableSize=512"); - assertThat(str).contains("totalKeys=300"); - assertThat(str).contains("totalDataSize=400"); - assertThat(str).contains("readAmp="); - assertThat(str).contains("hitRate="); - assertThat(str).contains("useBtree=false"); - assertThat(str).contains("totalTombstones=5"); - assertThat(str).contains("tombstoneRatio="); - assertThat(str).contains("levelSizes=["); - assertThat(str).contains("levelNumSSTables=["); - assertThat(str).contains("levelKeyCounts=["); - assertThat(str).contains("levelTombstoneCounts=["); - } + @Test + void describesEveryCode() { + int[] codes = { + TidesDBException.ERR_SUCCESS, TidesDBException.ERR_MEMORY, + TidesDBException.ERR_INVALID_ARGS, TidesDBException.ERR_NOT_FOUND, + TidesDBException.ERR_IO, TidesDBException.ERR_CORRUPTION, + TidesDBException.ERR_EXISTS, TidesDBException.ERR_CONFLICT, + TidesDBException.ERR_TOO_LARGE, TidesDBException.ERR_MEMORY_LIMIT, + TidesDBException.ERR_INVALID_DB, TidesDBException.ERR_UNKNOWN, + TidesDBException.ERR_LOCKED, TidesDBException.ERR_READONLY, + TidesDBException.ERR_TXN_EXPIRED, TidesDBException.ERR_NO_SPACE, + TidesDBException.ERR_TXN_ABORTED, TidesDBException.ERR_TOO_OLD}; + + for (int code : codes) { + String message = new TidesDBException("x", code).getErrorMessage(); + assertNotNull(message); + assertFalse(message.isEmpty()); + } + assertEquals("unknown error", new TidesDBException("x", -999).getErrorMessage()); + } - @Test - void stats_toString_nullArraysOmitted() { - Stats stats = new Stats( - 0, 0, null, null, ColumnFamilyConfig.defaultConfig(), - 0, 0, 0, 0, null, 0, 0, - false, 0, 0, 0.0, - 0, 0, null, 0, 0, - 0, 0, 0, 0, 0, 0, 0); - - String str = stats.toString(); - assertThat(str).doesNotContain("levelSizes="); - assertThat(str).doesNotContain("levelNumSSTables="); - assertThat(str).doesNotContain("levelKeyCounts="); - assertThat(str).doesNotContain("levelTombstoneCounts="); + @Test + void identifiesTheRetryableCode() { + assertTrue(new TidesDBException("x", TidesDBException.ERR_LOCKED).isRetryable()); + assertFalse(new TidesDBException("x", TidesDBException.ERR_CONFLICT).isRetryable()); + assertFalse(new TidesDBException("x", TidesDBException.ERR_IO).isRetryable()); + } + + @Test + void errorCodesMatchTheNativeNumbering() { + assertEquals(0, TidesDBException.ERR_SUCCESS); + assertEquals(-1, TidesDBException.ERR_MEMORY); + assertEquals(-2, TidesDBException.ERR_INVALID_ARGS); + assertEquals(-3, TidesDBException.ERR_NOT_FOUND); + assertEquals(-4, TidesDBException.ERR_IO); + assertEquals(-5, TidesDBException.ERR_CORRUPTION); + assertEquals(-6, TidesDBException.ERR_EXISTS); + assertEquals(-7, TidesDBException.ERR_CONFLICT); + assertEquals(-8, TidesDBException.ERR_TOO_LARGE); + assertEquals(-9, TidesDBException.ERR_MEMORY_LIMIT); + assertEquals(-10, TidesDBException.ERR_INVALID_DB); + assertEquals(-11, TidesDBException.ERR_UNKNOWN); + assertEquals(-12, TidesDBException.ERR_LOCKED); + assertEquals(-13, TidesDBException.ERR_READONLY); + assertEquals(-14, TidesDBException.ERR_TXN_EXPIRED); + assertEquals(-15, TidesDBException.ERR_NO_SPACE); + assertEquals(-16, TidesDBException.ERR_TXN_ABORTED); + assertEquals(-17, TidesDBException.ERR_TOO_OLD); + } } - @Test - void stats_toString_btreeFieldsIncludedWhenEnabled() { - Stats stats = new Stats( - 1, 0, - new long[0], new int[0], - ColumnFamilyConfig.defaultConfig(), - 0, 0, - 0, 0, new long[0], - 0, 0, - true, 50, 5, 3.5, - 0, 0, new long[0], - 0, 0, - 0, 0, 0, 0, 0, 0, 0); - - String str = stats.toString(); - assertThat(str).contains("useBtree=true"); - assertThat(str).contains("btreeTotalNodes=50"); - assertThat(str).contains("btreeMaxHeight=5"); - assertThat(str).contains("btreeAvgHeight=3.5"); + @Nested + class NativeStatics { + + @Test + void reportsWhichCompressionBackendsAreLinkedIn() { + assertTrue(TidesDB.isCompressionAvailable(CompressionAlgorithm.NONE), + "no-compression is always available"); + for (CompressionAlgorithm algo : CompressionAlgorithm.values()) { + assertDoesNotThrow(() -> TidesDB.isCompressionAvailable(algo)); + } + assertThrows(IllegalArgumentException.class, + () -> TidesDB.isCompressionAvailable(null)); + } + + @Test + void describesResultCodes() { + assertNotNull(TidesDB.strerror(TidesDBException.ERR_SUCCESS)); + assertNotNull(TidesDB.strerror(TidesDBException.ERR_NOT_FOUND)); + assertNotNull(TidesDB.strerror(-999), "an unrecognised code still describes itself"); + } + + @Test + void reportsTheOpenFileCeiling() { + assertTrue(TidesDB.raiseOpenFileLimit(0) > 0); + } + + @Test + void loadsTheNativeLibrary() { + NativeLibrary.load(); + assertTrue(NativeLibrary.isLoaded()); + } } } diff --git a/src/test/java/com/tidesdb/TidesDBTest.java b/src/test/java/com/tidesdb/TidesDBTest.java index f8bb5df..ac91371 100644 --- a/src/test/java/com/tidesdb/TidesDBTest.java +++ b/src/test/java/com/tidesdb/TidesDBTest.java @@ -18,3333 +18,1656 @@ */ package com.tidesdb; -import org.junit.jupiter.api.*; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; -import java.time.Instant; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; -import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import static org.junit.jupiter.api.Assertions.*; /** - * Tests for TidesDB Java bindings. + * Behavioural tests for the TidesDB Java binding against a live engine. */ -@TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class TidesDBTest { - + @TempDir Path tempDir; - - @Test - @Order(1) - void testOpenClose() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - assertNotNull(db); + + private static byte[] b(String s) { + return s.getBytes(StandardCharsets.UTF_8); + } + + private static String s(byte[] v) { + return v == null ? null : new String(v, StandardCharsets.UTF_8); + } + + /** Opens a database under a fresh subdirectory, with logging off. */ + private TidesDB open(String name) throws TidesDBException { + return TidesDB.open(Config.builder(tempDir.resolve(name).toString()) + .logLevel(LogLevel.NONE) + .build()); + } + + /** Opens a database with one column family already created, and returns both. */ + private TidesDB openWithCf(String name, String cfName) throws TidesDBException { + TidesDB db = open(name); + db.createColumnFamily(cfName, ColumnFamilyConfig.builder().build()); + return db; + } + + /** Writes one committed key/value pair. */ + private static void write(TidesDB db, ColumnFamily cf, String key, String value) + throws TidesDBException { + try (Transaction txn = db.beginTransaction()) { + txn.put(cf, b(key), b(value)); + txn.commit(); } } - - @Test - @Order(2) - void testCreateDropColumnFamily() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb2").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - - db.createColumnFamily("test_cf", cfConfig); - - ColumnFamily cf = db.getColumnFamily("test_cf"); - assertNotNull(cf); - assertEquals("test_cf", cf.getName()); - - String[] families = db.listColumnFamilies(); - assertTrue(families.length > 0); - - db.dropColumnFamily("test_cf"); + + /** Reads one key back in its own transaction. */ + private static String read(TidesDB db, ColumnFamily cf, String key) throws TidesDBException { + try (Transaction txn = db.beginTransaction()) { + String value = s(txn.get(cf, b(key))); + txn.rollback(); + return value; } } - - @Test - @Order(3) - void testTransactionPutGetDelete() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb3").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - db.createColumnFamily("test_cf", cfConfig); - - ColumnFamily cf = db.getColumnFamily("test_cf"); - - byte[] key = "key".getBytes(StandardCharsets.UTF_8); - byte[] value = "value".getBytes(StandardCharsets.UTF_8); - - try (Transaction txn = db.beginTransaction()) { - txn.put(cf, key, value); - txn.commit(); + + @Nested + class Lifecycle { + + @Test + void opensAndCloses() throws TidesDBException { + try (TidesDB db = open("open-close")) { + assertNotNull(db); } - - try (Transaction txn = db.beginTransaction()) { - byte[] result = txn.get(cf, key); - assertNotNull(result); - assertArrayEquals(value, result); + } + + @Test + void closeIsIdempotent() throws TidesDBException { + TidesDB db = open("double-close"); + db.close(); + assertDoesNotThrow(db::close); + } + + @Test + void operationsOnAClosedDatabaseThrow() throws TidesDBException { + TidesDB db = open("closed-guard"); + db.close(); + assertThrows(IllegalStateException.class, db::listColumnFamilies); + assertThrows(IllegalStateException.class, db::beginTransaction); + assertThrows(IllegalStateException.class, db::getDbStats); + } + + @Test + void rejectsAMissingPath() { + assertThrows(IllegalArgumentException.class, () -> TidesDB.open(null)); + assertThrows(IllegalArgumentException.class, + () -> TidesDB.open(Config.builder("").build())); + } + + @Test + void refusesASecondHandleOnTheSameDirectory() throws TidesDBException { + try (TidesDB first = open("locked")) { + TidesDBException e = assertThrows(TidesDBException.class, () -> open("locked")); + assertEquals(TidesDBException.ERR_LOCKED, e.getErrorCode()); + assertTrue(e.isRetryable()); } - - try (Transaction txn = db.beginTransaction()) { - txn.delete(cf, key); - txn.commit(); + } + + @Test + void dataSurvivesAReopen() throws TidesDBException { + Config config = Config.builder(tempDir.resolve("reopen").toString()) + .logLevel(LogLevel.NONE) + .build(); + + try (TidesDB db = TidesDB.open(config)) { + db.createColumnFamily("cf", ColumnFamilyConfig.builder().build()); + write(db, db.getColumnFamily("cf"), "durable", "value"); } - - try (Transaction txn = db.beginTransaction()) { - assertThrows(TidesDBException.class, () -> txn.get(cf, key)); + try (TidesDB db = TidesDB.open(config)) { + assertEquals("value", read(db, db.getColumnFamily("cf"), "durable")); } } } - - @Test - @Order(4) - void testTransactionWithTTL() throws TidesDBException, InterruptedException { - Config config = Config.builder(tempDir.resolve("testdb4").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - db.createColumnFamily("test_cf", cfConfig); - - ColumnFamily cf = db.getColumnFamily("test_cf"); - - byte[] key = "temp_key".getBytes(StandardCharsets.UTF_8); - byte[] value = "temp_value".getBytes(StandardCharsets.UTF_8); - - // Set TTL to 2 seconds from now - long ttl = Instant.now().getEpochSecond() + 2; - - try (Transaction txn = db.beginTransaction()) { - txn.put(cf, key, value, ttl); - txn.commit(); + + @Nested + class ColumnFamilies { + + @Test + void createsListsAndDrops() throws TidesDBException { + try (TidesDB db = open("cf-crud")) { + db.createColumnFamily("alpha", ColumnFamilyConfig.builder().build()); + db.createColumnFamily("beta", ColumnFamilyConfig.builder().build()); + + assertEquals(2, db.listColumnFamilies().length); + assertTrue(Arrays.asList(db.listColumnFamilies()).contains("alpha")); + + db.dropColumnFamily("alpha"); + assertFalse(Arrays.asList(db.listColumnFamilies()).contains("alpha")); } - - // Verify key exists before expiration - try (Transaction txn = db.beginTransaction()) { - byte[] result = txn.get(cf, key); - assertNotNull(result); - assertArrayEquals(value, result); - } - - Thread.sleep(3000); - - // Verify key is expired - try (Transaction txn = db.beginTransaction()) { - assertThrows(TidesDBException.class, () -> txn.get(cf, key)); + } + + @Test + void listsNothingOnAFreshDatabase() throws TidesDBException { + try (TidesDB db = open("cf-empty")) { + assertEquals(0, db.listColumnFamilies().length); } } - } - - @Test - @Order(5) - void testMultiOperationTransaction() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb5").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - db.createColumnFamily("test_cf", cfConfig); - - ColumnFamily cf = db.getColumnFamily("test_cf"); - - // Multiple operations in one transaction - try (Transaction txn = db.beginTransaction()) { - txn.put(cf, "key1".getBytes(), "value1".getBytes()); - txn.put(cf, "key2".getBytes(), "value2".getBytes()); - txn.put(cf, "key3".getBytes(), "value3".getBytes()); - txn.commit(); + + @Test + void rejectsADuplicateName() throws TidesDBException { + try (TidesDB db = openWithCf("cf-dup", "cf")) { + TidesDBException e = assertThrows(TidesDBException.class, + () -> db.createColumnFamily("cf", ColumnFamilyConfig.builder().build())); + assertEquals(TidesDBException.ERR_EXISTS, e.getErrorCode()); } - - // Verify all keys exist - try (Transaction txn = db.beginTransaction()) { - for (int i = 1; i <= 3; i++) { - byte[] key = ("key" + i).getBytes(); - byte[] expectedValue = ("value" + i).getBytes(); - byte[] result = txn.get(cf, key); - assertArrayEquals(expectedValue, result); - } + } + + @Test + void reportsAMissingFamilyAsNotFound() throws TidesDBException { + try (TidesDB db = open("cf-missing")) { + TidesDBException e = + assertThrows(TidesDBException.class, () -> db.getColumnFamily("absent")); + assertEquals(TidesDBException.ERR_NOT_FOUND, e.getErrorCode()); } } - } - - @Test - @Order(6) - void testTransactionRollback() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb6").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - db.createColumnFamily("test_cf", cfConfig); - - ColumnFamily cf = db.getColumnFamily("test_cf"); - - byte[] key = "rollback_key".getBytes(); - byte[] value = "rollback_value".getBytes(); - - try (Transaction txn = db.beginTransaction()) { - txn.put(cf, key, value); - txn.rollback(); + + @Test + void renamesInPlace() throws TidesDBException { + try (TidesDB db = openWithCf("cf-rename", "before")) { + write(db, db.getColumnFamily("before"), "k", "v"); + db.renameColumnFamily("before", "after"); + + assertTrue(Arrays.asList(db.listColumnFamilies()).contains("after")); + assertFalse(Arrays.asList(db.listColumnFamilies()).contains("before")); + assertEquals("v", read(db, db.getColumnFamily("after"), "k")); } - - // Verify key does not exist - try (Transaction txn = db.beginTransaction()) { - assertThrows(TidesDBException.class, () -> txn.get(cf, key)); + } + + @Test + void clonesAtAPointInTime() throws TidesDBException { + try (TidesDB db = openWithCf("cf-clone", "src")) { + ColumnFamily src = db.getColumnFamily("src"); + write(db, src, "before-clone", "yes"); + + db.cloneColumnFamily("src", "dst"); + write(db, src, "after-clone", "yes"); + + ColumnFamily dst = db.getColumnFamily("dst"); + assertEquals("yes", read(db, dst, "before-clone")); + assertNull(read(db, dst, "after-clone"), "a clone is a point-in-time copy"); } } - } - - @Test - @Order(7) - void testSavepoints() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb7").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - db.createColumnFamily("test_cf", cfConfig); - - ColumnFamily cf = db.getColumnFamily("test_cf"); - - try (Transaction txn = db.beginTransaction()) { - txn.put(cf, "key1".getBytes(), "value1".getBytes()); - - txn.savepoint("sp1"); - txn.put(cf, "key2".getBytes(), "value2".getBytes()); - - // Rollback to savepoint -- key2 is discarded, key1 remains - txn.rollbackToSavepoint("sp1"); - - // Add different operation after rollback - txn.put(cf, "key3".getBytes(), "value3".getBytes()); - - txn.commit(); + + @Test + void rejectsEmptyNames() throws TidesDBException { + try (TidesDB db = open("cf-names")) { + ColumnFamilyConfig cfg = ColumnFamilyConfig.builder().build(); + assertThrows(IllegalArgumentException.class, () -> db.createColumnFamily(null, cfg)); + assertThrows(IllegalArgumentException.class, () -> db.createColumnFamily("", cfg)); + assertThrows(IllegalArgumentException.class, + () -> db.createColumnFamily("cf", null)); + assertThrows(IllegalArgumentException.class, () -> db.dropColumnFamily("")); + assertThrows(IllegalArgumentException.class, () -> db.getColumnFamily("")); } - - try (Transaction txn = db.beginTransaction()) { - // key1 should exist - assertNotNull(txn.get(cf, "key1".getBytes())); - - // key2 should not exist (rolled back) - assertThrows(TidesDBException.class, () -> txn.get(cf, "key2".getBytes())); - - // key3 should exist - assertNotNull(txn.get(cf, "key3".getBytes())); + } + + @Test + void keepsFamiliesIsolated() throws TidesDBException { + try (TidesDB db = openWithCf("cf-isolated", "one")) { + db.createColumnFamily("two", ColumnFamilyConfig.builder().build()); + ColumnFamily one = db.getColumnFamily("one"); + ColumnFamily two = db.getColumnFamily("two"); + + write(db, one, "shared-key", "from-one"); + write(db, two, "shared-key", "from-two"); + + assertEquals("from-one", read(db, one, "shared-key")); + assertEquals("from-two", read(db, two, "shared-key")); } } } - - @Test - @Order(8) - void testIterator() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb8").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - db.createColumnFamily("test_cf", cfConfig); - - ColumnFamily cf = db.getColumnFamily("test_cf"); - - try (Transaction txn = db.beginTransaction()) { - for (int i = 0; i < 10; i++) { - String key = String.format("key%02d", i); - String value = "value" + i; - txn.put(cf, key.getBytes(), value.getBytes()); - } - txn.commit(); - } - - try (Transaction txn = db.beginTransaction()) { - try (TidesDBIterator iter = txn.newIterator(cf)) { - iter.seekToFirst(); - - int count = 0; - while (iter.isValid()) { - byte[] key = iter.key(); - byte[] value = iter.value(); - assertNotNull(key); - assertNotNull(value); - count++; - iter.next(); - } - assertEquals(10, count); - } - } - - try (Transaction txn = db.beginTransaction()) { - try (TidesDBIterator iter = txn.newIterator(cf)) { - iter.seekToLast(); - - int count = 0; - while (iter.isValid()) { - byte[] key = iter.key(); - byte[] value = iter.value(); - assertNotNull(key); - assertNotNull(value); - count++; - iter.prev(); - } - assertEquals(10, count); - } + + @Nested + class ReadsAndWrites { + + @Test + void putsAndGets() throws TidesDBException { + try (TidesDB db = openWithCf("rw-basic", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + write(db, cf, "key", "value"); + assertEquals("value", read(db, cf, "key")); } } - } - - @Test - @Order(9) - void testIsolationLevels() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb9").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - db.createColumnFamily("test_cf", cfConfig); - - for (IsolationLevel level : IsolationLevel.values()) { - try (Transaction txn = db.beginTransaction(level)) { - assertNotNull(txn); - } + + @Test + void reportsAnAbsentKeyAsNull() throws TidesDBException { + try (TidesDB db = openWithCf("rw-absent", "cf")) { + assertNull(read(db, db.getColumnFamily("cf"), "never-written")); } } - } - - @Test - @Order(10) - void testColumnFamilyStats() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb10").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - db.createColumnFamily("test_cf", cfConfig); - - ColumnFamily cf = db.getColumnFamily("test_cf"); - - try (Transaction txn = db.beginTransaction()) { - for (int i = 0; i < 100; i++) { - txn.put(cf, ("key" + i).getBytes(), ("value" + i).getBytes()); + + @Test + void distinguishesAnEmptyValueFromAnAbsence() throws TidesDBException { + try (TidesDB db = openWithCf("rw-empty", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + try (Transaction txn = db.beginTransaction()) { + txn.put(cf, b("present-but-empty"), new byte[0]); + txn.commit(); + } + try (Transaction txn = db.beginTransaction()) { + byte[] value = txn.get(cf, b("present-but-empty")); + assertNotNull(value, "an empty value is a present key, not an absence"); + assertEquals(0, value.length); + assertTrue(txn.contains(cf, b("present-but-empty"))); + txn.rollback(); } - txn.commit(); } - - Stats stats = cf.getStats(); - assertNotNull(stats); - assertTrue(stats.getNumLevels() >= 0); - assertTrue(stats.getTotalKeys() >= 0); - assertTrue(stats.getTotalDataSize() >= 0); - assertTrue(stats.getAvgKeySize() >= 0); - assertTrue(stats.getAvgValueSize() >= 0); - assertTrue(stats.getReadAmp() >= 0); - assertTrue(stats.getHitRate() >= 0.0 && stats.getHitRate() <= 1.0); - assertFalse(stats.isUseBtree()); - } - } - - @Test - @Order(11) - void testCacheStats() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb11").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - CacheStats stats = db.getCacheStats(); - assertNotNull(stats); } - } - - @Test - @Order(12) - void testCustomColumnFamilyConfig() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb12").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.builder() - .writeBufferSize(128 * 1024 * 1024) - .levelSizeRatio(10) - .minLevels(5) - .compressionAlgorithm(CompressionAlgorithm.LZ4_COMPRESSION) - .enableBloomFilter(true) - .bloomFPR(0.01) - .enableBlockIndexes(true) - .syncMode(SyncMode.SYNC_INTERVAL) - .syncIntervalUs(128000) - .defaultIsolationLevel(IsolationLevel.READ_COMMITTED) - .build(); - - db.createColumnFamily("custom_cf", cfConfig); - - ColumnFamily cf = db.getColumnFamily("custom_cf"); - assertNotNull(cf); - assertEquals("custom_cf", cf.getName()); + + @Test + void overwritesInPlace() throws TidesDBException { + try (TidesDB db = openWithCf("rw-overwrite", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + write(db, cf, "key", "first"); + write(db, cf, "key", "second"); + assertEquals("second", read(db, cf, "key")); + } } - } - @Test - @Order(13) - void testBtreeColumnFamily() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb13").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.builder() - .writeBufferSize(128 * 1024 * 1024) - .levelSizeRatio(10) - .minLevels(5) - .compressionAlgorithm(CompressionAlgorithm.LZ4_COMPRESSION) - .enableBloomFilter(true) - .bloomFPR(0.01) - .enableBlockIndexes(true) - .syncMode(SyncMode.SYNC_FULL) - .useBtree(true) - .build(); - - db.createColumnFamily("btree_cf", cfConfig); - - ColumnFamily cf = db.getColumnFamily("btree_cf"); - assertNotNull(cf); - assertEquals("btree_cf", cf.getName()); - - try (Transaction txn = db.beginTransaction()) { - for (int i = 0; i < 100; i++) { - txn.put(cf, ("key" + i).getBytes(), ("value" + i).getBytes()); + @Test + void roundTripsBinaryValues() throws TidesDBException { + try (TidesDB db = openWithCf("rw-binary", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + byte[] key = {0x00, 0x01, (byte) 0xFF, 0x7F}; + byte[] value = new byte[512]; + for (int i = 0; i < value.length; i++) { + value[i] = (byte) i; + } + + try (Transaction txn = db.beginTransaction()) { + txn.put(cf, key, value); + txn.commit(); + } + try (Transaction txn = db.beginTransaction()) { + assertArrayEquals(value, txn.get(cf, key)); + txn.rollback(); } - txn.commit(); } - - try (Transaction txn = db.beginTransaction()) { - byte[] result = txn.get(cf, "key50".getBytes()); - assertNotNull(result); - assertArrayEquals("value50".getBytes(), result); - } - - Stats stats = cf.getStats(); - assertNotNull(stats); - assertTrue(stats.isUseBtree()); - assertTrue(stats.getBtreeTotalNodes() >= 0); - assertTrue(stats.getBtreeMaxHeight() >= 0); - assertTrue(stats.getBtreeAvgHeight() >= 0.0); } - } - - @Test - @Order(14) - void testBtreeIterator() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb14").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.builder() - .writeBufferSize(128 * 1024 * 1024) - .compressionAlgorithm(CompressionAlgorithm.LZ4_COMPRESSION) - .enableBloomFilter(true) - .useBtree(true) - .build(); - - db.createColumnFamily("btree_iter_cf", cfConfig); - - ColumnFamily cf = db.getColumnFamily("btree_iter_cf"); - - try (Transaction txn = db.beginTransaction()) { - for (int i = 0; i < 10; i++) { - String key = String.format("key%02d", i); - String value = "value" + i; - txn.put(cf, key.getBytes(), value.getBytes()); + + @Test + void roundTripsALargeValueThroughTheValueLog() throws TidesDBException { + try (TidesDB db = openWithCf("rw-large", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + byte[] value = new byte[256 * 1024]; + Arrays.fill(value, (byte) 'x'); + + try (Transaction txn = db.beginTransaction()) { + txn.put(cf, b("big"), value); + txn.commit(); } - txn.commit(); - } - - try (Transaction txn = db.beginTransaction()) { - try (TidesDBIterator iter = txn.newIterator(cf)) { - iter.seekToFirst(); - - int count = 0; - while (iter.isValid()) { - byte[] key = iter.key(); - byte[] value = iter.value(); - assertNotNull(key); - assertNotNull(value); - count++; - iter.next(); - } - assertEquals(10, count); + db.flushMemtable(); + try (Transaction txn = db.beginTransaction()) { + assertArrayEquals(value, txn.get(cf, b("big"))); + txn.rollback(); } } } - } - - @Test - @Order(15) - void testCloneColumnFamily() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb15").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - db.createColumnFamily("source_cf", cfConfig); - - ColumnFamily sourceCf = db.getColumnFamily("source_cf"); - - // Insert data into source - try (Transaction txn = db.beginTransaction()) { - for (int i = 0; i < 10; i++) { - txn.put(sourceCf, ("key" + i).getBytes(), ("value" + i).getBytes()); + + @Test + void readsNothingFromARolledBackTransaction() throws TidesDBException { + try (TidesDB db = openWithCf("rw-rollback", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + try (Transaction txn = db.beginTransaction()) { + txn.put(cf, b("discarded"), b("v")); + txn.rollback(); } - txn.commit(); + assertNull(read(db, cf, "discarded")); } - - // Clone the column family - db.cloneColumnFamily("source_cf", "cloned_cf"); - - // Verify clone exists - ColumnFamily clonedCf = db.getColumnFamily("cloned_cf"); - assertNotNull(clonedCf); - assertEquals("cloned_cf", clonedCf.getName()); - - // Verify both column families are listed - String[] families = db.listColumnFamilies(); - assertTrue(families.length >= 2); - - // Verify data exists in clone - try (Transaction txn = db.beginTransaction()) { - for (int i = 0; i < 10; i++) { - byte[] result = txn.get(clonedCf, ("key" + i).getBytes()); - assertNotNull(result); - assertArrayEquals(("value" + i).getBytes(), result); + } + + @Test + void doesNotTrackANoTrackRead() throws TidesDBException { + try (TidesDB db = openWithCf("rw-notrack", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + write(db, cf, "probe", "value"); + + try (Transaction txn = db.beginTransaction()) { + assertEquals("value", s(txn.getNoTrack(cf, b("probe")))); + assertNull(txn.getNoTrack(cf, b("absent"))); + txn.rollback(); } } - - // Verify independence: insert into clone, should not appear in source - try (Transaction txn = db.beginTransaction()) { - txn.put(clonedCf, "clone_only_key".getBytes(), "clone_only_value".getBytes()); - txn.commit(); - } - - try (Transaction txn = db.beginTransaction()) { - assertThrows(TidesDBException.class, () -> txn.get(sourceCf, "clone_only_key".getBytes())); - } } - } - - @Test - @Order(16) - void testCheckpoint() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb16").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - db.createColumnFamily("test_cf", cfConfig); - - ColumnFamily cf = db.getColumnFamily("test_cf"); - - // Insert some data - try (Transaction txn = db.beginTransaction()) { - for (int i = 0; i < 10; i++) { - txn.put(cf, ("key" + i).getBytes(), ("value" + i).getBytes()); + + @Test + void reportsExistenceWithoutReadingTheValue() throws TidesDBException { + try (TidesDB db = openWithCf("rw-contains", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + write(db, cf, "here", "v"); + + try (Transaction txn = db.beginTransaction()) { + assertTrue(txn.contains(cf, b("here"))); + assertFalse(txn.contains(cf, b("not-here"))); + txn.rollback(); } - txn.commit(); } - - // Create checkpoint - String checkpointDir = tempDir.resolve("testdb16_checkpoint").toString(); - db.checkpoint(checkpointDir); - - // Open the checkpoint as a separate database and verify data - Config checkpointConfig = Config.builder(checkpointDir) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB checkpointDb = TidesDB.open(checkpointConfig)) { - ColumnFamily checkpointCf = checkpointDb.getColumnFamily("test_cf"); - assertNotNull(checkpointCf); - - try (Transaction txn = checkpointDb.beginTransaction()) { - for (int i = 0; i < 10; i++) { - byte[] result = txn.get(checkpointCf, ("key" + i).getBytes()); - assertNotNull(result); - assertArrayEquals(("value" + i).getBytes(), result); - } + } + + @Test + void rejectsBadArguments() throws TidesDBException { + try (TidesDB db = openWithCf("rw-args", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + try (Transaction txn = db.beginTransaction()) { + assertThrows(IllegalArgumentException.class, () -> txn.put(null, b("k"), b("v"))); + assertThrows(IllegalArgumentException.class, () -> txn.put(cf, null, b("v"))); + assertThrows(IllegalArgumentException.class, + () -> txn.put(cf, new byte[0], b("v"))); + assertThrows(IllegalArgumentException.class, () -> txn.put(cf, b("k"), null)); + assertThrows(IllegalArgumentException.class, () -> txn.get(cf, null)); + assertThrows(IllegalArgumentException.class, () -> txn.delete(cf, new byte[0])); + txn.rollback(); } } } - } - - @Test - @Order(17) - void testCheckpointNullDir() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb16b").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - assertThrows(IllegalArgumentException.class, () -> db.checkpoint(null)); - assertThrows(IllegalArgumentException.class, () -> db.checkpoint("")); + + @Test + void refusesOperationsOnAFreedTransaction() throws TidesDBException { + try (TidesDB db = openWithCf("rw-freed", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + Transaction txn = db.beginTransaction(); + txn.free(); + txn.free(); + + assertThrows(IllegalStateException.class, () -> txn.put(cf, b("k"), b("v"))); + assertThrows(IllegalStateException.class, () -> txn.get(cf, b("k"))); + assertThrows(IllegalStateException.class, txn::commit); + assertDoesNotThrow(txn::requestAbort); + } } } - - @Test - @Order(18) - void testTransactionPutGetDeleteBadKey() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb3").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - db.createColumnFamily("test_cf", cfConfig); + @Nested + class Deletes { - ColumnFamily cf = db.getColumnFamily("test_cf"); + @Test + void deletesAKey() throws TidesDBException { + try (TidesDB db = openWithCf("del-basic", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + write(db, cf, "doomed", "v"); - byte[] key = new byte[0]; // Bad key (empty) - byte[] value = "value".getBytes(StandardCharsets.UTF_8); - - assertThrows(IllegalArgumentException.class, () -> { try (Transaction txn = db.beginTransaction()) { - txn.put(cf, key, value); + txn.delete(cf, b("doomed")); + txn.commit(); } - }); + assertNull(read(db, cf, "doomed")); + } + } - assertThrows(IllegalArgumentException.class, () -> { - try (Transaction txn = db.beginTransaction()) { - byte[] result = txn.get(cf, key); - assertNotNull(result); - assertArrayEquals(value, result); - } - }); + @Test + void singleDeletesAKeyWrittenOnce() throws TidesDBException { + try (TidesDB db = openWithCf("del-single", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + write(db, cf, "once", "v"); - assertThrows(IllegalArgumentException.class, () -> { try (Transaction txn = db.beginTransaction()) { - txn.delete(cf, key); + txn.singleDelete(cf, b("once")); txn.commit(); } - }); - } - } - - @Test - @Order(19) - void testTransactionReset() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb17").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - db.createColumnFamily("test_cf", cfConfig); - - ColumnFamily cf = db.getColumnFamily("test_cf"); - - // Begin transaction and do first batch of work - Transaction txn = db.beginTransaction(); - txn.put(cf, "key1".getBytes(), "value1".getBytes()); - txn.commit(); - - // Reset instead of free + begin - txn.reset(IsolationLevel.READ_COMMITTED); - - // Second batch of work using the same transaction - txn.put(cf, "key2".getBytes(), "value2".getBytes()); - txn.commit(); - - // Free once when done - txn.free(); - - // Verify both keys exist - try (Transaction readTxn = db.beginTransaction()) { - byte[] result1 = readTxn.get(cf, "key1".getBytes()); - assertNotNull(result1); - assertArrayEquals("value1".getBytes(), result1); - - byte[] result2 = readTxn.get(cf, "key2".getBytes()); - assertNotNull(result2); - assertArrayEquals("value2".getBytes(), result2); + assertNull(read(db, cf, "once")); } } - } - - @Test - @Order(20) - void testTransactionResetWithDifferentIsolation() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb18").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - db.createColumnFamily("test_cf", cfConfig); - - ColumnFamily cf = db.getColumnFamily("test_cf"); - - // Begin with READ_COMMITTED - Transaction txn = db.beginTransaction(IsolationLevel.READ_COMMITTED); - txn.put(cf, "key1".getBytes(), "value1".getBytes()); - txn.commit(); - - // Reset with different isolation level (REPEATABLE_READ) - txn.reset(IsolationLevel.REPEATABLE_READ); - txn.put(cf, "key2".getBytes(), "value2".getBytes()); - txn.commit(); - - // Reset again with SERIALIZABLE - txn.reset(IsolationLevel.SERIALIZABLE); - txn.put(cf, "key3".getBytes(), "value3".getBytes()); - txn.commit(); - - txn.free(); - - // Verify all keys exist - try (Transaction readTxn = db.beginTransaction()) { - for (int i = 1; i <= 3; i++) { - byte[] result = readTxn.get(cf, ("key" + i).getBytes()); - assertNotNull(result); - assertArrayEquals(("value" + i).getBytes(), result); + + @Test + void deletesAHalfOpenRange() throws TidesDBException { + try (TidesDB db = openWithCf("del-range", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + try (Transaction txn = db.beginTransaction()) { + for (int i = 0; i < 20; i++) { + txn.put(cf, b(String.format("k%02d", i)), b("v" + i)); + } + txn.commit(); } - } - } - } - - @Test - @Order(22) - void testRangeCost() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb20").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - db.createColumnFamily("test_cf", cfConfig); - - ColumnFamily cf = db.getColumnFamily("test_cf"); - - // Insert data - try (Transaction txn = db.beginTransaction()) { - for (int i = 0; i < 100; i++) { - String key = String.format("key%04d", i); - txn.put(cf, key.getBytes(), ("value" + i).getBytes()); + try (Transaction txn = db.beginTransaction()) { + txn.deleteRange(cf, b("k05"), b("k10")); + txn.commit(); } - txn.commit(); + + assertNotNull(read(db, cf, "k04"), "below the lower bound"); + assertNull(read(db, cf, "k05"), "the lower bound is inclusive"); + assertNull(read(db, cf, "k09")); + assertNotNull(read(db, cf, "k10"), "the upper bound is exclusive"); } - - // Estimate cost for a range - double cost = cf.rangeCost("key0000".getBytes(), "key0099".getBytes()); - assertTrue(cost >= 0.0, "Range cost should be non-negative"); } - } - - @Test - @Order(23) - void testRangeCostComparison() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb21").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - db.createColumnFamily("test_cf", cfConfig); - - ColumnFamily cf = db.getColumnFamily("test_cf"); - - // Insert data - try (Transaction txn = db.beginTransaction()) { - for (int i = 0; i < 1000; i++) { - String key = String.format("key%04d", i); - txn.put(cf, key.getBytes(), ("value" + i).getBytes()); + + @Test + void deletesToTheEndOfTheFamily() throws TidesDBException { + try (TidesDB db = openWithCf("del-range-open", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + try (Transaction txn = db.beginTransaction()) { + for (int i = 0; i < 10; i++) { + txn.put(cf, b("k" + i), b("v")); + } + txn.commit(); } - txn.commit(); - } - - // Both costs should be non-negative - double costSmall = cf.rangeCost("key0000".getBytes(), "key0010".getBytes()); - double costLarge = cf.rangeCost("key0000".getBytes(), "key0999".getBytes()); - assertTrue(costSmall >= 0.0, "Small range cost should be non-negative"); - assertTrue(costLarge >= 0.0, "Large range cost should be non-negative"); - } - } - - @Test - @Order(24) - void testRangeCostNullKeys() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb22").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - db.createColumnFamily("test_cf", cfConfig); - - ColumnFamily cf = db.getColumnFamily("test_cf"); - - assertThrows(IllegalArgumentException.class, - () -> cf.rangeCost(null, "key".getBytes())); - assertThrows(IllegalArgumentException.class, - () -> cf.rangeCost("key".getBytes(), null)); - assertThrows(IllegalArgumentException.class, - () -> cf.rangeCost(new byte[0], "key".getBytes())); - assertThrows(IllegalArgumentException.class, - () -> cf.rangeCost("key".getBytes(), new byte[0])); - } - } - - @Test - @Order(25) - void testCommitHookBasic() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb23").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - db.createColumnFamily("test_cf", cfConfig); - - ColumnFamily cf = db.getColumnFamily("test_cf"); - - List received = new ArrayList<>(); - AtomicLong lastSeq = new AtomicLong(); - - cf.setCommitHook((ops, commitSeq) -> { - received.add(ops); - lastSeq.set(commitSeq); - return 0; - }); - - // Commit a put operation - try (Transaction txn = db.beginTransaction()) { - txn.put(cf, "key1".getBytes(), "value1".getBytes()); - txn.commit(); + try (Transaction txn = db.beginTransaction()) { + txn.deleteRange(cf, b("k5"), null); + txn.commit(); + } + + assertNotNull(read(db, cf, "k4")); + assertNull(read(db, cf, "k5")); + assertNull(read(db, cf, "k9")); } - - // Hook fires synchronously, so data is available immediately - assertEquals(1, received.size()); - assertEquals(1, received.get(0).length); - assertArrayEquals("key1".getBytes(), received.get(0)[0].getKey()); - assertArrayEquals("value1".getBytes(), received.get(0)[0].getValue()); - assertFalse(received.get(0)[0].isDelete()); - assertTrue(lastSeq.get() > 0); - - cf.clearCommitHook(); } - } - - @Test - @Order(26) - void testCommitHookMultipleOps() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb24").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - db.createColumnFamily("test_cf", cfConfig); - - ColumnFamily cf = db.getColumnFamily("test_cf"); - - List received = new ArrayList<>(); - - cf.setCommitHook((ops, commitSeq) -> { - received.add(ops); - return 0; - }); - - // Commit multiple operations in one transaction - try (Transaction txn = db.beginTransaction()) { - txn.put(cf, "key1".getBytes(), "value1".getBytes()); - txn.put(cf, "key2".getBytes(), "value2".getBytes()); - txn.delete(cf, "key1".getBytes()); - txn.commit(); + + @Test + void deletesUnderAPrefix() throws TidesDBException { + try (TidesDB db = openWithCf("del-prefix", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + try (Transaction txn = db.beginTransaction()) { + txn.put(cf, b("user:1"), b("a")); + txn.put(cf, b("user:2"), b("b")); + txn.put(cf, b("order:1"), b("c")); + txn.commit(); + } + try (Transaction txn = db.beginTransaction()) { + txn.deletePrefix(cf, b("user:")); + txn.commit(); + } + + assertNull(read(db, cf, "user:1")); + assertNull(read(db, cf, "user:2")); + assertNotNull(read(db, cf, "order:1"), "another prefix is untouched"); } - - // Should fire once with all operations - assertEquals(1, received.size()); - assertEquals(3, received.get(0).length); - - // Last op should be a delete - assertTrue(received.get(0)[2].isDelete()); - assertArrayEquals("key1".getBytes(), received.get(0)[2].getKey()); - - cf.clearCommitHook(); } - } - - @Test - @Order(27) - void testCommitHookClear() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb25").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - db.createColumnFamily("test_cf", cfConfig); - - ColumnFamily cf = db.getColumnFamily("test_cf"); - - List received = new ArrayList<>(); - - cf.setCommitHook((ops, commitSeq) -> { - received.add(ops); - return 0; - }); - - // First commit - hook should fire - try (Transaction txn = db.beginTransaction()) { - txn.put(cf, "key1".getBytes(), "value1".getBytes()); - txn.commit(); - } - assertEquals(1, received.size()); - - // Clear the hook - cf.clearCommitHook(); - - // Second commit - hook should NOT fire - try (Transaction txn = db.beginTransaction()) { - txn.put(cf, "key2".getBytes(), "value2".getBytes()); - txn.commit(); + + @Test + void letsANewerWriteSurviveARangeDelete() throws TidesDBException { + try (TidesDB db = openWithCf("del-range-newer", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + write(db, cf, "k05", "old"); + + try (Transaction txn = db.beginTransaction()) { + txn.deleteRange(cf, b("k00"), b("k10")); + txn.put(cf, b("k05"), b("new")); + txn.commit(); + } + assertEquals("new", read(db, cf, "k05")); } - assertEquals(1, received.size(), "Hook should not fire after clearing"); - } - } - - @Test - @Order(28) - void testCommitHookNullThrows() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb26").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - db.createColumnFamily("test_cf", cfConfig); - - ColumnFamily cf = db.getColumnFamily("test_cf"); - - assertThrows(IllegalArgumentException.class, () -> cf.setCommitHook(null)); } - } - - @Test - @Order(29) - void testMaxMemoryUsageConfig() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb27").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .maxMemoryUsage(0) - .build(); - - assertEquals(0, config.getMaxMemoryUsage()); - - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - db.createColumnFamily("test_cf", cfConfig); - - ColumnFamily cf = db.getColumnFamily("test_cf"); - - try (Transaction txn = db.beginTransaction()) { - txn.put(cf, "key1".getBytes(), "value1".getBytes()); - txn.commit(); - } - - try (Transaction txn = db.beginTransaction()) { - byte[] result = txn.get(cf, "key1".getBytes()); - assertNotNull(result); - assertArrayEquals("value1".getBytes(), result); + + @Test + void boundsRangeAndPrefixArguments() throws TidesDBException { + try (TidesDB db = openWithCf("del-bounds", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + byte[] tooLong = new byte[Transaction.MAX_RANGE_BOUND_SIZE + 1]; + Arrays.fill(tooLong, (byte) 'a'); + + try (Transaction txn = db.beginTransaction()) { + assertThrows(IllegalArgumentException.class, + () -> txn.deleteRange(cf, null, b("z"))); + assertThrows(IllegalArgumentException.class, + () -> txn.deleteRange(cf, new byte[0], b("z"))); + assertThrows(IllegalArgumentException.class, + () -> txn.deleteRange(cf, tooLong, b("z"))); + assertThrows(IllegalArgumentException.class, + () -> txn.deleteRange(cf, b("a"), tooLong)); + assertThrows(IllegalArgumentException.class, + () -> txn.deletePrefix(cf, new byte[0])); + assertThrows(IllegalArgumentException.class, + () -> txn.deletePrefix(cf, tooLong)); + txn.rollback(); + } } } } - - @Test - @Order(30) - void testMultiColumnFamilyTransaction() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb28").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - db.createColumnFamily("users", cfConfig); - db.createColumnFamily("orders", cfConfig); - - ColumnFamily usersCf = db.getColumnFamily("users"); - ColumnFamily ordersCf = db.getColumnFamily("orders"); - - // Atomic transaction across multiple column families - try (Transaction txn = db.beginTransaction()) { - txn.put(usersCf, "user:1000".getBytes(), "John Doe".getBytes()); - txn.put(ordersCf, "order:5000".getBytes(), "user:1000|product:A".getBytes()); - txn.commit(); - } - - // Verify data in both column families - try (Transaction txn = db.beginTransaction()) { - byte[] user = txn.get(usersCf, "user:1000".getBytes()); - assertNotNull(user); - assertArrayEquals("John Doe".getBytes(), user); - - byte[] order = txn.get(ordersCf, "order:5000".getBytes()); - assertNotNull(order); - assertArrayEquals("user:1000|product:A".getBytes(), order); + + @Nested + class TimeToLive { + + @Test + void keepsAnUnexpiredEntry() throws TidesDBException { + try (TidesDB db = openWithCf("ttl-live", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + try (Transaction txn = db.beginTransaction()) { + txn.put(cf, b("lives"), b("v"), 3600); + txn.commit(); + } + assertEquals("v", read(db, cf, "lives")); } } - } - - @Test - @Order(31) - void testPurgeCf() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb29").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - db.createColumnFamily("test_cf", cfConfig); - - ColumnFamily cf = db.getColumnFamily("test_cf"); - - // Insert data - try (Transaction txn = db.beginTransaction()) { - for (int i = 0; i < 100; i++) { - txn.put(cf, ("key" + i).getBytes(), ("value" + i).getBytes()); + + @Test + void treatsZeroAndNegativeAsNoExpiry() throws TidesDBException { + try (TidesDB db = openWithCf("ttl-none", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + try (Transaction txn = db.beginTransaction()) { + txn.put(cf, b("zero"), b("v"), 0); + txn.put(cf, b("negative"), b("v"), -1); + txn.commit(); } - txn.commit(); - } - - // Purge the column family (synchronous flush + compaction) - cf.purge(); - - // Verify data still accessible after purge - try (Transaction txn = db.beginTransaction()) { - byte[] result = txn.get(cf, "key50".getBytes()); - assertNotNull(result); - assertArrayEquals("value50".getBytes(), result); + assertEquals("v", read(db, cf, "zero")); + assertEquals("v", read(db, cf, "negative")); } } - } - - @Test - @Order(32) - void testPurgeDb() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb30").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - db.createColumnFamily("cf1", cfConfig); - db.createColumnFamily("cf2", cfConfig); - - ColumnFamily cf1 = db.getColumnFamily("cf1"); - ColumnFamily cf2 = db.getColumnFamily("cf2"); - - // Insert data into both column families - try (Transaction txn = db.beginTransaction()) { - for (int i = 0; i < 50; i++) { - txn.put(cf1, ("key" + i).getBytes(), ("value" + i).getBytes()); - txn.put(cf2, ("key" + i).getBytes(), ("value" + i).getBytes()); + + @Test + void expiresAnEntryWhoseDeadlineHasPassed() throws Exception { + try (TidesDB db = openWithCf("ttl-expired", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + try (Transaction txn = db.beginTransaction()) { + txn.put(cf, b("brief"), b("v"), 1); + txn.commit(); } - txn.commit(); - } - - // Purge entire database - db.purge(); - - // Verify data still accessible after purge - try (Transaction txn = db.beginTransaction()) { - byte[] result1 = txn.get(cf1, "key25".getBytes()); - assertNotNull(result1); - assertArrayEquals("value25".getBytes(), result1); - - byte[] result2 = txn.get(cf2, "key25".getBytes()); - assertNotNull(result2); - assertArrayEquals("value25".getBytes(), result2); + // deadlines are judged against the same once-a-second clock + Thread.sleep(2500); + assertNull(read(db, cf, "brief"), "the entry is past its deadline"); } } } - - @Test - @Order(33) - void testSyncWal() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb31").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.builder() - .syncMode(SyncMode.SYNC_NONE) - .build(); - db.createColumnFamily("test_cf", cfConfig); - - ColumnFamily cf = db.getColumnFamily("test_cf"); - - // Write some data - try (Transaction txn = db.beginTransaction()) { - txn.put(cf, "key1".getBytes(), "value1".getBytes()); - txn.commit(); - } - - // Force WAL sync - cf.syncWal(); - - // Verify data accessible - try (Transaction txn = db.beginTransaction()) { - byte[] result = txn.get(cf, "key1".getBytes()); - assertNotNull(result); - assertArrayEquals("value1".getBytes(), result); + + @Nested + class Savepoints { + + @Test + void rollsBackToAMark() throws TidesDBException { + try (TidesDB db = openWithCf("sp-rollback", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + try (Transaction txn = db.beginTransaction()) { + txn.put(cf, b("kept"), b("v")); + txn.savepoint("mark"); + txn.put(cf, b("discarded"), b("v")); + txn.rollbackToSavepoint("mark"); + txn.commit(); + } + + assertEquals("v", read(db, cf, "kept")); + assertNull(read(db, cf, "discarded")); } } - } - - @Test - @Order(34) - void testGetDbStats() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb32").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - db.createColumnFamily("cf1", cfConfig); - db.createColumnFamily("cf2", cfConfig); - - ColumnFamily cf1 = db.getColumnFamily("cf1"); - - // Insert some data - try (Transaction txn = db.beginTransaction()) { - for (int i = 0; i < 100; i++) { - txn.put(cf1, ("key" + i).getBytes(), ("value" + i).getBytes()); + + @Test + void releasesAMarkWithoutRollingBack() throws TidesDBException { + try (TidesDB db = openWithCf("sp-release", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + try (Transaction txn = db.beginTransaction()) { + txn.savepoint("mark"); + txn.put(cf, b("kept"), b("v")); + txn.releaseSavepoint("mark"); + txn.commit(); } - txn.commit(); + assertEquals("v", read(db, cf, "kept")); } - - DbStats dbStats = db.getDbStats(); - assertNotNull(dbStats); - assertEquals(2, dbStats.getNumColumnFamilies()); - assertTrue(dbStats.getTotalMemory() > 0); - assertTrue(dbStats.getResolvedMemoryLimit() > 0); - assertTrue(dbStats.getMemoryPressureLevel() >= 0); - assertTrue(dbStats.getGlobalSeq() > 0); - assertTrue(dbStats.getTotalMemtableBytes() >= 0); - assertTrue(dbStats.getTotalSstableCount() >= 0); - assertTrue(dbStats.getTotalDataSizeBytes() >= 0); - } - } - - @Test - @Order(35) - void testGetDbStatsToString() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb33").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - db.createColumnFamily("test_cf", cfConfig); - - DbStats dbStats = db.getDbStats(); - assertNotNull(dbStats); - String str = dbStats.toString(); - assertTrue(str.contains("numColumnFamilies=")); - assertTrue(str.contains("totalMemory=")); } - } - - @Test - @Order(36) - void testUnifiedMemtableConfig() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb_unified").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .unifiedMemtable(true) - .unifiedMemtableWriteBufferSize(0) - .unifiedMemtableSkipListMaxLevel(0) - .unifiedMemtableSkipListProbability(0) - .unifiedMemtableSyncMode(0) - .unifiedMemtableSyncIntervalUs(0) - .build(); - - assertTrue(config.isUnifiedMemtable()); - - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - db.createColumnFamily("test_cf", cfConfig); - - ColumnFamily cf = db.getColumnFamily("test_cf"); - try (Transaction txn = db.beginTransaction()) { - txn.put(cf, "key1".getBytes(), "value1".getBytes()); - txn.commit(); - } + @Test + void nestsMarks() throws TidesDBException { + try (TidesDB db = openWithCf("sp-nested", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + try (Transaction txn = db.beginTransaction()) { + txn.put(cf, b("a"), b("v")); + txn.savepoint("outer"); + txn.put(cf, b("b"), b("v")); + txn.savepoint("inner"); + txn.put(cf, b("c"), b("v")); + txn.rollbackToSavepoint("inner"); + txn.commit(); + } - try (Transaction txn = db.beginTransaction()) { - byte[] result = txn.get(cf, "key1".getBytes()); - assertNotNull(result); - assertArrayEquals("value1".getBytes(), result); + assertEquals("v", read(db, cf, "a")); + assertEquals("v", read(db, cf, "b")); + assertNull(read(db, cf, "c")); } - - DbStats dbStats = db.getDbStats(); - assertNotNull(dbStats); - assertTrue(dbStats.isUnifiedMemtableEnabled()); } - } - @Test - @Order(37) - void testDeleteColumnFamily() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb_delcf").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - db.createColumnFamily("delete_me", cfConfig); - - ColumnFamily cf = db.getColumnFamily("delete_me"); - assertNotNull(cf); - - // Insert some data first - try (Transaction txn = db.beginTransaction()) { - txn.put(cf, "key1".getBytes(), "value1".getBytes()); - txn.commit(); + @Test + void reportsAnUnknownMark() throws TidesDBException { + try (TidesDB db = openWithCf("sp-unknown", "cf")) { + try (Transaction txn = db.beginTransaction()) { + TidesDBException e = assertThrows(TidesDBException.class, + () -> txn.rollbackToSavepoint("never-marked")); + assertEquals(TidesDBException.ERR_NOT_FOUND, e.getErrorCode()); + txn.rollback(); + } } - - // Delete the column family via handle - db.deleteColumnFamily(cf); - - // Verify it's gone - assertThrows(TidesDBException.class, () -> db.getColumnFamily("delete_me")); } - } - - @Test - @Order(38) - void testDeleteColumnFamilyNull() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb_delcf_null").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - try (TidesDB db = TidesDB.open(config)) { - assertThrows(IllegalArgumentException.class, () -> db.deleteColumnFamily(null)); + @Test + void rejectsAnEmptyName() throws TidesDBException { + try (TidesDB db = openWithCf("sp-args", "cf")) { + try (Transaction txn = db.beginTransaction()) { + assertThrows(IllegalArgumentException.class, () -> txn.savepoint(null)); + assertThrows(IllegalArgumentException.class, () -> txn.savepoint("")); + txn.rollback(); + } + } } } - @Test - @Order(39) - void testIteratorKeyValue() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb_kv").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - db.createColumnFamily("test_cf", cfConfig); - - ColumnFamily cf = db.getColumnFamily("test_cf"); + @Nested + class Iterators { + /** Writes {@code count} keys named k000..k(count-1) and commits them. */ + private void seed(TidesDB db, ColumnFamily cf, int count) throws TidesDBException { try (Transaction txn = db.beginTransaction()) { - for (int i = 0; i < 10; i++) { - String key = String.format("key%02d", i); - String value = "value" + i; - txn.put(cf, key.getBytes(), value.getBytes()); + for (int i = 0; i < count; i++) { + txn.put(cf, b(String.format("k%03d", i)), b("v" + i)); } txn.commit(); } + } - // Test combined keyValue() method - try (Transaction txn = db.beginTransaction()) { - try (TidesDBIterator iter = txn.newIterator(cf)) { - iter.seekToFirst(); - - int count = 0; - while (iter.isValid()) { - KeyValue kv = iter.keyValue(); - assertNotNull(kv); - assertNotNull(kv.getKey()); - assertNotNull(kv.getValue()); - count++; - iter.next(); + @Test + void walksForwardInKeyOrder() throws TidesDBException { + try (TidesDB db = openWithCf("iter-forward", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + seed(db, cf, 25); + + List keys = new ArrayList<>(); + try (Transaction txn = db.beginTransaction(); + TidesDBIterator it = txn.newIterator(cf)) { + it.seekToFirst(); + while (it.isValid()) { + keys.add(s(it.key())); + it.next(); } - assertEquals(10, count); + txn.rollback(); } - } - } - } - @Test - @Order(40) - void testDbStatsUnifiedFields() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb_stats_unified").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - db.createColumnFamily("test_cf", cfConfig); - - DbStats dbStats = db.getDbStats(); - assertNotNull(dbStats); - - // With default config, unified memtable should be disabled - assertFalse(dbStats.isUnifiedMemtableEnabled()); - assertFalse(dbStats.isObjectStoreEnabled()); - assertFalse(dbStats.isReplicaMode()); - assertTrue(dbStats.getUnifiedMemtableBytes() >= 0); - assertTrue(dbStats.getUnifiedImmutableCount() >= 0); - assertTrue(dbStats.getLocalCacheBytesUsed() >= 0); - assertTrue(dbStats.getTotalUploads() >= 0); - assertTrue(dbStats.getTotalUploadFailures() >= 0); - - // Single-writer fencing epochs default to 0 outside object-store primary mode - assertEquals(0, dbStats.getPrimaryEpoch()); - assertTrue(dbStats.getSeenEpoch() >= 0); - - // Verify toString includes new fields - String str = dbStats.toString(); - assertTrue(str.contains("unifiedMemtableEnabled=")); - assertTrue(str.contains("objectStoreEnabled=")); - assertTrue(str.contains("replicaMode=")); - assertTrue(str.contains("primaryEpoch=")); - assertTrue(str.contains("seenEpoch=")); + assertEquals(25, keys.size()); + List sorted = new ArrayList<>(keys); + sorted.sort(String::compareTo); + assertEquals(sorted, keys, "keys are ordered byte-wise"); + } } - } - @Test - @Order(41) - void testLogLevelNoneValue() { - assertEquals(99, LogLevel.NONE.getValue()); - assertEquals(LogLevel.NONE, LogLevel.fromValue(99)); - } - - @Test - @Order(21) - void testTransactionResetNullIsolation() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb19").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - db.createColumnFamily("test_cf", cfConfig); - - ColumnFamily cf = db.getColumnFamily("test_cf"); - - Transaction txn = db.beginTransaction(); - txn.put(cf, "key1".getBytes(), "value1".getBytes()); - txn.commit(); + @Test + void walksBackwardOverTheSameRows() throws TidesDBException { + try (TidesDB db = openWithCf("iter-backward", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + seed(db, cf, 25); - // Null isolation level should throw IllegalArgumentException - assertThrows(IllegalArgumentException.class, () -> txn.reset(null)); + List backward = new ArrayList<>(); + try (Transaction txn = db.beginTransaction(); + TidesDBIterator it = txn.newIterator(cf)) { + it.seekToLast(); + while (it.isValid()) { + backward.add(s(it.key())); + it.prev(); + } + txn.rollback(); + } - txn.free(); + assertEquals(25, backward.size()); + List forward = new ArrayList<>(backward); + java.util.Collections.reverse(forward); + List sorted = new ArrayList<>(forward); + sorted.sort(String::compareTo); + assertEquals(sorted, forward); + } + } + + @Test + void readsKeyAndValueInOneCall() throws TidesDBException { + try (TidesDB db = openWithCf("iter-kv", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + seed(db, cf, 5); + + try (Transaction txn = db.beginTransaction(); + TidesDBIterator it = txn.newIterator(cf)) { + it.seekToFirst(); + assertTrue(it.isValid()); + KeyValue kv = it.keyValue(); + assertEquals(s(it.key()), s(kv.getKey())); + assertEquals(s(it.value()), s(kv.getValue())); + txn.rollback(); + } + } } - } - - @Test - @Order(42) - void testTransactionSingleDelete() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb_single_delete").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - db.createColumnFamily("test_cf", cfConfig); - ColumnFamily cf = db.getColumnFamily("test_cf"); + @Test + void seeksToTheFirstKeyAtOrAfterATarget() throws TidesDBException { + try (TidesDB db = openWithCf("iter-seek", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + seed(db, cf, 30); - byte[] key = "single_key".getBytes(StandardCharsets.UTF_8); - byte[] value = "single_value".getBytes(StandardCharsets.UTF_8); + try (Transaction txn = db.beginTransaction(); + TidesDBIterator it = txn.newIterator(cf)) { + it.seek(b("k010")); + assertTrue(it.isValid()); + assertEquals("k010", s(it.key())); - try (Transaction txn = db.beginTransaction()) { - txn.put(cf, key, value); - txn.commit(); + it.seekForPrev(b("k010")); + assertTrue(it.isValid()); + assertEquals("k010", s(it.key())); + txn.rollback(); + } } + } - try (Transaction txn = db.beginTransaction()) { - byte[] result = txn.get(cf, key); - assertNotNull(result); - assertArrayEquals(value, result); - } + @Test + void leavesTheCursorInvalidPastTheEnd() throws TidesDBException { + try (TidesDB db = openWithCf("iter-past-end", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + seed(db, cf, 5); - try (Transaction txn = db.beginTransaction()) { - txn.singleDelete(cf, key); - txn.commit(); + try (Transaction txn = db.beginTransaction(); + TidesDBIterator it = txn.newIterator(cf)) { + assertDoesNotThrow(() -> it.seek(b("zzzz"))); + assertFalse(it.isValid(), "seeking past the end is not an error"); + txn.rollback(); + } } + } - try (Transaction txn = db.beginTransaction()) { - assertThrows(TidesDBException.class, () -> txn.get(cf, key)); + @Test + void isImmediatelyInvalidOnAnEmptyFamily() throws TidesDBException { + try (TidesDB db = openWithCf("iter-empty", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + try (Transaction txn = db.beginTransaction(); + TidesDBIterator it = txn.newIterator(cf)) { + assertDoesNotThrow(it::seekToFirst); + assertFalse(it.isValid()); + assertDoesNotThrow(it::seekToLast); + assertFalse(it.isValid()); + txn.rollback(); + } } } - } - - @Test - @Order(44) - void testTombstoneCfConfigRoundTrip() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb_tombstone_cfg").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.builder() - .tombstoneDensityTrigger(0.5) - .tombstoneDensityMinEntries(256) - .build(); - db.createColumnFamily("ts_cf", cfConfig); - ColumnFamily cf = db.getColumnFamily("ts_cf"); + @Test + void scansOnlyTheRangeItWasGiven() throws TidesDBException { + try (TidesDB db = openWithCf("iter-range", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + seed(db, cf, 100); + db.flushMemtable(); - ColumnFamilyConfig readback = cf.getStats().getConfig(); - assertNotNull(readback); - assertEquals(0.5, readback.getTombstoneDensityTrigger(), 0.0); - assertEquals(256L, readback.getTombstoneDensityMinEntries()); + List keys = new ArrayList<>(); + try (Transaction txn = db.beginTransaction(); + TidesDBIterator it = txn.newRangeIterator(cf, b("k020"), b("k030"))) { + it.seek(b("k020")); + while (it.isValid() && s(it.key()).compareTo("k030") < 0) { + keys.add(s(it.key())); + it.next(); + } + txn.rollback(); + } - // Defaults from the C library should be sensible (min entries ~= 1024) - ColumnFamilyConfig defaults = ColumnFamilyConfig.defaultConfig(); - assertTrue(defaults.getTombstoneDensityMinEntries() > 0, - "default tombstoneDensityMinEntries should be non-zero (sourced from C library)"); - assertTrue(defaults.getTombstoneDensityTrigger() >= 0.0 - && defaults.getTombstoneDensityTrigger() <= 1.0, - "default tombstoneDensityTrigger should be in [0.0, 1.0]"); + assertEquals(10, keys.size()); + assertEquals("k020", keys.get(0)); + assertEquals("k029", keys.get(keys.size() - 1)); + } } - } - @Test - @Order(45) - void testTombstoneStatsPopulated() throws TidesDBException, InterruptedException { - Config config = Config.builder(tempDir.resolve("testdb_tombstone_stats").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - db.createColumnFamily("ts_stats_cf", ColumnFamilyConfig.defaultConfig()); - ColumnFamily cf = db.getColumnFamily("ts_stats_cf"); - - final int n = 200; - try (Transaction txn = db.beginTransaction()) { - for (int i = 0; i < n; i++) { - txn.put(cf, ("key" + i).getBytes(StandardCharsets.UTF_8), - ("value" + i).getBytes(StandardCharsets.UTF_8)); + @Test + void refusesOperationsOnAFreedIterator() throws TidesDBException { + try (TidesDB db = openWithCf("iter-freed", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + seed(db, cf, 3); + + try (Transaction txn = db.beginTransaction()) { + TidesDBIterator it = txn.newIterator(cf); + it.free(); + it.free(); + + assertFalse(it.isValid(), "a freed iterator reports invalid rather than throwing"); + assertThrows(IllegalStateException.class, it::next); + assertThrows(IllegalStateException.class, it::key); + txn.rollback(); } - txn.commit(); } - cf.flushMemtable(); + } - try (Transaction txn = db.beginTransaction()) { - for (int i = 0; i < n / 2; i++) { - txn.delete(cf, ("key" + i).getBytes(StandardCharsets.UTF_8)); + @Test + void rejectsMissingRangeBounds() throws TidesDBException { + try (TidesDB db = openWithCf("iter-range-args", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + try (Transaction txn = db.beginTransaction()) { + assertThrows(IllegalArgumentException.class, + () -> txn.newRangeIterator(cf, null, b("z"))); + assertThrows(IllegalArgumentException.class, + () -> txn.newRangeIterator(cf, b("a"), new byte[0])); + txn.rollback(); } - txn.commit(); } - cf.flushMemtable(); - - // Wait for the flush to land so the stats include the tombstones - Thread.sleep(500); - - Stats stats = cf.getStats(); - assertNotNull(stats); - assertTrue(stats.getTotalTombstones() > 0, - "expected total_tombstones > 0 after deletes + flush"); - assertTrue(stats.getTombstoneRatio() >= 0.0 && stats.getTombstoneRatio() <= 1.0, - "tombstone_ratio must be within [0.0, 1.0]"); - assertTrue(stats.getMaxSstDensity() >= 0.0 && stats.getMaxSstDensity() <= 1.0, - "max_sst_density must be within [0.0, 1.0]"); - assertTrue(stats.getMaxSstDensityLevel() >= 0, - "max_sst_density_level must be non-negative"); - - long[] perLevel = stats.getLevelTombstoneCounts(); - assertNotNull(perLevel, "level_tombstone_counts must be populated"); - assertEquals(stats.getNumLevels(), perLevel.length, - "level_tombstone_counts length must match num_levels"); } } - @Test - @Order(46) - void testCompactRange() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb_compact_range").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - // Small write buffer keeps each batch falling into its own SSTable - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.builder() - .writeBufferSize(64 * 1024) - .build(); - db.createColumnFamily("range_cf", cfConfig); - ColumnFamily cf = db.getColumnFamily("range_cf"); + @Nested + class Isolation { - // Multi-batch insert + flush to spread keys across several SSTables - for (int batch = 0; batch < 4; batch++) { - try (Transaction txn = db.beginTransaction()) { - for (int i = 0; i < 50; i++) { - int n = batch * 50 + i; - byte[] key = String.format("k%05d", n).getBytes(StandardCharsets.UTF_8); - byte[] value = ("value" + n).getBytes(StandardCharsets.UTF_8); - txn.put(cf, key, value); + @Test + void beginsAtEveryLevel() throws TidesDBException { + try (TidesDB db = openWithCf("iso-levels", "cf")) { + for (IsolationLevel level : IsolationLevel.values()) { + try (Transaction txn = db.beginTransaction(level)) { + assertEquals(TransactionState.ACTIVE, txn.state()); + txn.rollback(); } - txn.commit(); } - cf.flushMemtable(); } + } - // Narrow range compaction over a slice of the keyspace - byte[] start = "k00050".getBytes(StandardCharsets.UTF_8); - byte[] end = "k00100".getBytes(StandardCharsets.UTF_8); - cf.compactRange(start, end); - - // Both endpoints null should be rejected with INVALID_ARGS - TidesDBException ex = assertThrows(TidesDBException.class, - () -> cf.compactRange(null, null)); - assertEquals(-2, ex.getErrorCode(), "expected TDB_ERR_INVALID_ARGS for both-null range"); - - // Both empty should also be rejected - assertThrows(TidesDBException.class, - () -> cf.compactRange(new byte[0], new byte[0])); + @Test + void takesTheFamilyDefault() throws TidesDBException { + try (TidesDB db = open("iso-cf-default")) { + db.createColumnFamily("cf", ColumnFamilyConfig.builder() + .defaultIsolationLevel(IsolationLevel.SERIALIZABLE) + .build()); + ColumnFamily cf = db.getColumnFamily("cf"); - // A key outside the compacted range must still read back unchanged - try (Transaction txn = db.beginTransaction()) { - byte[] outside = txn.get(cf, "k00150".getBytes(StandardCharsets.UTF_8)); - assertNotNull(outside); - assertArrayEquals("value150".getBytes(StandardCharsets.UTF_8), outside); + try (Transaction txn = db.beginTransaction(cf)) { + assertEquals(TransactionState.ACTIVE, txn.state()); + txn.rollback(); + } } } - } - @Test - @Order(47) - void testMaxConcurrentFlushes() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb_max_flushes").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .maxConcurrentFlushes(1) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - db.createColumnFamily("flush_cf", ColumnFamilyConfig.defaultConfig()); - ColumnFamily cf = db.getColumnFamily("flush_cf"); + @Test + void refusesTheSecondCommitterOnAContendedKey() throws TidesDBException { + try (TidesDB db = openWithCf("iso-conflict", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); - try (Transaction txn = db.beginTransaction()) { - txn.put(cf, "k".getBytes(StandardCharsets.UTF_8), - "v".getBytes(StandardCharsets.UTF_8)); - txn.commit(); + try (Transaction first = db.beginTransaction(IsolationLevel.SNAPSHOT); + Transaction second = db.beginTransaction(IsolationLevel.SNAPSHOT)) { + first.put(cf, b("contended"), b("first")); + second.put(cf, b("contended"), b("second")); + + first.commit(); + TidesDBException e = assertThrows(TidesDBException.class, second::commit); + assertEquals(TidesDBException.ERR_CONFLICT, e.getErrorCode()); + } + assertEquals("first", read(db, cf, "contended")); } - cf.flushMemtable(); } - // defaultConfig() should source maxConcurrentFlushes from the C library. The engine's - // default is 0, which is the "auto" sentinel meaning "pin to the resolved - // num_flush_threads" -- so the only invariant we can assert is that it is non-negative. - Config defaults = Config.defaultConfig(); - assertTrue(defaults.getMaxConcurrentFlushes() >= 0, - "default maxConcurrentFlushes should be sourced from tidesdb_default_config()"); - } - - @Test - @Order(43) - void testTransactionSingleDeleteNullArgs() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb_single_delete_null").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); + @Test + void holdsAFrozenCeilingUnderRepeatableRead() throws TidesDBException { + try (TidesDB db = openWithCf("iso-repeatable", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + write(db, cf, "key", "before"); - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - db.createColumnFamily("test_cf", cfConfig); + try (Transaction reader = db.beginTransaction(IsolationLevel.REPEATABLE_READ)) { + assertEquals("before", s(reader.get(cf, b("key")))); + write(db, cf, "key", "after"); + assertEquals("before", s(reader.get(cf, b("key"))), + "the ceiling was frozen when the transaction began"); + reader.rollback(); + } + assertEquals("after", read(db, cf, "key")); + } + } - ColumnFamily cf = db.getColumnFamily("test_cf"); + @Test + void reportsAReadCeiling() throws TidesDBException { + try (TidesDB db = openWithCf("iso-ceiling", "cf")) { + // a committed write advances the global sequence, so the frozen + // ceiling below is something other than the initial zero + write(db, db.getColumnFamily("cf"), "seed", "v"); - try (Transaction txn = db.beginTransaction()) { - assertThrows(IllegalArgumentException.class, - () -> txn.singleDelete(null, "k".getBytes())); - assertThrows(IllegalArgumentException.class, - () -> txn.singleDelete(cf, null)); - assertThrows(IllegalArgumentException.class, - () -> txn.singleDelete(cf, new byte[0])); + try (Transaction txn = db.beginTransaction(IsolationLevel.REPEATABLE_READ)) { + assertTrue(txn.getReadSnapshot() > 0); + txn.rollback(); + } + try (Transaction txn = db.beginTransaction(IsolationLevel.READ_UNCOMMITTED)) { + assertEquals(-1L, txn.getReadSnapshot(), + "read-uncommitted filters at an unsigned UINT64_MAX"); + txn.rollback(); + } } } - } - @Test - @Order(48) - void testRaiseOpenFileLimit() { - // Reporting-only call (desired <= 0) returns the current ceiling without changing it. - long current = TidesDB.raiseOpenFileLimit(0); - assertTrue(current > 0, "current open-file ceiling should be positive"); + @Test + void resetsForReuse() throws TidesDBException { + try (TidesDB db = openWithCf("iso-reset", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + try (Transaction txn = db.beginTransaction()) { + txn.put(cf, b("first-use"), b("v")); + txn.commit(); - // A raise attempt is non-fatal and returns the ceiling in effect afterwards (>= current). - long after = TidesDB.raiseOpenFileLimit(current); - assertTrue(after >= current, "ceiling after a raise attempt should not be lower"); - } + txn.reset(IsolationLevel.SERIALIZABLE); + assertEquals(TransactionState.ACTIVE, txn.state()); - @Test - @Order(49) - void testCancelBackgroundWork() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb_cancel_bg").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - db.createColumnFamily("cancel_cf", ColumnFamilyConfig.defaultConfig()); - ColumnFamily cf = db.getColumnFamily("cancel_cf"); + txn.put(cf, b("second-use"), b("v")); + txn.commit(); + } - try (Transaction txn = db.beginTransaction()) { - txn.put(cf, "k".getBytes(StandardCharsets.UTF_8), - "v".getBytes(StandardCharsets.UTF_8)); - txn.commit(); + assertEquals("v", read(db, cf, "first-use")); + assertEquals("v", read(db, cf, "second-use")); } - - // Sticky db-wide cancel of background compaction; flushes are unaffected. - assertDoesNotThrow(db::cancelBackgroundWork); } - } - @Test - @Order(50) - void testFinishCompactionsOnClose() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb_finish_compactions").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .finishCompactionsOnClose(true) - .build(); - - assertTrue(config.isFinishCompactionsOnClose()); - - try (TidesDB db = TidesDB.open(config)) { - db.createColumnFamily("finish_cf", ColumnFamilyConfig.defaultConfig()); - ColumnFamily cf = db.getColumnFamily("finish_cf"); - try (Transaction txn = db.beginTransaction()) { - txn.put(cf, "k".getBytes(StandardCharsets.UTF_8), - "v".getBytes(StandardCharsets.UTF_8)); - txn.commit(); + @Test + void rejectsANullLevel() throws TidesDBException { + try (TidesDB db = openWithCf("iso-args", "cf")) { + assertThrows(IllegalArgumentException.class, + () -> db.beginTransaction((IsolationLevel) null)); + assertThrows(IllegalArgumentException.class, + () -> db.beginTransaction((ColumnFamily) null)); + try (Transaction txn = db.beginTransaction()) { + assertThrows(IllegalArgumentException.class, () -> txn.reset(null)); + txn.rollback(); + } } - cf.flushMemtable(); } - // close() returning without error is the observable contract for this flag. - } - - @Test - @Order(51) - void testCfConfigIniRoundTrip() throws TidesDBException { - String iniFile = tempDir.resolve("cf_config.ini").toString(); - String section = "round_trip_cf"; - - ColumnFamilyConfig original = ColumnFamilyConfig.builder() - .writeBufferSize(96 * 1024 * 1024) - .levelSizeRatio(8) - .minLevels(4) - .klogValueThreshold(1024) - .compressionAlgorithm(CompressionAlgorithm.ZSTD_COMPRESSION) - .enableBloomFilter(true) - .bloomFPR(0.02) - .enableBlockIndexes(true) - .indexSampleRatio(2) - .blockIndexPrefixLen(8) - .syncMode(SyncMode.SYNC_INTERVAL) - .syncIntervalUs(250000) - .defaultIsolationLevel(IsolationLevel.SNAPSHOT) - .l1FileCountTrigger(6) - .l0QueueStallThreshold(15) - .tombstoneDensityTrigger(0.4) - .tombstoneDensityMinEntries(2048) - .minDiskSpace(50 * 1024 * 1024) - .useBtree(true) - .objectLazyCompaction(true) - .objectPrefetchCompaction(false) - .build(); - - original.saveToIni(iniFile, section); - - ColumnFamilyConfig loaded = ColumnFamilyConfig.loadFromIni(iniFile, section); - - assertEquals(original.getWriteBufferSize(), loaded.getWriteBufferSize()); - assertEquals(original.getLevelSizeRatio(), loaded.getLevelSizeRatio()); - assertEquals(original.getMinLevels(), loaded.getMinLevels()); - assertEquals(original.getKlogValueThreshold(), loaded.getKlogValueThreshold()); - assertEquals(original.getCompressionAlgorithm(), loaded.getCompressionAlgorithm()); - assertEquals(original.isEnableBloomFilter(), loaded.isEnableBloomFilter()); - assertEquals(original.getBloomFPR(), loaded.getBloomFPR(), 1e-9); - assertEquals(original.isEnableBlockIndexes(), loaded.isEnableBlockIndexes()); - assertEquals(original.getIndexSampleRatio(), loaded.getIndexSampleRatio()); - assertEquals(original.getBlockIndexPrefixLen(), loaded.getBlockIndexPrefixLen()); - assertEquals(original.getSyncMode(), loaded.getSyncMode()); - assertEquals(original.getSyncIntervalUs(), loaded.getSyncIntervalUs()); - assertEquals(original.getDefaultIsolationLevel(), loaded.getDefaultIsolationLevel()); - assertEquals(original.getL1FileCountTrigger(), loaded.getL1FileCountTrigger()); - assertEquals(original.getL0QueueStallThreshold(), loaded.getL0QueueStallThreshold()); - assertEquals(original.getTombstoneDensityTrigger(), loaded.getTombstoneDensityTrigger(), 1e-9); - assertEquals(original.getTombstoneDensityMinEntries(), loaded.getTombstoneDensityMinEntries()); - assertEquals(original.getMinDiskSpace(), loaded.getMinDiskSpace()); - assertEquals(original.isUseBtree(), loaded.isUseBtree()); - assertEquals(original.isObjectLazyCompaction(), loaded.isObjectLazyCompaction()); - assertEquals(original.isObjectPrefetchCompaction(), loaded.isObjectPrefetchCompaction()); } - @Test - @Order(52) - void testCfConfigIniInvalidArgs() { - assertThrows(IllegalArgumentException.class, - () -> ColumnFamilyConfig.defaultConfig().saveToIni(null, "s")); - assertThrows(IllegalArgumentException.class, - () -> ColumnFamilyConfig.defaultConfig().saveToIni("f.ini", "")); - assertThrows(IllegalArgumentException.class, - () -> ColumnFamilyConfig.loadFromIni("", "s")); - assertThrows(IllegalArgumentException.class, - () -> ColumnFamilyConfig.loadFromIni("f.ini", null)); - // Reading a non-existent INI file surfaces as a TidesDBException, not a crash. - assertThrows(TidesDBException.class, - () -> ColumnFamilyConfig.loadFromIni( - tempDir.resolve("does_not_exist.ini").toString(), "nope")); - } + @Nested + class TimeoutsAndAborts { - @Test - @Order(53) - void testWriteAmplificationCounters() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb_write_amp").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - db.createColumnFamily("wa_cf", ColumnFamilyConfig.defaultConfig()); - ColumnFamily cf = db.getColumnFamily("wa_cf"); - - for (int i = 0; i < 200; i++) { + @Test + void setsAndClearsATimeout() throws TidesDBException { + try (TidesDB db = openWithCf("abort-timeout", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); try (Transaction txn = db.beginTransaction()) { - txn.put(cf, ("key" + i).getBytes(StandardCharsets.UTF_8), - ("value" + i).getBytes(StandardCharsets.UTF_8)); + txn.setTimeout(3600); + txn.put(cf, b("k"), b("v")); + txn.setTimeout(0); txn.commit(); } + assertEquals("v", read(db, cf, "k")); } - cf.flushMemtable(); - cf.purge(); - - Stats stats = cf.getStats(); - // Counters are non-negative and user bytes reflect the committed payload. - assertTrue(stats.getUserBytesWritten() > 0, "user bytes should be recorded"); - assertTrue(stats.getWalBytesWritten() >= 0); - assertTrue(stats.getFlushBytesWritten() >= 0); - assertTrue(stats.getCompactionBytesWritten() >= 0); - assertTrue(stats.getCompactionBytesRead() >= 0); - assertTrue(stats.getFlushCount() >= 0); - assertTrue(stats.getCompactionCount() >= 0); - - DbStats dbStats = db.getDbStats(); - assertTrue(dbStats.getUserBytesWritten() > 0, "db-wide user bytes should be recorded"); - assertTrue(dbStats.getUwalBytesWritten() >= 0); - assertTrue(dbStats.getWalBytesWritten() >= 0); - assertTrue(dbStats.getFlushBytesWritten() >= 0); - assertTrue(dbStats.getCompactionBytesWritten() >= 0); - assertTrue(dbStats.getCompactionBytesRead() >= 0); - assertTrue(dbStats.getFlushCount() >= 0); - assertTrue(dbStats.getCompactionCount() >= 0); } - } - @Test - @Order(54) - void testS3ConfigBuilderValidation() { - // Required fields must be present - assertThrows(IllegalArgumentException.class, () -> S3Config.builder().build()); - assertThrows(IllegalArgumentException.class, - () -> S3Config.builder().endpoint("s3.amazonaws.com").build()); - assertThrows(IllegalArgumentException.class, - () -> S3Config.builder().endpoint("s3.amazonaws.com").bucket("b").build()); - assertThrows(IllegalArgumentException.class, - () -> S3Config.builder().endpoint("s3.amazonaws.com").bucket("b").accessKey("ak").build()); - - // A fully specified config builds and exposes its values, with secure defaults - S3Config s3 = S3Config.builder() - .endpoint("s3.amazonaws.com") - .bucket("my-bucket") - .prefix("prod/db1/") - .accessKey("AKID") - .secretKey("SECRET") - .region("us-east-1") - .build(); - assertEquals("s3.amazonaws.com", s3.getEndpoint()); - assertEquals("my-bucket", s3.getBucket()); - assertEquals("prod/db1/", s3.getPrefix()); - assertEquals("us-east-1", s3.getRegion()); - assertTrue(s3.isUseSsl(), "TLS should be on by default"); - assertFalse(s3.isUsePathStyle(), "virtual-hosted by default"); - assertFalse(s3.isTlsInsecureSkipVerify(), "TLS verification on by default"); - assertEquals(0, s3.getMultipartThreshold()); - assertEquals(0, s3.getMultipartPartSize()); - } + @Test + void expiresATransactionPastItsDeadline() throws Exception { + try (TidesDB db = openWithCf("abort-expired", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + try (Transaction txn = db.beginTransaction()) { + txn.setTimeout(1); + // the engine ages a transaction against a clock a background + // ticker publishes once a second, and expiry is strictly past + // the deadline, so a one second timeout needs more than two + // seconds of wall clock to be observed whatever the tick phase + Thread.sleep(3200); + + TidesDBException e = + assertThrows(TidesDBException.class, () -> txn.put(cf, b("k"), b("v"))); + assertEquals(TidesDBException.ERR_TXN_EXPIRED, e.getErrorCode()); + } + } + } - @Test - @Order(55) - void testS3Availability() { - // Probe must be callable and return a definite boolean without throwing. - boolean available = TidesDB.isS3Available(); - assertTrue(available || !available); - } + @Test + void stopsATransactionOnRequest() throws TidesDBException { + try (TidesDB db = openWithCf("abort-request", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + try (Transaction txn = db.beginTransaction()) { + txn.put(cf, b("k"), b("v")); + txn.requestAbort(); - @Test - @Order(56) - void testOpenWithS3Config() throws TidesDBException { - S3Config s3 = S3Config.builder() - .endpoint("127.0.0.1:9000") - .bucket("tidesdb-test") - .accessKey("minioadmin") - .secretKey("minioadmin") - .usePathStyle(true) - .useSsl(false) - .build(); - - Config config = Config.builder(tempDir.resolve("testdb_s3").toString()) - .objectStoreS3Config(s3) - .build(); - - if (TidesDB.isS3Available()) { - // With a built-in S3 backend but no live MinIO/S3 endpoint, connector creation or - // open is expected to fail -- but it must surface as a TidesDBException, not a crash. - assertThrows(TidesDBException.class, () -> { TidesDB.open(config).close(); }); - } else { - // No S3 support compiled in: opening must throw a clear, catchable exception. - TidesDBException ex = - assertThrows(TidesDBException.class, () -> TidesDB.open(config)); - assertTrue(ex.getMessage().toLowerCase().contains("s3"), - "exception should explain S3 is unavailable, was: " + ex.getMessage()); + TidesDBException e = assertThrows(TidesDBException.class, txn::commit); + assertEquals(TidesDBException.ERR_TXN_ABORTED, e.getErrorCode(), + "an outside ruling is distinct from the engine's own conflict verdict"); + } + assertNull(read(db, cf, "k")); + } } - } - @Test - @Order(57) - void testCommitHookReplaceAndVerify() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb_hook_replace").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); + @Test + void abortsFromAnotherThread() throws Exception { + try (TidesDB db = openWithCf("abort-cross-thread", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + Transaction txn = db.beginTransaction(); + txn.put(cf, b("k"), b("v")); + + Thread aborter = new Thread(txn::requestAbort); + aborter.start(); + aborter.join(); - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - db.createColumnFamily("test_cf", cfConfig); + assertThrows(TidesDBException.class, txn::commit); + txn.free(); + } + } + } - ColumnFamily cf = db.getColumnFamily("test_cf"); + @Nested + class Snapshots { - List receivedA = new ArrayList<>(); + @Test + void readsAsOfThePointItNamed() throws TidesDBException { + try (TidesDB db = openWithCf("snap-read", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + write(db, cf, "key", "before"); - cf.setCommitHook((ops, commitSeq) -> { - receivedA.add(ops); - return 0; - }); + try (Snapshot snapshot = db.createSnapshot()) { + assertTrue(snapshot.getSeq() > 0); + write(db, cf, "key", "after"); - // Commit first put -- hookA should fire - try (Transaction txn = db.beginTransaction()) { - txn.put(cf, "keyA".getBytes(StandardCharsets.UTF_8), - "valueA".getBytes(StandardCharsets.UTF_8)); - txn.commit(); + try (Transaction txn = db.beginTransactionAtSnapshot(snapshot)) { + assertEquals("before", s(txn.get(cf, b("key")))); + txn.rollback(); + } + assertEquals("after", read(db, cf, "key")); + } } + } - assertEquals(1, receivedA.size()); - assertEquals(1, receivedA.get(0).length); - assertArrayEquals("keyA".getBytes(StandardCharsets.UTF_8), - receivedA.get(0)[0].getKey()); - assertArrayEquals("valueA".getBytes(StandardCharsets.UTF_8), - receivedA.get(0)[0].getValue()); + @Test + void readsAsOfAnExplicitSequence() throws TidesDBException { + try (TidesDB db = openWithCf("snap-seq", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + write(db, cf, "key", "before"); - // Replace with a second hook - List receivedB = new ArrayList<>(); + try (Snapshot snapshot = db.createSnapshot()) { + write(db, cf, "key", "after"); - cf.setCommitHook((ops, commitSeq) -> { - receivedB.add(ops); - return 0; - }); + try (Transaction txn = db.beginTransactionAtSeq(snapshot.getSeq())) { + assertEquals("before", s(txn.get(cf, b("key")))); + txn.rollback(); + } + } + } + } - // Commit second put -- hookB should fire, hookA should NOT fire again - try (Transaction txn = db.beginTransaction()) { - txn.put(cf, "keyB".getBytes(StandardCharsets.UTF_8), - "valueB".getBytes(StandardCharsets.UTF_8)); - txn.commit(); + @Test + void releaseIsIdempotent() throws TidesDBException { + try (TidesDB db = open("snap-release")) { + Snapshot snapshot = db.createSnapshot(); + long seq = snapshot.getSeq(); + + snapshot.release(); + snapshot.release(); + + assertTrue(snapshot.isReleased()); + assertEquals(seq, snapshot.getSeq(), "the sequence stays readable after release"); + assertThrows(IllegalStateException.class, + () -> db.beginTransactionAtSnapshot(snapshot)); } + } - assertEquals(1, receivedA.size(), - "Old hook should not fire again after replacement"); - assertEquals(1, receivedB.size()); - assertEquals(1, receivedB.get(0).length); - assertArrayEquals("keyB".getBytes(StandardCharsets.UTF_8), - receivedB.get(0)[0].getKey()); - assertArrayEquals("valueB".getBytes(StandardCharsets.UTF_8), - receivedB.get(0)[0].getValue()); + @Test + void reportsTheOldestReadableSequence() throws TidesDBException { + try (TidesDB db = openWithCf("snap-floor", "cf")) { + assertDoesNotThrow(db::getOldestReadableSeq); + } + } - cf.clearCommitHook(); + @Test + void rejectsANullSnapshot() throws TidesDBException { + try (TidesDB db = open("snap-args")) { + assertThrows(IllegalArgumentException.class, + () -> db.beginTransactionAtSnapshot(null)); + } } } - @Test - @Order(58) - void testCommitHookOldHookSurvivesFailure() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb_hook_survive").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - db.createColumnFamily("test_cf", cfConfig); + @Nested + class TwoPhaseCommit { - ColumnFamily cf = db.getColumnFamily("test_cf"); + @Test + void appliesAPreparedTransactionOnCommit() throws TidesDBException { + try (TidesDB db = openWithCf("2pc-commit", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); - List receivedA = new ArrayList<>(); + try (Transaction txn = db.beginTransaction()) { + txn.put(cf, b("decided"), b("v")); + txn.prepare(b("xid-1")); + assertEquals(TransactionState.PREPARED, txn.state()); - cf.setCommitHook((ops, commitSeq) -> { - receivedA.add(ops); - return 0; - }); + assertNull(read(db, cf, "decided"), "a prepared write stays invisible"); - // Commit a put -- hookA fires - try (Transaction txn = db.beginTransaction()) { - txn.put(cf, "key1".getBytes(StandardCharsets.UTF_8), - "value1".getBytes(StandardCharsets.UTF_8)); - txn.commit(); + txn.commitPrepared(); + assertEquals(TransactionState.COMMITTED, txn.state()); + } + assertEquals("v", read(db, cf, "decided")); } - assertEquals(1, receivedA.size()); + } - // Clear the hook - cf.clearCommitHook(); + @Test + void discardsAPreparedTransactionOnRollback() throws TidesDBException { + try (TidesDB db = openWithCf("2pc-rollback", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); - // Commit another put -- hookA should NOT fire - try (Transaction txn = db.beginTransaction()) { - txn.put(cf, "key2".getBytes(StandardCharsets.UTF_8), - "value2".getBytes(StandardCharsets.UTF_8)); - txn.commit(); + try (Transaction txn = db.beginTransaction()) { + txn.put(cf, b("undecided"), b("v")); + txn.prepare(b("xid-2")); + txn.rollbackPrepared(); + assertEquals(TransactionState.ABORTED, txn.state()); + } + assertNull(read(db, cf, "undecided")); } - assertEquals(1, receivedA.size(), - "Hook should not fire after clearing"); - - // Re-register hookA - cf.setCommitHook((ops, commitSeq) -> { - receivedA.add(ops); - return 0; - }); + } - // Commit a third put -- re-registered hook should fire - try (Transaction txn = db.beginTransaction()) { - txn.put(cf, "key3".getBytes(StandardCharsets.UTF_8), - "value3".getBytes(StandardCharsets.UTF_8)); - txn.commit(); + @Test + void refusesPhaseTwoOnAnUnpreparedTransaction() throws TidesDBException { + try (TidesDB db = openWithCf("2pc-unprepared", "cf")) { + try (Transaction txn = db.beginTransaction()) { + assertThrows(TidesDBException.class, txn::commitPrepared); + txn.rollback(); + } } - assertEquals(2, receivedA.size(), - "Re-registered hook should fire"); - assertEquals(1, receivedA.get(1).length); - assertArrayEquals("key3".getBytes(StandardCharsets.UTF_8), - receivedA.get(1)[0].getKey()); + } - cf.clearCommitHook(); + @Test + void listsNothingInDoubtAfterACleanRun() throws TidesDBException { + try (TidesDB db = openWithCf("2pc-recover", "cf")) { + PreparedTransaction[] inDoubt = db.recoverPrepared(); + assertNotNull(inDoubt); + assertEquals(0, inDoubt.length); + } } - } - @Test - @Order(59) - void testCommitHookConcurrentReplaceAndCommit() throws TidesDBException, InterruptedException { - Config config = Config.builder(tempDir.resolve("testdb_hook_concurrent").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - db.createColumnFamily("test_cf", cfConfig); - - ColumnFamily cf = db.getColumnFamily("test_cf"); - - AtomicInteger hookCounter = new AtomicInteger(0); - CountDownLatch startLatch = new CountDownLatch(1); - CountDownLatch stopLatch = new CountDownLatch(1); - - // Set an initial hook - cf.setCommitHook((ops, commitSeq) -> { - hookCounter.incrementAndGet(); - return 0; - }); - - // Writer threads that commit puts in a loop - int numWriters = 4; - Thread[] writers = new Thread[numWriters]; - for (int w = 0; w < numWriters; w++) { - final int writerId = w; - writers[w] = new Thread(() -> { - try { - startLatch.await(); - } catch (InterruptedException e) { - return; - } - while (stopLatch.getCount() > 0) { - try { - try (Transaction txn = db.beginTransaction()) { - byte[] key = ("concurrent_key_" + writerId + "_" + - System.nanoTime()).getBytes(StandardCharsets.UTF_8); - txn.put(cf, key, "value".getBytes(StandardCharsets.UTF_8)); - txn.commit(); - } - } catch (TidesDBException e) { - // Expected during hook transitions - } catch (IllegalStateException e) { - // May occur if db is being closed - break; - } - } - }); - writers[w].setDaemon(true); - writers[w].start(); - } - - // Replacer thread that alternates set and clear hook - Thread replacer = new Thread(() -> { - try { - startLatch.await(); - } catch (InterruptedException e) { - return; - } - for (int i = 0; i < 100 && stopLatch.getCount() > 0; i++) { - try { - if (i % 2 == 0) { - cf.setCommitHook((ops, commitSeq) -> { - hookCounter.incrementAndGet(); - return 0; - }); - } else { - cf.clearCommitHook(); - } - } catch (TidesDBException e) { - // Expected during transitions - } catch (IllegalStateException e) { - break; - } + @Test + void rejectsAnEmptyXid() throws TidesDBException { + try (TidesDB db = openWithCf("2pc-args", "cf")) { + try (Transaction txn = db.beginTransaction()) { + assertThrows(IllegalArgumentException.class, () -> txn.prepare(null)); + assertThrows(IllegalArgumentException.class, () -> txn.prepare(new byte[0])); + txn.rollback(); } - }); - replacer.setDaemon(true); - replacer.start(); - - // Start all threads - startLatch.countDown(); - - // Let them run for 3 seconds - Thread.sleep(3000); + } + } + } - // Signal stop - stopLatch.countDown(); + @Nested + class CommitHooks { - // Wait for threads to finish - for (Thread w : writers) { - w.join(5000); - } - replacer.join(5000); + @Test + void deliversTheCommittedBatch() throws TidesDBException { + try (TidesDB db = openWithCf("hook-batch", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + AtomicInteger opCount = new AtomicInteger(); + AtomicLong lastSeq = new AtomicLong(); + List keys = java.util.Collections.synchronizedList(new ArrayList<>()); - // No crash occurred -- verify final hook state is coherent - // Do a final commit to verify no crash - try { cf.setCommitHook((ops, commitSeq) -> { - hookCounter.incrementAndGet(); + opCount.addAndGet(ops.length); + lastSeq.set(commitSeq); + for (CommitOp op : ops) { + keys.add(s(op.getKey())); + } return 0; }); try (Transaction txn = db.beginTransaction()) { - txn.put(cf, "final_key".getBytes(StandardCharsets.UTF_8), - "final_value".getBytes(StandardCharsets.UTF_8)); + txn.put(cf, b("a"), b("1")); + txn.put(cf, b("b"), b("2")); txn.commit(); } - assertTrue(hookCounter.get() > 0, - "At least one hook invocation should have occurred"); - + assertEquals(2, opCount.get()); + assertTrue(lastSeq.get() > 0); + assertTrue(keys.containsAll(Arrays.asList("a", "b"))); cf.clearCommitHook(); - } catch (TidesDBException e) { - fail("Final commit after concurrent stress should not throw: " + e.getMessage()); } } - } - - @Test - @Order(60) - void testCommitHookRepeatedTransitions() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb_hook_transitions").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - db.createColumnFamily("test_cf", cfConfig); - - ColumnFamily cf = db.getColumnFamily("test_cf"); - - final int iterations = 50; - - for (int i = 0; i < iterations; i++) { - List received = new ArrayList<>(); + @Test + void marksDeletesAndCarriesTtl() throws TidesDBException { + try (TidesDB db = openWithCf("hook-ops", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + List seen = java.util.Collections.synchronizedList(new ArrayList<>()); cf.setCommitHook((ops, commitSeq) -> { - received.add(ops); + seen.addAll(Arrays.asList(ops)); return 0; }); - // Commit with hook active -- hook should fire try (Transaction txn = db.beginTransaction()) { - byte[] key = ("key_set_" + i).getBytes(StandardCharsets.UTF_8); - byte[] value = ("value_set_" + i).getBytes(StandardCharsets.UTF_8); - txn.put(cf, key, value); + txn.put(cf, b("plain"), b("v")); + txn.put(cf, b("expiring"), b("v"), 3600); + txn.delete(cf, b("gone")); txn.commit(); } - assertEquals(1, received.size(), - "Hook should fire at iteration " + i); - assertArrayEquals(("key_set_" + i).getBytes(StandardCharsets.UTF_8), - received.get(0)[0].getKey()); + assertEquals(3, seen.size()); + CommitOp delete = seen.stream().filter(CommitOp::isDelete).findFirst().orElseThrow(); + assertEquals("gone", s(delete.getKey())); + assertNull(delete.getValue(), "a delete carries no value"); - // Clear the hook - cf.clearCommitHook(); + CommitOp expiring = seen.stream() + .filter(o -> "expiring".equals(s(o.getKey()))).findFirst().orElseThrow(); + assertTrue(expiring.getTtl() > 0, "the hook sees the absolute deadline"); - // Commit with hook cleared -- hook should NOT fire - received.clear(); - try (Transaction txn = db.beginTransaction()) { - byte[] key = ("key_clear_" + i).getBytes(StandardCharsets.UTF_8); - byte[] value = ("value_clear_" + i).getBytes(StandardCharsets.UTF_8); - txn.put(cf, key, value); - txn.commit(); - } + CommitOp plain = seen.stream() + .filter(o -> "plain".equals(s(o.getKey()))).findFirst().orElseThrow(); + assertEquals(-1, plain.getTtl(), "an entry that never expires reports -1"); - assertEquals(0, received.size(), - "Hook should not fire after clearing at iteration " + i); + cf.clearCommitHook(); } } - } - - @Test - @Order(61) - void testCloseWithInstalledHookReleasesCallback() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb_close_hook_leak").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - List received = new ArrayList<>(); - - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - db.createColumnFamily("test_cf", cfConfig); - - ColumnFamily cf = db.getColumnFamily("test_cf"); + @Test + void stopsFiringOnceCleared() throws TidesDBException { + try (TidesDB db = openWithCf("hook-clear", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + AtomicInteger calls = new AtomicInteger(); + cf.setCommitHook((ops, seq) -> { + calls.incrementAndGet(); + return 0; + }); - cf.setCommitHook((ops, commitSeq) -> { - received.add(ops); - return 0; - }); + write(db, cf, "before-clear", "v"); + int afterFirst = calls.get(); + assertTrue(afterFirst > 0); - // Commit some data so the hook is exercised - try (Transaction txn = db.beginTransaction()) { - txn.put(cf, "key1".getBytes(StandardCharsets.UTF_8), - "value1".getBytes(StandardCharsets.UTF_8)); - txn.commit(); + cf.clearCommitHook(); + write(db, cf, "after-clear", "v"); + assertEquals(afterFirst, calls.get()); } - - assertEquals(1, received.size()); - - // close() via try-with-resources WITHOUT calling clearCommitHook() } - // After close, trigger GC and verify no callbacks fire post-close - System.gc(); + @Test + void replacesAnInstalledHook() throws TidesDBException { + try (TidesDB db = openWithCf("hook-replace", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + AtomicInteger first = new AtomicInteger(); + AtomicInteger second = new AtomicInteger(); - // The hook should not have fired again after close - assertEquals(1, received.size(), - "Hook callback should not fire after database close"); - } + cf.setCommitHook((ops, seq) -> { first.incrementAndGet(); return 0; }); + cf.setCommitHook((ops, seq) -> { second.incrementAndGet(); return 0; }); - @Test - @Order(62) - void testCloseWithInstalledHookIdempotent() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb_close_idempotent").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); + write(db, cf, "k", "v"); - TidesDB db = TidesDB.open(config); - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - db.createColumnFamily("test_cf", cfConfig); + assertEquals(0, first.get(), "the replaced hook no longer fires"); + assertTrue(second.get() > 0); + cf.clearCommitHook(); + } + } - ColumnFamily cf = db.getColumnFamily("test_cf"); + @Test + void survivesAThrowingHook() throws TidesDBException { + try (TidesDB db = openWithCf("hook-throws", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + cf.setCommitHook((ops, seq) -> { + throw new RuntimeException("hook failure"); + }); - cf.setCommitHook((ops, commitSeq) -> { - return 0; - }); + assertDoesNotThrow(() -> write(db, cf, "k", "v")); + assertEquals("v", read(db, cf, "k"), "the commit is already durable"); + cf.clearCommitHook(); + } + } - // First close - db.close(); + @Test + void isDetachedWhenTheDatabaseCloses() throws TidesDBException { + TidesDB db = openWithCf("hook-close", "cf"); + ColumnFamily cf = db.getColumnFamily("cf"); + cf.setCommitHook((ops, seq) -> 0); + assertDoesNotThrow(db::close); + } - // Second close should not throw - assertDoesNotThrow(db::close); + @Test + void rejectsANullHook() throws TidesDBException { + try (TidesDB db = openWithCf("hook-args", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + assertThrows(IllegalArgumentException.class, () -> cf.setCommitHook(null)); + } + } } - @Test - @Order(63) - void testColumnFamilyOperationsThrowAfterOwnerClose() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb_cf_after_close").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - ColumnFamily cf; - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - db.createColumnFamily("test_cf", cfConfig); - cf = db.getColumnFamily("test_cf"); - } - // db is now closed; cf still references it - - assertThrows(IllegalStateException.class, cf::getStats); - assertThrows(IllegalStateException.class, cf::compact); - assertThrows(IllegalStateException.class, - () -> cf.compactRange("a".getBytes(), "z".getBytes())); - assertThrows(IllegalStateException.class, cf::flushMemtable); - assertThrows(IllegalStateException.class, cf::isFlushing); - assertThrows(IllegalStateException.class, cf::isCompacting); - assertThrows(IllegalStateException.class, - () -> cf.updateRuntimeConfig(ColumnFamilyConfig.defaultConfig(), false)); - assertThrows(IllegalStateException.class, - () -> cf.rangeCost("a".getBytes(), "z".getBytes())); - assertThrows(IllegalStateException.class, - () -> cf.setCommitHook((ops, seq) -> 0)); - assertThrows(IllegalStateException.class, cf::clearCommitHook); - assertThrows(IllegalStateException.class, cf::purge); - assertThrows(IllegalStateException.class, cf::syncWal); - } + @Nested + class Maintenance { - @Test - @Order(64) - void testSetClearHookAfterCloseThrows() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb_hook_after_close").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - ColumnFamily cf; - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - db.createColumnFamily("test_cf", cfConfig); - cf = db.getColumnFamily("test_cf"); - } - // db is now closed - - assertThrows(IllegalStateException.class, - () -> cf.setCommitHook((ops, seq) -> 0)); - assertThrows(IllegalStateException.class, cf::clearCommitHook); - } + @Test + void flushesAndReportsFlushState() throws TidesDBException { + try (TidesDB db = openWithCf("maint-flush", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + write(db, cf, "k", "v"); - @Test - @Order(65) - void testMultipleColumnFamiliesWithHooksCloseCleanly() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb_multi_cf_hooks").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - List hook1Received = new ArrayList<>(); - List hook2Received = new ArrayList<>(); - - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - db.createColumnFamily("cf1", cfConfig); - db.createColumnFamily("cf2", cfConfig); - db.createColumnFamily("cf3", cfConfig); - - ColumnFamily cf1 = db.getColumnFamily("cf1"); - ColumnFamily cf2 = db.getColumnFamily("cf2"); - ColumnFamily cf3 = db.getColumnFamily("cf3"); - - // Install hooks on cf1 and cf2, leave cf3 without a hook - cf1.setCommitHook((ops, seq) -> { - hook1Received.add(ops); - return 0; - }); - cf2.setCommitHook((ops, seq) -> { - hook2Received.add(ops); - return 0; - }); - - // Commit data to all three CFs - try (Transaction txn = db.beginTransaction()) { - txn.put(cf1, "k1".getBytes(), "v1".getBytes()); - txn.put(cf2, "k2".getBytes(), "v2".getBytes()); - txn.put(cf3, "k3".getBytes(), "v3".getBytes()); - txn.commit(); + assertDoesNotThrow(db::flushMemtable); + assertDoesNotThrow(db::isFlushing); + assertEquals("v", read(db, cf, "k")); } - - assertEquals(1, hook1Received.size()); - assertEquals(1, hook2Received.size()); - - // close() via try-with-resources WITHOUT clearing hooks } - // After close, no deferred callbacks should fire - System.gc(); - assertEquals(1, hook1Received.size(), - "Hook1 should not fire after close"); - assertEquals(1, hook2Received.size(), - "Hook2 should not fire after close"); - } + @Test + void syncsTheWriteAheadLog() throws TidesDBException { + try (TidesDB db = openWithCf("maint-sync", "cf")) { + write(db, db.getColumnFamily("cf"), "k", "v"); + assertDoesNotThrow(db::syncWal); + } + } - @Test - @Order(69) - void testRepeatedS3OpenFailureDoesNotCrash() { - if (!TidesDB.isS3Available()) { - // S3 support not compiled in; nothing to verify. - return; - } - - // Use a port that is almost certainly unused (distinct from 9000 used by other tests) - S3Config s3 = S3Config.builder() - .endpoint("127.0.0.1:19000") - .bucket("tidesdb-test") - .accessKey("minioadmin") - .secretKey("minioadmin") - .usePathStyle(true) - .useSsl(false) - .build(); - - Config config = Config.builder(tempDir.resolve("testdb_s3_leak").toString()) - .objectStoreS3Config(s3) - .build(); - - // Each iteration creates an S3 connector then fails tidesdb_open. - // Without the fix the connector leaks native memory. - for (int i = 0; i < 50; i++) { - assertThrows(TidesDBException.class, () -> TidesDB.open(config)); + @Test + void establishesADurabilityBarrier() throws TidesDBException { + try (TidesDB db = openWithCf("maint-checkpoint", "cf")) { + write(db, db.getColumnFamily("cf"), "k", "v"); + assertDoesNotThrow(db::checkpoint); + } } - } - @Test - @Order(66) - void testSetClearHookNotRacingClose() throws TidesDBException, InterruptedException { - Config config = Config.builder(tempDir.resolve("testdb_hook_race_close").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - db.createColumnFamily("test_cf", cfConfig); - - ColumnFamily cf = db.getColumnFamily("test_cf"); - - // Set initial hook - cf.setCommitHook((ops, seq) -> { - return 0; - }); - - CountDownLatch startLatch = new CountDownLatch(1); - CountDownLatch stopLatch = new CountDownLatch(1); - - // Writer thread that commits in a loop - Thread writer = new Thread(() -> { - try { - startLatch.await(); - } catch (InterruptedException e) { - return; - } - while (stopLatch.getCount() > 0) { - try (Transaction txn = db.beginTransaction()) { - byte[] key = ("key_" + System.nanoTime()).getBytes(StandardCharsets.UTF_8); - txn.put(cf, key, "value".getBytes(StandardCharsets.UTF_8)); - txn.commit(); - } catch (TidesDBException | IllegalStateException e) { - // Expected during close - } - } - }); - writer.setDaemon(true); - - // Replacer thread that alternates set/clear hook - Thread replacer = new Thread(() -> { - try { - startLatch.await(); - } catch (InterruptedException e) { - return; - } - for (int i = 0; i < 200 && stopLatch.getCount() > 0; i++) { - try { - if (i % 2 == 0) { - cf.setCommitHook((ops, seq) -> 0); - } else { - cf.clearCommitHook(); - } - } catch (TidesDBException | IllegalStateException e) { - // Expected during close + @Test + void compactsAFamily() throws TidesDBException { + try (TidesDB db = openWithCf("maint-compact", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + try (Transaction txn = db.beginTransaction()) { + for (int i = 0; i < 200; i++) { + txn.put(cf, b(String.format("k%04d", i)), b("v" + i)); } + txn.commit(); } - }); - replacer.setDaemon(true); - - writer.start(); - replacer.start(); - startLatch.countDown(); - - // Let them run for ~500ms then close - Thread.sleep(500); - - // Close should not hang, crash, or throw - assertDoesNotThrow(db::close); + db.flushMemtable(); - stopLatch.countDown(); - writer.join(5000); - replacer.join(5000); + assertDoesNotThrow(cf::compact); + assertDoesNotThrow(cf::isCompacting); + assertEquals("v100", read(db, cf, "k0100")); + } } - } - - @Test - @Order(71) - void testOpenWithObjectStoreFsPathFailsOnBadPath() throws Exception { - // A regular file (not a directory) as objectStoreFsPath must cause tidesdb_objstore_fs_create - // to return NULL, which should surface as TidesDBException instead of silently opening - // an ordinary local database. - Path osFile = tempDir.resolve("os_file"); - Files.createFile(osFile); - - Config config = Config.builder(tempDir.resolve("testdb_fs_badpath").toString()) - .objectStoreFsPath(osFile.toString()) - .build(); - - TidesDBException ex = assertThrows(TidesDBException.class, () -> TidesDB.open(config)); - assertEquals(TidesDBException.ERR_IO, ex.getErrorCode()); - } - - @Test - @Order(72) - void testOpenWithObjectStoreFsPathSucceedsWithValidDirectory() throws TidesDBException { - Path osDir = tempDir.resolve("os_dir"); - osDir.toFile().mkdirs(); - Config config = Config.builder(tempDir.resolve("testdb_fs_goodpath").toString()) - .objectStoreFsPath(osDir.toString()) - .build(); + @Test + void compactsAKeyRange() throws TidesDBException { + try (TidesDB db = openWithCf("maint-compact-range", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + try (Transaction txn = db.beginTransaction()) { + for (int i = 0; i < 100; i++) { + txn.put(cf, b(String.format("k%03d", i)), b("v")); + } + txn.commit(); + } + db.flushMemtable(); - try (TidesDB db = TidesDB.open(config)) { - assertNotNull(db); - DbStats dbStats = db.getDbStats(); - assertNotNull(dbStats); - assertTrue(dbStats.isObjectStoreEnabled()); + assertDoesNotThrow(() -> cf.compactRange(b("k000"), b("k050"))); + assertEquals("v", read(db, cf, "k025")); + } } - } - @Test - @Order(73) - void testOpenWithoutObjectStoreFsPathSucceeds() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("plaindb").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); + @Test + void writesAnOpenableBackup() throws Exception { + Path backupDir = tempDir.resolve("backup"); + try (TidesDB db = openWithCf("maint-backup", "cf")) { + write(db, db.getColumnFamily("cf"), "backed-up", "v"); + db.flushMemtable(); + db.backup(backupDir.toString()); + } - try (TidesDB db = TidesDB.open(config)) { - assertNotNull(db); + assertTrue(Files.isDirectory(backupDir)); + try (TidesDB restored = TidesDB.open( + Config.builder(backupDir.toString()).logLevel(LogLevel.NONE).build())) { + assertEquals("v", read(restored, restored.getColumnFamily("cf"), "backed-up")); + } } - } - @Test - @Order(75) - void testConfigValidationRejectsNegativeUnsignedFields() { - // logTruncationAt < 0 - assertThrows(IllegalArgumentException.class, - () -> Config.builder(tempDir.toString()).logTruncationAt(-1).build()); - try { - Config.builder(tempDir.toString()).logTruncationAt(-1).build(); - } catch (IllegalArgumentException e) { - assertTrue(e.getMessage().contains("logTruncationAt"), - "message should mention field name, was: " + e.getMessage()); - assertTrue(e.getMessage().toLowerCase().contains("negative"), - "message should mention negative, was: " + e.getMessage()); - } - - // maxMemoryUsage < 0 - assertThrows(IllegalArgumentException.class, - () -> Config.builder(tempDir.toString()).maxMemoryUsage(-1).build()); - try { - Config.builder(tempDir.toString()).maxMemoryUsage(-1).build(); - } catch (IllegalArgumentException e) { - assertTrue(e.getMessage().contains("maxMemoryUsage"), - "message should mention field name, was: " + e.getMessage()); - assertTrue(e.getMessage().toLowerCase().contains("negative"), - "message should mention negative, was: " + e.getMessage()); - } - - // unifiedMemtableWriteBufferSize < 0 - assertThrows(IllegalArgumentException.class, - () -> Config.builder(tempDir.toString()).unifiedMemtableWriteBufferSize(-1).build()); - try { - Config.builder(tempDir.toString()).unifiedMemtableWriteBufferSize(-1).build(); - } catch (IllegalArgumentException e) { - assertTrue(e.getMessage().contains("unifiedMemtableWriteBufferSize"), - "message should mention field name, was: " + e.getMessage()); - assertTrue(e.getMessage().toLowerCase().contains("negative"), - "message should mention negative, was: " + e.getMessage()); - } - - // unifiedMemtableSyncIntervalUs < 0 - assertThrows(IllegalArgumentException.class, - () -> Config.builder(tempDir.toString()).unifiedMemtableSyncIntervalUs(-1).build()); - try { - Config.builder(tempDir.toString()).unifiedMemtableSyncIntervalUs(-1).build(); - } catch (IllegalArgumentException e) { - assertTrue(e.getMessage().contains("unifiedMemtableSyncIntervalUs"), - "message should mention field name, was: " + e.getMessage()); - assertTrue(e.getMessage().toLowerCase().contains("negative"), - "message should mention negative, was: " + e.getMessage()); + @Test + void rejectsAnEmptyBackupDirectory() throws TidesDBException { + try (TidesDB db = openWithCf("maint-backup-args", "cf")) { + assertThrows(IllegalArgumentException.class, () -> db.backup(null)); + assertThrows(IllegalArgumentException.class, () -> db.backup("")); + } } - } - - @Test - @Order(76) - void testObjectStoreConfigValidation() { - // Reject negative unsigned-native fields (zero sentinel accepted) - assertThrows(IllegalArgumentException.class, - () -> ObjectStoreConfig.builder().localCacheMaxBytes(-1).build()); - assertThrows(IllegalArgumentException.class, - () -> ObjectStoreConfig.builder().multipartThreshold(-1).build()); - assertThrows(IllegalArgumentException.class, - () -> ObjectStoreConfig.builder().multipartPartSize(-1).build()); - assertThrows(IllegalArgumentException.class, - () -> ObjectStoreConfig.builder().walSyncThresholdBytes(-1).build()); - assertThrows(IllegalArgumentException.class, - () -> ObjectStoreConfig.builder().replicaSyncIntervalUs(-1).build()); - - // Positive-required (<= 0 rejected) - assertThrows(IllegalArgumentException.class, - () -> ObjectStoreConfig.builder().maxConcurrentUploads(0).build()); - try { - ObjectStoreConfig.builder().maxConcurrentUploads(0).build(); - } catch (IllegalArgumentException e) { - assertTrue(e.getMessage().contains("maxConcurrentUploads"), - "message should mention field name, was: " + e.getMessage()); - assertTrue(e.getMessage().toLowerCase().contains("positive"), - "message should mention positive, was: " + e.getMessage()); - } - - assertThrows(IllegalArgumentException.class, - () -> ObjectStoreConfig.builder().maxConcurrentDownloads(-1).build()); - try { - ObjectStoreConfig.builder().maxConcurrentDownloads(-1).build(); - } catch (IllegalArgumentException e) { - assertTrue(e.getMessage().contains("maxConcurrentDownloads"), - "message should mention field name, was: " + e.getMessage()); - assertTrue(e.getMessage().toLowerCase().contains("positive"), - "message should mention positive, was: " + e.getMessage()); - } - - // Zero sentinel accepted for localCacheMaxBytes - assertDoesNotThrow(() -> ObjectStoreConfig.builder().localCacheMaxBytes(0).build()); - } - @Test - @Order(77) - void testS3ConfigValidation() { - S3Config.Builder baseBuilder = S3Config.builder() - .endpoint("s3.amazonaws.com") - .bucket("b") - .accessKey("ak") - .secretKey("sk"); - - // Reject negative multipart fields - assertThrows(IllegalArgumentException.class, - () -> baseBuilder.multipartThreshold(-1).build()); - assertThrows(IllegalArgumentException.class, - () -> S3Config.builder() - .endpoint("s3.amazonaws.com") - .bucket("b") - .accessKey("ak") - .secretKey("sk") - .multipartPartSize(-1) - .build()); - - // Zero sentinels accepted - assertDoesNotThrow(() -> S3Config.builder() - .endpoint("s3.amazonaws.com") - .bucket("b") - .accessKey("ak") - .secretKey("sk") - .multipartThreshold(0) - .multipartPartSize(0) - .build()); + @Test + void appliesARuntimeConfigChange() throws TidesDBException { + try (TidesDB db = openWithCf("maint-runtime-config", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + write(db, cf, "k", "v"); - // Existing required-string validation still passes - assertThrows(IllegalArgumentException.class, () -> S3Config.builder().build()); - } + ColumnFamilyConfig updated = ColumnFamilyConfig.builder() + .compression(CompressionAlgorithm.ZSTD) + .enableBloomFilter(true) + .bloomFpr(0.05) + .build(); + cf.updateRuntimeConfig(updated, true); - @Test - @Order(78) - void testColumnFamilyConfigValidation() { - // Reject negative unsigned-native fields (zero sentinel accepted) - assertThrows(IllegalArgumentException.class, - () -> ColumnFamilyConfig.builder().klogValueThreshold(-1).build()); - assertThrows(IllegalArgumentException.class, - () -> ColumnFamilyConfig.builder().syncIntervalUs(-1).build()); - assertThrows(IllegalArgumentException.class, - () -> ColumnFamilyConfig.builder().minDiskSpace(-1).build()); - - // Positive-required fields (<= 0 rejected) - assertThrows(IllegalArgumentException.class, - () -> ColumnFamilyConfig.builder().writeBufferSize(0).build()); - assertThrows(IllegalArgumentException.class, - () -> ColumnFamilyConfig.builder().levelSizeRatio(0).build()); - assertThrows(IllegalArgumentException.class, - () -> ColumnFamilyConfig.builder().minLevels(0).build()); - assertThrows(IllegalArgumentException.class, - () -> ColumnFamilyConfig.builder().indexSampleRatio(0).build()); - assertThrows(IllegalArgumentException.class, - () -> ColumnFamilyConfig.builder().l1FileCountTrigger(0).build()); - assertThrows(IllegalArgumentException.class, - () -> ColumnFamilyConfig.builder().l0QueueStallThreshold(0).build()); - assertThrows(IllegalArgumentException.class, - () -> ColumnFamilyConfig.builder().tombstoneDensityMinEntries(0).build()); - - // Non-negative-int fields (zero acceptable, negative rejected) - assertThrows(IllegalArgumentException.class, - () -> ColumnFamilyConfig.builder().dividingLevelOffset(-1).build()); - assertThrows(IllegalArgumentException.class, - () -> ColumnFamilyConfig.builder().blockIndexPrefixLen(-1).build()); - assertThrows(IllegalArgumentException.class, - () -> ColumnFamilyConfig.builder().skipListMaxLevel(-1).build()); - - // Float/double NaN/infinity/range rejection: bloomFPR - assertThrows(IllegalArgumentException.class, - () -> ColumnFamilyConfig.builder().bloomFPR(Double.NaN).build()); - assertThrows(IllegalArgumentException.class, - () -> ColumnFamilyConfig.builder().bloomFPR(Double.POSITIVE_INFINITY).build()); - assertThrows(IllegalArgumentException.class, - () -> ColumnFamilyConfig.builder().bloomFPR(-0.1).build()); - assertThrows(IllegalArgumentException.class, - () -> ColumnFamilyConfig.builder().bloomFPR(1.1).build()); - - // Float/double NaN/infinity/range rejection: skipListProbability - assertThrows(IllegalArgumentException.class, - () -> ColumnFamilyConfig.builder().skipListProbability(Float.NaN).build()); - assertThrows(IllegalArgumentException.class, - () -> ColumnFamilyConfig.builder().skipListProbability(Float.POSITIVE_INFINITY).build()); - assertThrows(IllegalArgumentException.class, - () -> ColumnFamilyConfig.builder().skipListProbability(-0.1f).build()); - assertThrows(IllegalArgumentException.class, - () -> ColumnFamilyConfig.builder().skipListProbability(1.1f).build()); - - // Float/double NaN/infinity/range rejection: tombstoneDensityTrigger - assertThrows(IllegalArgumentException.class, - () -> ColumnFamilyConfig.builder().tombstoneDensityTrigger(Double.NaN).build()); - assertThrows(IllegalArgumentException.class, - () -> ColumnFamilyConfig.builder().tombstoneDensityTrigger(Double.POSITIVE_INFINITY).build()); - assertThrows(IllegalArgumentException.class, - () -> ColumnFamilyConfig.builder().tombstoneDensityTrigger(-0.1).build()); - assertThrows(IllegalArgumentException.class, - () -> ColumnFamilyConfig.builder().tombstoneDensityTrigger(1.1).build()); - - // Nullable-enum null rejection - assertThrows(IllegalArgumentException.class, - () -> ColumnFamilyConfig.builder().compressionAlgorithm(null).build()); - assertThrows(IllegalArgumentException.class, - () -> ColumnFamilyConfig.builder().syncMode(null).build()); - assertThrows(IllegalArgumentException.class, - () -> ColumnFamilyConfig.builder().defaultIsolationLevel(null).build()); - - // fromNative compatibility: engine-supplied values must be accepted - assertDoesNotThrow(() -> ColumnFamilyConfig.defaultConfig()); - } + ColumnFamilyConfig applied = cf.getStats().getConfig(); + assertArrayEquals(new int[]{CompressionAlgorithm.ZSTD.getValue()}, + applied.getEncodingPipeline()); + assertEquals(0.05, applied.getBloomFpr(), 1e-9); + assertEquals("cf", applied.getName(), "the family keeps its identity"); + assertEquals("v", read(db, cf, "k")); + } + } - @Test - @Order(74) - void testS3PrecedenceOverFsPathOnFailure() throws TidesDBException { - Path osFile = tempDir.resolve("os_file_s3"); - try { - Files.createFile(osFile); - } catch (java.io.IOException e) { - throw new RuntimeException(e); - } - - S3Config s3 = S3Config.builder() - .endpoint("127.0.0.1:19001") - .bucket("tidesdb-test") - .accessKey("minioadmin") - .secretKey("minioadmin") - .usePathStyle(true) - .useSsl(false) - .build(); - - Config config = Config.builder(tempDir.resolve("testdb_s3_over_fs").toString()) - .objectStoreS3Config(s3) - .objectStoreFsPath(osFile.toString()) - .build(); - - // S3 takes precedence over fs path. If S3 is available, the S3 connector creation or - // open fails with TidesDBException. If S3 is unavailable, the S3 connector creation - // itself throws. In neither case does a silent filesystem fallback occur. - assertThrows(TidesDBException.class, () -> TidesDB.open(config)); + @Test + void rejectsANullRuntimeConfig() throws TidesDBException { + try (TidesDB db = openWithCf("maint-runtime-args", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + assertThrows(IllegalArgumentException.class, + () -> cf.updateRuntimeConfig(null, false)); + } + } } - @Test - @Order(70) - void testJniBufferMethodsCoverage() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb_coverage").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - ColumnFamilyConfig cfConfig = ColumnFamilyConfig.defaultConfig(); - db.createColumnFamily("coverage_cf", cfConfig); + @Nested + class Statistics { - ColumnFamily cf = db.getColumnFamily("coverage_cf"); - - // Insert 100 entries into one transaction and commit - try (Transaction txn = db.beginTransaction()) { - for (int i = 0; i < 100; i++) { - byte[] key = String.format("cov_key%04d", i).getBytes(StandardCharsets.UTF_8); - byte[] value = ("cov_value" + i).getBytes(StandardCharsets.UTF_8); - txn.put(cf, key, value); + @Test + void reportsColumnFamilyStatistics() throws TidesDBException { + try (TidesDB db = openWithCf("stats-cf", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + try (Transaction txn = db.beginTransaction()) { + for (int i = 0; i < 50; i++) { + txn.put(cf, b("k" + i), b("v" + i)); + } + txn.commit(); } - txn.commit(); - } - - // 1. Transaction put/get/delete/singleDelete with various byte arrays - try (Transaction txn = db.beginTransaction()) { - // put with a 0-length value and a 1-byte key - byte[] tinyKey = new byte[]{0x42}; - byte[] emptyValue = new byte[0]; - txn.put(cf, tinyKey, emptyValue); - - // get of that entry - byte[] got = txn.get(cf, tinyKey); - assertNotNull(got, "get should return a non-null result for the 0-length value entry"); - assertEquals(0, got.length, "value should be 0-length"); + db.flushMemtable(); + + CfStats stats = cf.getStats(); + assertNotNull(stats); + assertTrue(stats.getTotalKeys() > 0); + assertEquals("cf", stats.getConfig().getName()); + assertEquals(CfStats.MAX_LEVELS, stats.getLevelSizes().length); + assertEquals(CfStats.MAX_LEVELS, stats.getLevelNumSstables().length); + assertEquals(CfStats.MAX_LEVELS, stats.getLevelKeyCounts().length); + assertEquals(CfStats.MAX_LEVELS, stats.getLevelTombstoneCounts().length); + assertTrue(stats.getNumLevels() >= 0); + assertTrue(stats.getUserBytesWritten() > 0); + assertNotNull(stats.toString()); + } + } + + @Test + void estimatesCardinality() throws TidesDBException { + try (TidesDB db = openWithCf("stats-cardinality", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + try (Transaction txn = db.beginTransaction()) { + for (int i = 0; i < 100; i++) { + txn.put(cf, b("k" + i), b("v")); + } + txn.commit(); + } + db.flushMemtable(); - // delete that entry - txn.delete(cf, tinyKey); + assertTrue(cf.estimateCardinality() >= 0); + } + } - // singleDelete of a fresh entry - byte[] sdKey = new byte[]{0x43}; - byte[] sdValue = new byte[]{0x01}; - txn.put(cf, sdKey, sdValue); - txn.singleDelete(cf, sdKey); + @Test + void reportsDatabaseStatistics() throws TidesDBException { + try (TidesDB db = openWithCf("stats-db", "cf")) { + write(db, db.getColumnFamily("cf"), "k", "v"); - txn.commit(); + DbStats stats = db.getDbStats(); + assertNotNull(stats); + assertEquals(1, stats.getNumColumnFamilies()); + assertTrue(stats.getGlobalSeq() > 0); + assertTrue(stats.getUserBytesWritten() > 0); + assertTrue(stats.getWalBytesWritten() > 0); + assertNotNull(stats.toString()); } + } - // Confirm get of deleted key throws TidesDBException - try (Transaction txn = db.beginTransaction()) { - assertThrows(TidesDBException.class, () -> txn.get(cf, new byte[]{0x42})); - assertThrows(TidesDBException.class, () -> txn.get(cf, new byte[]{0x43})); + @Test + void reportsCacheStatistics() throws TidesDBException { + try (TidesDB db = openWithCf("stats-cache", "cf")) { + CacheStats stats = db.getCacheStats(); + assertNotNull(stats); + assertTrue(stats.getHits() >= 0); + assertTrue(stats.getMisses() >= 0); + assertTrue(stats.getNumPartitions() > 0); + assertNotNull(stats.toString()); } + } - // 2. Iterator seek/seekForPrev/key/value/keyValue - try (Transaction txn = db.beginTransaction()) { - try (TidesDBIterator iter = txn.newIterator(cf)) { - iter.seekToFirst(); - int count = 0; - while (iter.isValid()) { - byte[] k = iter.key(); - byte[] v = iter.value(); - KeyValue kv = iter.keyValue(); - assertNotNull(k, "iterator key should not be null"); - assertNotNull(v, "iterator value should not be null"); - assertNotNull(kv, "iterator keyValue should not be null"); - assertNotNull(kv.getKey(), "KeyValue.getKey() should not be null"); - assertNotNull(kv.getValue(), "KeyValue.getValue() should not be null"); - count++; - iter.next(); - } - assertEquals(100, count, "iterator should visit exactly 100 entries"); + @Test + void reportsWhereWritersWaited() throws TidesDBException { + try (TidesDB db = openWithCf("stats-stall", "cf")) { + write(db, db.getColumnFamily("cf"), "k", "v"); + + StallStats stats = db.getStallStats(); + assertNotNull(stats); + assertEquals(StallReason.values().length, stats.getReasons().length); + for (StallReason reason : StallReason.values()) { + assertNotNull(stats.get(reason)); + assertTrue(stats.get(reason).getCount() >= 0); } + assertTrue(stats.getTotalUs() >= 0); + assertThrows(IllegalArgumentException.class, () -> stats.get(null)); } - - // seek and seekForPrev - try (Transaction txn = db.beginTransaction()) { - try (TidesDBIterator iter = txn.newIterator(cf)) { - byte[] targetKey = String.format("cov_key%04d", 50).getBytes(StandardCharsets.UTF_8); - iter.seek(targetKey); - assertTrue(iter.isValid(), "iterator should be valid after seek"); - byte[] seekedKey = iter.key(); - assertNotNull(seekedKey); - assertTrue(seekedKey.length > 0, "seeked key should not be empty"); - - iter.seekForPrev(targetKey); - assertTrue(iter.isValid(), "iterator should be valid after seekForPrev"); - byte[] seekedForPrevKey = iter.key(); - assertNotNull(seekedForPrevKey); - assertTrue(seekedForPrevKey.length > 0, "seekForPrev key should not be empty"); - } - } - - // 3. compactRange with single-byte bounds and with one-null-one-nonnull - cf.flushMemtable(); - assertDoesNotThrow(() -> cf.compactRange(new byte[]{0x10}, new byte[]{0x20}), - "compactRange with valid bounds should succeed"); - assertDoesNotThrow(() -> cf.compactRange(null, new byte[]{0x20}), - "compactRange with null start should succeed"); - assertDoesNotThrow(() -> cf.compactRange(new byte[]{0x10}, null), - "compactRange with null end should succeed"); - - // 4. rangeCost with single-byte bounds - double cost = cf.rangeCost(new byte[]{0x01}, new byte[]{(byte) 0xFF}); - assertTrue(cost >= 0.0, "rangeCost should be non-negative"); - - // 5. getStats on the CF after data insertion - Stats stats = cf.getStats(); - assertNotNull(stats, "getStats should return non-null"); - - // 6. getDbStats on the database handle - DbStats dbStats = db.getDbStats(); - assertNotNull(dbStats, "getDbStats should return non-null"); - - // 7. listColumnFamilies and iterate the result - String[] families = db.listColumnFamilies(); - assertNotNull(families, "listColumnFamilies should return non-null"); - assertTrue(families.length > 0, "should have at least one column family"); - boolean found = false; - for (String f : families) { - if ("coverage_cf".equals(f)) { - found = true; - break; - } - } - assertTrue(found, "coverage_cf should be in the list"); - - // 8. getCacheStats on the database handle - CacheStats cacheStats = db.getCacheStats(); - assertNotNull(cacheStats, "getCacheStats should return non-null"); } - } - @Test - @Order(79) - void testIteratorSeekNullAndEmptyKey() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb_iter_seek_null").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - db.createColumnFamily("test_cf", ColumnFamilyConfig.defaultConfig()); - ColumnFamily cf = db.getColumnFamily("test_cf"); + @Test + void reportsWhatEachFileClassAskedOfTheDevice() throws TidesDBException { + try (TidesDB db = openWithCf("stats-io", "cf")) { + write(db, db.getColumnFamily("cf"), "k", "v"); + db.flushMemtable(); - try (Transaction txn = db.beginTransaction()) { - txn.put(cf, "k".getBytes(), "v".getBytes()); - txn.commit(); - } - - try (Transaction txn = db.beginTransaction()) { - try (TidesDBIterator iter = txn.newIterator(cf)) { - assertThrows(IllegalArgumentException.class, () -> iter.seek(null)); - assertThrows(IllegalArgumentException.class, () -> iter.seek(new byte[0])); - assertThrows(IllegalArgumentException.class, () -> iter.seekForPrev(null)); - assertThrows(IllegalArgumentException.class, () -> iter.seekForPrev(new byte[0])); + IoStats stats = db.getIoStats(); + assertNotNull(stats); + assertEquals(IoClass.values().length, stats.getClasses().length); + for (IoClass cls : IoClass.values()) { + assertNotNull(stats.get(cls)); + assertTrue(stats.get(cls).getBytesPerSecond() >= 0.0); } + assertTrue(stats.getTotalBytes() > 0); + assertThrows(IllegalArgumentException.class, () -> stats.get(null)); } } - } - @Test - @Order(80) - void testIteratorOperationsAfterFree() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb_iter_after_free").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - db.createColumnFamily("test_cf", ColumnFamilyConfig.defaultConfig()); - ColumnFamily cf = db.getColumnFamily("test_cf"); - - try (Transaction txn = db.beginTransaction()) { - txn.put(cf, "k".getBytes(), "v".getBytes()); - txn.commit(); - } + @Test + void reportsWhatEachEncodingChainAchieved() throws TidesDBException { + try (TidesDB db = open("stats-encoding")) { + db.createColumnFamily("cf", ColumnFamilyConfig.builder() + .compression(CompressionAlgorithm.LZ4) + .build()); + ColumnFamily cf = db.getColumnFamily("cf"); - try (Transaction txn = db.beginTransaction()) { - TidesDBIterator iter = txn.newIterator(cf); - iter.seekToFirst(); - assertTrue(iter.isValid()); - iter.free(); - - // isValid returns false after free - assertFalse(iter.isValid()); - - // All operations throw IllegalStateException after free - assertThrows(IllegalStateException.class, iter::seekToFirst); - assertThrows(IllegalStateException.class, iter::seekToLast); - assertThrows(IllegalStateException.class, () -> iter.seek("k".getBytes())); - assertThrows(IllegalStateException.class, () -> iter.seekForPrev("k".getBytes())); - assertThrows(IllegalStateException.class, iter::next); - assertThrows(IllegalStateException.class, iter::prev); - assertThrows(IllegalStateException.class, iter::key); - assertThrows(IllegalStateException.class, iter::value); - assertThrows(IllegalStateException.class, iter::keyValue); - - // free() is idempotent -- no exception - assertDoesNotThrow(iter::free); - assertDoesNotThrow(iter::close); + try (Transaction txn = db.beginTransaction()) { + for (int i = 0; i < 200; i++) { + txn.put(cf, b(String.format("k%04d", i)), b("a repetitive value " + i)); + } + txn.commit(); + } + db.flushMemtable(); + + EncodingStats[] klog = db.getKlogEncodingStats(); + assertNotNull(klog); + assertTrue(klog.length <= EncodingStats.MAX_CHAINS); + for (EncodingStats e : klog) { + assertNotNull(e.getIds()); + assertTrue(e.getRatio() >= 0.0); + assertNotNull(e.toString()); + } + assertNotNull(db.getVlogEncodingStats()); } } - } - - @Test - @Order(81) - void testTransactionNullArgs() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb_txn_null").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - db.createColumnFamily("test_cf", ColumnFamilyConfig.defaultConfig()); - ColumnFamily cf = db.getColumnFamily("test_cf"); - - try (Transaction txn = db.beginTransaction()) { - // put: null cf, null key, null value - assertThrows(IllegalArgumentException.class, - () -> txn.put(null, "k".getBytes(), "v".getBytes())); - assertThrows(IllegalArgumentException.class, - () -> txn.put(cf, null, "v".getBytes())); - assertThrows(IllegalArgumentException.class, - () -> txn.put(cf, "k".getBytes(), null)); - - // get: null cf, null key - assertThrows(IllegalArgumentException.class, - () -> txn.get(null, "k".getBytes())); - assertThrows(IllegalArgumentException.class, - () -> txn.get(cf, null)); - - // delete: null cf, null key - assertThrows(IllegalArgumentException.class, - () -> txn.delete(null, "k".getBytes())); - assertThrows(IllegalArgumentException.class, - () -> txn.delete(cf, null)); - // singleDelete: null cf, null key (already tested but grouped here) - assertThrows(IllegalArgumentException.class, - () -> txn.singleDelete(null, "k".getBytes())); - assertThrows(IllegalArgumentException.class, - () -> txn.singleDelete(cf, null)); + @Test + void describesAKeyRangeForAPlanner() throws TidesDBException { + try (TidesDB db = openWithCf("stats-range", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + try (Transaction txn = db.beginTransaction()) { + for (int i = 0; i < 100; i++) { + txn.put(cf, b(String.format("k%03d", i)), b("v")); + } + txn.commit(); + } + db.flushMemtable(); - // newIterator: null cf - assertThrows(IllegalArgumentException.class, - () -> txn.newIterator(null)); + RangeStats stats = cf.rangeStats(b("k000"), b("k050")); + assertNotNull(stats); + assertTrue(stats.getSstablesOverlapping() >= 0); + assertTrue(stats.getEstimatedKeys() >= 0); + assertNotNull(stats.toString()); } } - } - - @Test - @Order(82) - void testTransactionSavepointNullAndEmpty() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb_txn_sp_null").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - try (TidesDB db = TidesDB.open(config)) { - db.createColumnFamily("test_cf", ColumnFamilyConfig.defaultConfig()); - - try (Transaction txn = db.beginTransaction()) { + @Test + void rejectsEmptyRangeBounds() throws TidesDBException { + try (TidesDB db = openWithCf("stats-range-args", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + assertThrows(IllegalArgumentException.class, () -> cf.rangeStats(null, b("z"))); assertThrows(IllegalArgumentException.class, - () -> txn.savepoint(null)); - assertThrows(IllegalArgumentException.class, - () -> txn.savepoint("")); - assertThrows(IllegalArgumentException.class, - () -> txn.rollbackToSavepoint(null)); - assertThrows(IllegalArgumentException.class, - () -> txn.rollbackToSavepoint("")); - assertThrows(IllegalArgumentException.class, - () -> txn.releaseSavepoint(null)); - assertThrows(IllegalArgumentException.class, - () -> txn.releaseSavepoint("")); + () -> cf.rangeStats(b("a"), new byte[0])); } } } - @Test - @Order(83) - void testTransactionOperationsAfterFree() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb_txn_after_free").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - db.createColumnFamily("test_cf", ColumnFamilyConfig.defaultConfig()); - ColumnFamily cf = db.getColumnFamily("test_cf"); - - Transaction txn = db.beginTransaction(); - txn.put(cf, "k".getBytes(), "v".getBytes()); - txn.commit(); - txn.free(); - - // All operations throw IllegalStateException after free - assertThrows(IllegalStateException.class, - () -> txn.put(cf, "k".getBytes(), "v".getBytes())); - assertThrows(IllegalStateException.class, - () -> txn.get(cf, "k".getBytes())); - assertThrows(IllegalStateException.class, - () -> txn.delete(cf, "k".getBytes())); - assertThrows(IllegalStateException.class, - () -> txn.singleDelete(cf, "k".getBytes())); - assertThrows(IllegalStateException.class, txn::commit); - assertThrows(IllegalStateException.class, txn::rollback); - assertThrows(IllegalStateException.class, - () -> txn.savepoint("sp")); - assertThrows(IllegalStateException.class, - () -> txn.rollbackToSavepoint("sp")); - assertThrows(IllegalStateException.class, - () -> txn.releaseSavepoint("sp")); - assertThrows(IllegalStateException.class, - () -> txn.newIterator(cf)); - assertThrows(IllegalStateException.class, - () -> txn.reset(IsolationLevel.READ_COMMITTED)); - - // free() and close() are idempotent - assertDoesNotThrow(txn::free); - assertDoesNotThrow(txn::close); - } - } + @Nested + class Concurrency { - @Test - @Order(84) - void testOpenWithObjectStoreConfig() throws TidesDBException { - Path osDir = tempDir.resolve("os_cfg_dir"); - osDir.toFile().mkdirs(); - - ObjectStoreConfig osc = ObjectStoreConfig.builder() - .localCachePath(osDir.resolve("cache").toString()) - .localCacheMaxBytes(1024 * 1024) - .cacheOnRead(true) - .cacheOnWrite(false) - .maxConcurrentUploads(2) - .maxConcurrentDownloads(4) - .multipartThreshold(1024 * 1024) - .multipartPartSize(256 * 1024) - .syncManifestToObject(false) - .replicateWal(false) - .walUploadSync(true) - .walSyncThresholdBytes(2048) - .walSyncOnCommit(true) - .replicaMode(false) - .replicaSyncIntervalUs(1000) - .replicaReplayWal(true) - .build(); - - Config config = Config.builder(tempDir.resolve("testdb_osc_cfg").toString()) - .objectStoreFsPath(osDir.toString()) - .objectStoreConfig(osc) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - assertNotNull(db); - DbStats dbStats = db.getDbStats(); - assertNotNull(dbStats); - assertTrue(dbStats.isObjectStoreEnabled()); - } - } + @Test + void servesConcurrentWritersOnDistinctKeys() throws Exception { + try (TidesDB db = openWithCf("conc-writers", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + int threads = 4; + int perThread = 50; + List workers = new ArrayList<>(); + List errors = java.util.Collections.synchronizedList(new ArrayList<>()); - @Test - @Order(85) - void testOpenNullConfig() { - assertThrows(IllegalArgumentException.class, () -> TidesDB.open(null)); - } + for (int t = 0; t < threads; t++) { + final int id = t; + Thread worker = new Thread(() -> { + try { + for (int i = 0; i < perThread; i++) { + try (Transaction txn = db.beginTransaction()) { + txn.put(cf, b("t" + id + "-k" + i), b("v")); + txn.commit(); + } + } + } catch (Throwable e) { + errors.add(e); + } + }); + workers.add(worker); + worker.start(); + } + for (Thread worker : workers) { + worker.join(); + } - @Test - @Order(86) - void testOpenEmptyDbPath() { - Config config = Config.builder("").build(); - assertThrows(IllegalArgumentException.class, () -> TidesDB.open(config)); - } + assertTrue(errors.isEmpty(), () -> "worker failures: " + errors); + for (int t = 0; t < threads; t++) { + assertEquals("v", read(db, cf, "t" + t + "-k0")); + assertEquals("v", read(db, cf, "t" + t + "-k" + (perThread - 1))); + } + } + } + + @Test + void servesConcurrentReadersDuringWrites() throws Exception { + try (TidesDB db = openWithCf("conc-readers", "cf")) { + ColumnFamily cf = db.getColumnFamily("cf"); + try (Transaction txn = db.beginTransaction()) { + for (int i = 0; i < 100; i++) { + txn.put(cf, b("k" + i), b("v" + i)); + } + txn.commit(); + } - @Test - @Order(87) - void testColumnFamilyUpdateRuntimeConfigNull() throws TidesDBException { - Config config = Config.builder(tempDir.resolve("testdb_cf_upd_null").toString()) - .numFlushThreads(2) - .numCompactionThreads(2) - .logLevel(LogLevel.INFO) - .blockCacheSize(64 * 1024 * 1024) - .maxOpenSSTables(256) - .build(); - - try (TidesDB db = TidesDB.open(config)) { - db.createColumnFamily("test_cf", ColumnFamilyConfig.defaultConfig()); - ColumnFamily cf = db.getColumnFamily("test_cf"); + List errors = java.util.Collections.synchronizedList(new ArrayList<>()); + List readers = new ArrayList<>(); + for (int t = 0; t < 4; t++) { + Thread reader = new Thread(() -> { + try { + for (int i = 0; i < 100; i++) { + assertEquals("v" + i, read(db, cf, "k" + i)); + } + } catch (Throwable e) { + errors.add(e); + } + }); + readers.add(reader); + reader.start(); + } + for (Thread reader : readers) { + reader.join(); + } - assertThrows(IllegalArgumentException.class, - () -> cf.updateRuntimeConfig(null, false)); + assertTrue(errors.isEmpty(), () -> "reader failures: " + errors); + } } } }