Skip to content

CAS: refuse to republish a namespace life retired during a cold admission #2570

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
filimonov wants to merge 3 commits into
base: antalya-26.6-20261008
Choose a base branch
from
Open

CAS: refuse to republish a namespace life retired during a cold admission #2570

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
06ac915
Refuse to republish a namespace life retired during a cold admission
filimonov Oct 13, 2026
e8177d9
Record a namespace retirement even when a successor is resident
filimonov Oct 13, 2026
19b5df8
Use Allman braces in the new ref-name retirement accessor
filimonov Oct 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPool.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1131,6 +1131,10 @@ class Pool : public std::enable_shared_from_this<Pool>
{
return ref_ledger.refTableRuntimeAdmittedFenceGenerationForTest(ns);
}
uint64_t refNameRetirementsForTest(const RootNamespace & ns) const
{
return ref_ledger.refNameRetirementsForTest(ns);
}
std::optional<NamespaceLifeId> refTableLifeForTest(const RootNamespace & ns)
{
return ref_ledger.refTableLifeForTest(ns);
Expand Down
69 changes: 53 additions & 16 deletions src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefLedger.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -565,20 +565,31 @@ std::shared_ptr<CasRefLedger::RefTableRuntime> CasRefLedger::lookupRefTableRunti
return it == ref_name_slots.end() ? nullptr : it->second.current;
}

uint64_t CasRefLedger::refNameRetirements(const RootNamespace & ns) const
{
std::lock_guard lock(ref_queue_mutex);
const auto it = ref_name_slots.find(ns.string());
return it == ref_name_slots.end() ? 0 : it->second.retirements;
}

std::shared_ptr<CasRefLedger::RefTableRuntime> CasRefLedger::acquireRefTableRuntime(
const NamespaceLifeId & life, uint64_t admitted_generation)
const NamespaceLifeId & life, uint64_t admitted_generation, uint64_t retirements_seen)
{
refuseUnlessAdmitted(mount_requests.resume(admitted_generation), "ref-table runtime install");

std::shared_ptr<RefTableRuntime> result;
bool generation_moved = false;
bool retired_since_observation = false;
bool identity_conflict = false;
{
std::lock_guard lock(ref_queue_mutex);
generation_moved = fence_generation_fn() != admitted_generation;
if (!generation_moved)
const auto it = ref_name_slots.find(life.ns.string());
/// The catalog row this life came from may have been removed and drained after the caller read
/// it; inserting then would publish a runtime for a life the janitor sweeps.
retired_since_observation = (it == ref_name_slots.end() ? 0 : it->second.retirements) != retirements_seen;
if (!generation_moved && !retired_since_observation)
{
const auto it = ref_name_slots.find(life.ns.string());
if (it != ref_name_slots.end() && it->second.current)
{
const auto & current = it->second.current;
Expand All @@ -596,13 +607,18 @@ std::shared_ptr<CasRefLedger::RefTableRuntime> CasRefLedger::acquireRefTableRunt
next_ref_runtime_id.fetch_add(1, std::memory_order_relaxed) + 1,
life,
admitted_generation);
ref_name_slots.emplace(life.ns.string(), RefNameSlot{.current = result});
ref_name_slots[life.ns.string()].current = result;
}
}
}

if (generation_moved)
refuseUnlessAdmitted(mount_requests.resume(admitted_generation), "ref-table runtime install");
if (retired_since_observation)
throwCasWriteRetryLater(fmt::format(
"CAS namespace '{}': a life of this name was retired while life {} was being admitted; "
"retry from a fresh catalog observation",
life.ns.string(), renderIncarnation(life.incarnation)));
if (identity_conflict)
throwCasWriteRetryLater(fmt::format(
"CAS namespace '{}': the cached runtime identity changed while publishing catalog life {}; "
Expand Down Expand Up @@ -636,6 +652,7 @@ std::shared_ptr<CasRefLedger::RefTableRuntime> CasRefLedger::acquireReadableRefT
}

const uint64_t admitted_generation = fence_generation_fn();
const uint64_t retirements_seen = refNameRetirements(ns);
CasOperation op = mount_requests.resume(admitted_generation);
const CasRefCatalog::Snapshot first_catalog = CasRefCatalog::read(op, layout);
refuseUnlessAdmitted(op, "cold readable runtime admission");
Expand Down Expand Up @@ -672,31 +689,34 @@ std::shared_ptr<CasRefLedger::RefTableRuntime> CasRefLedger::acquireReadableRefT
"retry from a fresh catalog observation",
ns.string(), renderIncarnation(life.incarnation)));

return acquireRefTableRuntime(life, admitted_generation);
return acquireRefTableRuntime(life, admitted_generation, retirements_seen);
}

std::shared_ptr<CasRefLedger::RefTableRuntime> CasRefLedger::acquireMutableRefTableRuntime(
const RootNamespace & ns)
{
/// Captured before `namespaceLife`, whose answer may come from a runtime retired right after.
const uint64_t admitted_generation = fence_generation_fn();
const uint64_t retirements_seen = refNameRetirements(ns);
const NamespaceLifeId life = namespaceLife(ns);
if (const auto current = lookupRefTableRuntime(ns))
{
if (current->life == life)
return current;
}
const uint64_t admitted_generation = fence_generation_fn();
return acquireRefTableRuntime(life, admitted_generation);
return acquireRefTableRuntime(life, admitted_generation, retirements_seen);
}

void CasRefLedger::invalidateRemovedCatalogLife(const NamespaceLifeId & life)
{
std::shared_ptr<RefTableRuntime> rt;
{
std::lock_guard lock(ref_queue_mutex);
const auto it = ref_name_slots.find(life.ns.string());
if (it == ref_name_slots.end())
return;
rt = it->second.current;
/// Recorded whatever is resident, a successor included: a cold admission of `life` may be
/// between its catalog read and its insertion, and the successor may be evicted before it.
RefNameSlot & slot = ref_name_slots[life.ns.string()];
++slot.retirements;
rt = slot.current;
if (!rt)
return;
}
Expand All @@ -718,7 +738,7 @@ void CasRefLedger::invalidateRemovedCatalogLife(const NamespaceLifeId & life)
std::lock_guard queue_lock(ref_queue_mutex);
const auto it = ref_name_slots.find(life.ns.string());
if (it != ref_name_slots.end() && it->second.current == rt)
ref_name_slots.erase(it);
it->second.current.reset();
}
rt->cv.notify_all();
rt->recovery_cv.notify_all();
Expand Down Expand Up @@ -1806,7 +1826,8 @@ void CasRefLedger::enforceRefTableCacheBudget(const RootNamespace & keep_ns)
continue; /// re-check under the still-held ref_queue_mutex
total -= c.weight;
evicted.push_back(std::move(rt)); /// keep alive past the erase and lock release
ref_name_slots.erase(it);
if (it->second.retirements == 0)
ref_name_slots.erase(it);
ProfileEvents::increment(ProfileEvents::CASRefTableEvictions);
}
}
Expand Down Expand Up @@ -4943,12 +4964,14 @@ NamespaceLifeId CasRefLedger::namespaceLife(const RootNamespace & ns)
if (rt)
{
ensureRefTableRecovered(ns, *rt);
refuseTerminalLife(ns, *rt);
return rt->life;
}
}

/// A cold mutation observes or births the durable identity before allocating any local state.
const uint64_t admitted_generation = fence_generation_fn();
const uint64_t retirements_seen = refNameRetirements(ns);
CasOperation op = mount_requests.resume(admitted_generation);
const CasRefCatalog::Snapshot catalog = CasRefCatalog::read(op, layout);
refuseUnlessAdmitted(op, "cold mutable namespace admission");
Expand All @@ -4964,11 +4987,23 @@ NamespaceLifeId CasRefLedger::namespaceLife(const RootNamespace & ns)
? NamespaceLifeId::fromCatalogEntry(entry_it->ns, entry_it->incarnation)
: resolveNamespaceLife(ns, admitted_generation, live_epoch_fn());
refuseUnlessAdmitted(op, "cold mutable namespace admission, after life resolution");
rt = acquireRefTableRuntime(life, admitted_generation);
rt = acquireRefTableRuntime(life, admitted_generation, retirements_seen);
ensureRefTableRecovered(ns, *rt);
refuseTerminalLife(ns, *rt);
return rt->life;
}

void CasRefLedger::refuseTerminalLife(const RootNamespace & ns, RefTableRuntime & rt)
{
/// Bytes written under a removed life are swept by the dead-life janitor, so the caller must not be
/// handed one to write into. A never-born stream is also `Removed` but has no `remove_txn_id`.
std::lock_guard lock(rt.state_mutex);
if (rt.state.getLifecycle() != RefLifecycle::Live && rt.state.getRemoveTxnId().has_value())
throwCasWriteRetryLater(fmt::format(
"CAS namespace '{}': its resident life {} is already removed; retry after the removal completes",
ns.string(), renderIncarnation(rt.life.incarnation)));
}

std::optional<NamespaceLifeId> CasRefLedger::namespaceFilesLifeIfReadable(const RootNamespace & ns)
{
const auto rt = acquireReadableRefTableRuntime(ns);
Expand Down Expand Up @@ -5016,6 +5051,7 @@ bool CasRefLedger::namespaceStillLogicallyPresent(const RootNamespace & ns)
/// the one answer that must never be manufactured by a race, so it alone is re-confirmed by a
/// second read of this namespace's row before being trusted.
const uint64_t admitted_generation = fence_generation_fn();
const uint64_t retirements_seen = refNameRetirements(ns);
CasOperation op = mount_requests.resume(admitted_generation);
const CasRefCatalog::Snapshot first_catalog = CasRefCatalog::read(op, layout);
refuseUnlessAdmitted(op, "namespace presence probe");
Expand Down Expand Up @@ -5060,7 +5096,7 @@ bool CasRefLedger::namespaceStillLogicallyPresent(const RootNamespace & ns)
/// therefore not proof that the CURRENT logical namespace `ns` is absent; a fresh catalog read is
/// required before answering `false` for the name.
const NamespaceLifeId life = NamespaceLifeId::fromCatalogEntry(entry->ns, entry->incarnation);
const auto rt = acquireRefTableRuntime(life, admitted_generation);
const auto rt = acquireRefTableRuntime(life, admitted_generation, retirements_seen);
ensureRefTableRecovered(ns, *rt);
bool this_incarnation_terminal = false;
{
Expand Down Expand Up @@ -5099,6 +5135,7 @@ DropNamespaceStats CasRefLedger::dropNamespaceImpl(
/// class shares the bigger complete-table byte budget (encodeRefLogTxn's own `checkBudget`, keyed
/// off the presence of a `RemoveNamespace` op) and is exempt from the ordinary per-op admission
/// check (it only ever shrinks state; see `flushRefBatch`'s `state_growing` filter).
const uint64_t retirements_seen = refNameRetirements(ns);
CasOperation initial_op = mount_requests.admit();
const CasRefCatalog::Snapshot initial_catalog = CasRefCatalog::read(initial_op, layout);
const auto initial_it = std::find_if(initial_catalog.catalog.entries.begin(), initial_catalog.catalog.entries.end(),
Expand Down Expand Up @@ -5143,7 +5180,7 @@ DropNamespaceStats CasRefLedger::dropNamespaceImpl(
const NamespaceLifeId observed_life
= NamespaceLifeId::fromCatalogEntry(initial_it->ns, initial_it->incarnation);
const uint64_t runtime_generation = fence_generation_fn();
const auto rt = acquireRefTableRuntime(observed_life, runtime_generation);
const auto rt = acquireRefTableRuntime(observed_life, runtime_generation, retirements_seen);
ensureRefTableRecovered(ns, *rt);
{
/// A real terminal transaction is idempotent. The representation also calls an empty, never-born
Expand Down
32 changes: 25 additions & 7 deletions src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefLedger.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,8 @@ class CasRefLedger
/// resolution, and the ONLY one that CREATES: recovery's step 0 (`resolveNamespaceLife`) mints a
/// life when the catalog names none, so the first namespace file a table ever writes births the
/// namespace exactly as its first ref op would. A read or a removal must not use this -- see the
/// sibling below for why that is a correctness matter and not a preference.
/// sibling below for why that is a correctness matter and not a preference. Throws retry-later for
/// a life whose terminal removal is already applied, since the janitor sweeps whatever is written there.
NamespaceLifeId namespaceLife(const RootNamespace & ns);

/// The life a READER (or a REMOVER) of this namespace's files must use, or `nullopt` when there are
Expand Down Expand Up @@ -474,15 +475,15 @@ class CasRefLedger
{
std::lock_guard<std::mutex> g(ref_queue_mutex);
const auto it = ref_name_slots.find(ns.string());
return it == ref_name_slots.end() ? 0 : it->second.current->pending.size();
return it == ref_name_slots.end() || !it->second.current ? 0 : it->second.current->pending.size();
}

/// Reports whether `ns` currently has an active append-lane leader (the baton). Under the queue mutex.
bool refLeaderActiveForTest(const RootNamespace & ns)
{
std::lock_guard<std::mutex> g(ref_queue_mutex);
const auto it = ref_name_slots.find(ns.string());
return it != ref_name_slots.end() && it->second.current->leader_active;
return it != ref_name_slots.end() && it->second.current && it->second.current->leader_active;
}

/// Returns the number of items carved by the current tenure and not yet released by its exit guard.
Expand All @@ -491,7 +492,7 @@ class CasRefLedger
{
std::lock_guard<std::mutex> g(ref_queue_mutex);
const auto it = ref_name_slots.find(ns.string());
return it == ref_name_slots.end() ? 0 : it->second.current->carved.size();
return it == ref_name_slots.end() || !it->second.current ? 0 : it->second.current->carved.size();
}

/// Returns whether the `carved` entry for `ref_name` (if any) is already completed. Lets a test
Expand All @@ -501,7 +502,7 @@ class CasRefLedger
{
std::lock_guard<std::mutex> g(ref_queue_mutex);
const auto it = ref_name_slots.find(ns.string());
if (it == ref_name_slots.end())
if (it == ref_name_slots.end() || !it->second.current)
return false;
for (const auto & item : it->second.current->carved)
if (item->scope.kind == MutationScope::Kind::Ref && item->scope.ref_name == ref_name)
Expand Down Expand Up @@ -552,6 +553,11 @@ class CasRefLedger
const auto it = ref_name_slots.find(ns.string());
return it == ref_name_slots.end() || !it->second.current ? 0 : it->second.current->admitted_fence_generation;
}
/// How many lives of `ns` catalog retirement has recorded since the last remount.
uint64_t refNameRetirementsForTest(const RootNamespace & ns) const
{
return refNameRetirements(ns);
}
/// The physical life currently pinned in the cached runtime, without resolving or recovering it.
std::optional<NamespaceLifeId> refTableLifeForTest(const RootNamespace & ns)
{
Expand Down Expand Up @@ -936,6 +942,10 @@ class CasRefLedger
struct RefNameSlot
{
std::shared_ptr<RefTableRuntime> current;
/// Bumped by `invalidateRemovedCatalogLife`. A slot with a non-zero count is never erased (only
/// remount clears it, and remount also moves the fence generation), so a cold admission compares
/// the count it captured before its catalog read with the one at insertion.
uint64_t retirements = 0;
};
static constexpr size_t kMaxRefBatch = 1000;
/// Recovery retries at most this many times when an object selected by LIST vanishes before GET.
Expand Down Expand Up @@ -1037,9 +1047,17 @@ class CasRefLedger

/// Publishes or returns one runtime for an exact catalog-observed life and admitted mount-fence
/// generation. A conflicting attached identity is a stale observation and fails closed rather than
/// retargeting either runtime.
/// retargeting either runtime. `retirements_seen` is `refNameRetirements` captured before the
/// catalog read that produced `life`; if a retirement of this name landed since, the observation is
/// stale and the call throws retry-later instead of republishing a possibly retired life.
std::shared_ptr<RefTableRuntime> acquireRefTableRuntime(
const NamespaceLifeId & life, uint64_t admitted_generation);
const NamespaceLifeId & life, uint64_t admitted_generation, uint64_t retirements_seen);

/// The name's retirement count, under `ref_queue_mutex`. Zero for a name with no slot.
uint64_t refNameRetirements(const RootNamespace & ns) const;

/// Throws retry-later when `rt` (already recovered) has applied its namespace's terminal removal.
void refuseTerminalLife(const RootNamespace & ns, RefTableRuntime & rt);

/// Lookup-first non-minting read acquisition. A cold name consults the catalog and materializes only
/// an exact `Live` life; absence/`Creating`/`Removing` returns no runtime.
Expand Down
Loading
Loading