Layout F: the index split, each level pack carrying its own index, the history leveled

packages/protocol/src/repo/layouts/kit.ts

@@ -95,8 +95,32 @@ export interface PackInfo {
   level?: number
   /** A pack of replaced versions a leveled merge moved out, by month written (`2026-10`). */
   history?: string
+  /**
+   * The pack carries its own index (`packIndex`): its records' places, in a layer of
+   * its own, so a leveled collection finds what its levels hold without the index
+   * every copy used to need an entry in.
+   */
+  indexed?: true
 }
 
+/** The layer indexing a pack's records (`PackInfo.indexed`): named after the pack. */
+export const packIndexId = (pack: string) => `${pack}i`
+export const packIndex = (p: PackInfo): LayerInfo => ({
+  id: packIndexId(p.id),
+  level: 0,
+  seq: 0,
+  entries: p.records,
+  bytes: 0,
+  min: p.lo!,
+  max: p.hi!,
+})
+/** The objects of a pack, with its index's when it has one. */
+export const packKeys = (p: { id: string; indexed?: true }) => [
+  packKey(p.id),
+  packDirKey(p.id),
+  ...(p.indexed ? [layerKey(packIndexId(p.id)), layerDirKey(packIndexId(p.id))] : []),
+]
+
 /** The record type a key leads with (recordKey); '' for a key without one. */
 const typeOf = (key: string) => {
   const i = key.indexOf('\u0000')
@@ -123,6 +147,7 @@ export class PackWriter {
   #type = ''
   #lo = ''
   #hi = ''
+  #index: LayerWriter | null = null
   readonly done: PackInfo[] = []
   readonly #writing = new Set<Promise<void>>()
   #error: unknown = null
@@ -136,6 +161,12 @@ export class PackWriter {
       history?: string
       /** Raw bytes a pack closes at (PACK_RAW). */
       packRaw?: number
+      /**
+       * Write each pack's own index (`PackInfo.indexed`) beside it; a promise is read
+       * when the pack is written (the index is built as records come, then dropped
+       * if it turns out unwanted).
+       */
+      index?: boolean | Promise<boolean>
     } = {},
   ) {}
 
@@ -152,10 +183,12 @@ export class PackWriter {
     if (!this.#id) {
       this.#id = newId()
       this.#lo = key
+      if (this.opts.index) this.#index = new LayerWriter(this.store, 0, packIndexId(this.#id))
     }
     this.#hi = key
     this.#lines.push(`${key}\t${line}`)
     const at = loc(this.#id, this.#blocks.length, this.#lines.length - 1)
+    this.#index?.add(key, at)
     this.#raw += key.length + line.length + 2
     this.#records++
     if (this.#raw >= BLOCK_RAW) this.#seal()
@@ -200,7 +233,9 @@ export class PackWriter {
       type: this.#type,
       lo: this.#lo,
       hi: this.#hi,
+      index: this.#index,
     }
+    this.#index = null
     this.#id = ''
     this.#blocks = []
     this.#records = 0
@@ -221,13 +256,17 @@ export class PackWriter {
     type: string
     lo: string
     hi: string
+    index: LayerWriter | null
   }) {
     const parts = await Promise.all(job.blocks)
     const offsets = [0]
     for (const p of parts) offsets.push(offsets.at(-1)! + p.byteLength)
     await this.opts.hooks?.before?.(job.id)
     await putJoined(this.store, packKey(job.id), parts, { ifAbsent: true })
     await this.store.put(packDirKey(job.id), JSON.stringify(offsets), { ifAbsent: true })
+    // Its index after the pack itself: a listed pack's index is always there.
+    const indexed = job.index !== null && (await this.opts.index) === true
+    if (indexed) await job.index!.finish()
     const pack: PackInfo = {
       id: job.id,
       tier: this.opts.tier ?? 0,
@@ -239,6 +278,7 @@ export class PackWriter {
       hi: job.hi,
       ...(this.opts.level ? { level: this.opts.level } : {}),
       ...(this.opts.history ? { history: this.opts.history } : {}),
+      ...(indexed ? { indexed: true as const } : {}),
     }
     this.done.push(pack)
     await this.opts.hooks?.after?.(pack)
@@ -539,7 +579,6 @@ function pageAt(dir: LayerDir, key: string): number {
 }
 
 export class LayerWriter {
-  readonly id = newId()
   #pages: Promise<Uint8Array>[] = []
   #first: string[] = []
   #lines: string[] = []
@@ -554,6 +593,7 @@ export class LayerWriter {
   constructor(
     readonly store: Store,
     readonly level: number,
+    readonly id = newId(),
   ) {}
 
   /** Add in key order (a repeated key keeps the first). */
@@ -888,6 +928,8 @@ export interface Manifest {
    * one's packs. `packs` above is level 0.
    */
   levels?: Levels
+  /** A leveled collection's history level 0: replaced versions merges moved out, not yet merged down. */
+  history?: PackInfo[]
 }
 
 const empty = (): Manifest => ({ seq: 0, layers: [], packs: [], retired: [], stats: {} })

packages/protocol/src/repo/layouts/levels.ts

@@ -15,6 +15,7 @@
  * key range touches. Readers never read them: records are found through the index.
  */
 import { compareUtf8 } from '../../utf8.js'
+import type { Lru } from '../lru.js'
 import type { Store } from '../types.js'
 import { newId, PACK_RAW, type PackInfo } from './kit.js'
 
@@ -44,6 +45,13 @@ export const levelConfig = {
   packRaw: PACK_RAW,
   /** Packs a shard lists at most. */
   shardPacks: 512,
+  /**
+   * Index entries that can no longer serve a lookup (Manifest.stats.indexStale) past
+   * which, once they're half the index, every run merges at once to drop them. The
+   * index is about level 0 and these, so that merge is small (50k entries: ~3 MB);
+   * until it runs, a lookup reads their runs for nothing, and merged-down packs stay.
+   */
+  staleEntries: 50_000,
 }
 
 /** A shard of a level's listing, as the manifest names it. */
@@ -57,8 +65,17 @@ export interface LevelShard {
   bytes: number
 }
 
+/**
+ * By level key: `1`…`6` the current versions' levels, `h1`…`h6` the history's (replaced
+ * versions, every one kept, leveled the same way so an old version's lookup reads a
+ * pack a level, not one a month).
+ */
 export type Levels = Record<string, LevelShard[]>
 
+/** The two trees a leveled collection keeps: current versions, and the history. */
+export type Tree = '' | 'h'
+export const levelKey = (tree: Tree, level: number) => `${tree}${level}`
+
 export const shardKey = (id: string) => `manifest/f/levels/${id}.json`
 
 /** Whether [aLo, aHi] and [bLo, bHi] share a key. */
@@ -68,8 +85,8 @@ export const overlaps = (aLo: string, aHi: string, bLo: string, bHi: string) =>
 const byLo = (a: PackInfo, b: PackInfo) => compareUtf8(a.lo!, b.lo!)
 
 /** Raw bytes a level holds. */
-export const levelRaw = (levels: Levels | undefined, level: number) =>
-  (levels?.[level] ?? []).reduce((n, s) => n + s.raw, 0)
+export const levelRaw = (levels: Levels | undefined, level: number, tree: Tree = '') =>
+  (levels?.[levelKey(tree, level)] ?? []).reduce((n, s) => n + s.raw, 0)
 
 /**
  * Each level's target size from the bottom up, and the base level, where level 0
@@ -78,9 +95,12 @@ export const levelRaw = (levels: Levels | undefined, level: number) =>
  * 0's merges and goes down a pack at a time). Levels above the base have no room
  * (target 0).
  */
-export function levelTargets(levels: Levels | undefined): { base: number; targets: number[] } {
+export function levelTargets(
+  levels: Levels | undefined,
+  tree: Tree = '',
+): { base: number; targets: number[] } {
   const targets = Array.from<number>({ length: MAX_LEVEL + 1 }).fill(0)
-  let t = levelRaw(levels, MAX_LEVEL)
+  let t = levelRaw(levels, MAX_LEVEL, tree)
   targets[MAX_LEVEL] = Infinity
   let base = MAX_LEVEL - 1
   targets[base] = t / LEVEL_RATIO
@@ -96,19 +116,34 @@ export function levelTargets(levels: Levels | undefined): { base: number; target
 
 /** The shards a leveled collection's levels are listed in (each read once: they never change). */
 export class LevelShards {
-  readonly #read = new Map<string, Promise<PackInfo[]>>()
+  readonly #loading = new Map<string, Promise<PackInfo[]>>()
+
+  /** Shards are written once: kept in the layout's memory cache, by id. */
+  constructor(
+    readonly store: Store,
+    readonly lru: Lru<unknown>,
+  ) {}
 
-  constructor(readonly store: Store) {}
+  #keep(id: string, packs: PackInfo[], bytes: number) {
+    this.lru.set(`shard:${id}`, { packs, approxBytes: bytes * 2 })
+  }
 
   read(id: string): Promise<PackInfo[]> {
-    let p = this.#read.get(id)
+    const hit = this.lru.get(`shard:${id}`) as { packs: PackInfo[] } | undefined
+    if (hit) return Promise.resolve(hit.packs)
+    let p = this.#loading.get(id)
     if (!p) {
-      p = this.store.get(shardKey(id)).then(async (o) => {
-        if (!o) throw new Error(`Missing level shard ${id}`)
-        return JSON.parse(await o.text()) as PackInfo[]
-      })
-      p.catch(() => this.#read.delete(id))
-      this.#read.set(id, p)
+      p = this.store
+        .get(shardKey(id))
+        .then(async (o) => {
+          if (!o) throw new Error(`Missing level shard ${id}`)
+          const text = await o.text()
+          const packs = JSON.parse(text) as PackInfo[]
+          this.#keep(id, packs, text.length)
+          return packs
+        })
+        .finally(() => this.#loading.delete(id))
+      this.#loading.set(id, p)
     }
     return p
   }
@@ -134,8 +169,9 @@ export class LevelShards {
     for (let i = 0; i < packs.length; i += levelConfig.shardPacks) {
       const ps = packs.slice(i, i + levelConfig.shardPacks)
       const id = newId()
-      this.#read.set(id, Promise.resolve(ps))
-      await this.store.put(shardKey(id), JSON.stringify(ps), { ifAbsent: true })
+      const body = JSON.stringify(ps)
+      await this.store.put(shardKey(id), body, { ifAbsent: true })
+      this.#keep(id, ps, body.length)
       out.push({
         id,
         lo: ps[0]!.lo!,