Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions docs/formats/codex.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,14 @@ are not followed, guarding against cycles; symlinked files still list). A file o
session if it contains a `session_meta` line carrying an `id`; discovery parses just those lines
and skips message payloads entirely. On load, a missing id falls back to the filename's uuid.

Codex's own `/archive` (TUI) and `codex archive`/`codex unarchive` (CLI) move a rollout out of
this dated tree into a flat sibling directory, `archived_sessions` (no `YYYY/MM/DD` sharding).
`CodexStore::default_root` sets `archived_sessions_dir` to that sibling, and discovery walks it
alongside `sessions_dir`, so an archived rollout is still listed — Codex's own session picker just
won't show it until it's unarchived back into `sessions/`. A `CodexStore` built directly from a
custom `sessions_dir` has no archived directory unless one is set with
`with_archived_sessions_dir`.

## Dissection of a transcript

Every line shares one envelope — upstream's `RolloutLine`: a `timestamp` (RFC 3339, millisecond
Expand Down
114 changes: 69 additions & 45 deletions src/harness/codex.rs
Original file line number Diff line number Diff line change
Expand Up @@ -688,23 +688,43 @@ fn meta_line_str(ts: &str, kind: &str, payload: Value) -> Line {
#[derive(Debug, Clone)]
pub struct CodexStore {
pub sessions_dir: PathBuf,
/// Codex's own `/archive` (TUI) and `codex archive`/`codex unarchive`
/// (CLI) move a rollout out of the dated `sessions_dir` tree into this
/// flat sibling directory, `archived_sessions`. `None` when unknown, as
/// for a `sessions_dir` built by hand that isn't under a Codex home.
pub archived_sessions_dir: Option<PathBuf>,
}

impl CodexStore {
pub fn new(sessions_dir: impl Into<PathBuf>) -> Self {
Self {
sessions_dir: sessions_dir.into(),
archived_sessions_dir: None,
}
}

/// Also discover rollouts Codex has archived into `dir`. New sessions
/// are always written to `sessions_dir`; this only widens discovery.
#[must_use]
pub fn with_archived_sessions_dir(mut self, dir: impl Into<PathBuf>) -> Self {
self.archived_sessions_dir = Some(dir.into());
self
}

/// The default sessions root: `$CODEX_HOME/sessions` when set (Codex
/// honors that override before its home lookup), else `~/.codex/sessions`.
/// `archived_sessions_dir` is set to the matching sibling
/// `archived_sessions` directory.
#[must_use]
pub fn default_root() -> Option<Self> {
std::env::var_os("CODEX_HOME")
.filter(|v| !v.is_empty())
.map(|codex_home| Self::new(PathBuf::from(codex_home).join("sessions")))
.or_else(|| home().map(|h| Self::new(h.join(".codex").join("sessions"))))
.map(PathBuf::from)
.or_else(|| home().map(|h| h.join(".codex")))
.map(|codex_home| {
Self::new(codex_home.join("sessions"))
.with_archived_sessions_dir(codex_home.join("archived_sessions"))
})
}
}

Expand All @@ -713,52 +733,56 @@ impl Store for CodexStore {
type Ref = PathBuf;

fn discover(&self) -> Result<Vec<Discovered<PathBuf>>> {
let mut files = Vec::new();
if self.sessions_dir.is_dir() {
let mut files = Vec::new();
collect_rollouts(&self.sessions_dir, &mut files);
Ok(super::filter_map_parallel(&files, |path| {
// A rollout that fails to read, or lacks a session_meta with
// an id, is not a resumable session. Only session_meta lines
// are parsed — message payloads are skipped whole — and the
// read stops at the first session_meta carrying the id, which
// is line one of a well-formed rollout. Reading the rest would
// mean pulling every byte of every rollout on the machine
// through a JSON probe to learn nothing more.
let has_id = |l: &Line| l.payload.get("id").and_then(Value::as_str).is_some();
let file = fs::File::open(path).ok()?;
let mut first: Option<Line> = None;
let mut found_id = false;
for line in BufReader::new(file).lines().map_while(std::io::Result::ok) {
if line.trim().is_empty() || !is_session_meta(&line) {
continue;
}
let Ok(parsed) = serde_json::from_str::<Line>(&line) else {
continue;
};
found_id = has_id(&parsed);
if first.is_none() {
first = Some(parsed);
}
if found_id {
break;
}
}
let first = first?;
found_id.then(|| {
let mut meta = meta_from_lines(std::slice::from_ref(&first));
if meta.id.is_empty() {
meta.id = jsonl::file_id(path);
}
Discovered {
meta,
reference: path.clone(),
}
})
}))
} else {
// A missing sessions root means no sessions, not an error.
Ok(Vec::new())
}
// A missing sessions root or archived directory means no sessions
// there, not an error; `files` is simply left short.
if let Some(archived) = self.archived_sessions_dir.as_deref()
&& archived.is_dir()
{
collect_rollouts(archived, &mut files);
}
Ok(super::filter_map_parallel(&files, |path| {
// A rollout that fails to read, or lacks a session_meta with
// an id, is not a resumable session. Only session_meta lines
// are parsed — message payloads are skipped whole — and the
// read stops at the first session_meta carrying the id, which
// is line one of a well-formed rollout. Reading the rest would
// mean pulling every byte of every rollout on the machine
// through a JSON probe to learn nothing more.
let has_id = |l: &Line| l.payload.get("id").and_then(Value::as_str).is_some();
let file = fs::File::open(path).ok()?;
let mut first: Option<Line> = None;
let mut found_id = false;
for line in BufReader::new(file).lines().map_while(std::io::Result::ok) {
if line.trim().is_empty() || !is_session_meta(&line) {
continue;
}
let Ok(parsed) = serde_json::from_str::<Line>(&line) else {
continue;
};
found_id = has_id(&parsed);
if first.is_none() {
first = Some(parsed);
}
if found_id {
break;
}
}
let first = first?;
found_id.then(|| {
let mut meta = meta_from_lines(std::slice::from_ref(&first));
if meta.id.is_empty() {
meta.id = jsonl::file_id(path);
}
Discovered {
meta,
reference: path.clone(),
}
})
}))
}

fn load(&self, reference: &PathBuf) -> Result<Transcript<Codex>> {
Expand Down
59 changes: 59 additions & 0 deletions tests/integration/codex.rs
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,65 @@ fn discover_extracts_metadata() {
assert_eq!(meta.cli_version.as_deref(), Some("0.104.0"));
}

/// Codex's own `/archive` (TUI) / `codex archive` (CLI) moves a rollout out
/// of the dated `sessions_dir` tree into a flat sibling `archived_sessions`
/// directory. Discovery must still find it there.
#[test]
fn discover_includes_archived_sessions() {
let dir = tempfile::tempdir().unwrap();
let sessions = dir.path().join("sessions");
let nested = sessions.join("2026").join("01").join("02");
std::fs::create_dir_all(&nested).unwrap();
std::fs::write(
nested.join("rollout-2026-01-02T03-04-05-sess-active.jsonl"),
format!(
"{}\n",
r#"{"timestamp":"2026-01-02T03:04:05.000Z","type":"session_meta","payload":{"id":"sess-active"}}"#,
),
)
.unwrap();

let archived = dir.path().join("archived_sessions");
std::fs::create_dir_all(&archived).unwrap();
std::fs::write(
archived.join("rollout-2025-12-01T00-00-00-sess-archived.jsonl"),
format!(
"{}\n",
r#"{"timestamp":"2025-12-01T00:00:00.000Z","type":"session_meta","payload":{"id":"sess-archived"}}"#,
),
)
.unwrap();

let store = codex::CodexStore::new(&sessions).with_archived_sessions_dir(&archived);
let mut ids: Vec<_> = store
.discover()
.unwrap()
.into_iter()
.map(|d| d.meta.id)
.collect();
ids.sort();
assert_eq!(ids, vec!["sess-active", "sess-archived"]);
}

/// A store with no known archived directory (the common case for a
/// hand-built `sessions_dir`) only discovers the active tree.
#[test]
fn discover_without_archived_dir_only_finds_active_sessions() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(
dir.path().join("rollout-2026-01-02T03-04-05-sess-1.jsonl"),
format!(
"{}\n",
r#"{"timestamp":"2026-01-02T03:04:05.000Z","type":"session_meta","payload":{"id":"sess-1"}}"#,
),
)
.unwrap();

let found = codex::CodexStore::new(dir.path()).discover().unwrap();
assert_eq!(found.len(), 1);
assert_eq!(found[0].meta.id, "sess-1");
}

/// Shaped at codex's granularity: each assistant block is its own message
/// (codex stores one `response_item` per line), every assistant turn carries a
/// model, only the final text turn carries usage, and `stop_reason` is None
Expand Down
Loading