From cb234167b87d9605558f59345fd8c40e90d9d827 Mon Sep 17 00:00:00 2001 From: Ishaan Gupta Date: Wed, 8 Jul 2026 12:03:07 +0530 Subject: [PATCH 1/3] Add LongMemEval-V2 benchmark adapter --- src/benchmarks/README.md | 1 + src/benchmarks/index.ts | 4 +- src/benchmarks/longmemeval-v2/index.ts | 305 +++++++++++++++++++++++++ src/benchmarks/longmemeval-v2/types.ts | 30 +++ src/cli/index.ts | 5 + src/types/benchmark.ts | 2 +- 6 files changed, 345 insertions(+), 2 deletions(-) create mode 100644 src/benchmarks/longmemeval-v2/index.ts create mode 100644 src/benchmarks/longmemeval-v2/types.ts diff --git a/src/benchmarks/README.md b/src/benchmarks/README.md index 8d43cf9..599a099 100644 --- a/src/benchmarks/README.md +++ b/src/benchmarks/README.md @@ -35,6 +35,7 @@ interface Benchmark { |-----------|--------|-------------| | `locomo` | GitHub snap-research/locomo | Long context memory benchmark | | `longmemeval` | HuggingFace xiaowu0162/longmemeval-cleaned | Long-term memory evaluation | +| `longmemeval-v2` | HuggingFace xiaowu0162/longmemeval-v2 | Memory over multimodal web-agent trajectories | | `convomem` | HuggingFace Salesforce/ConvoMem | Conversational memory benchmark | ## Question Types diff --git a/src/benchmarks/index.ts b/src/benchmarks/index.ts index b790e87..6c20ed4 100644 --- a/src/benchmarks/index.ts +++ b/src/benchmarks/index.ts @@ -1,11 +1,13 @@ import type { Benchmark, BenchmarkName } from "../types/benchmark" import { LoCoMoBenchmark } from "./locomo" import { LongMemEvalBenchmark } from "./longmemeval" +import { LongMemEvalV2Benchmark } from "./longmemeval-v2" import { ConvoMemBenchmark } from "./convomem" const benchmarks: Record Benchmark> = { locomo: LoCoMoBenchmark, longmemeval: LongMemEvalBenchmark, + "longmemeval-v2": LongMemEvalV2Benchmark, convomem: ConvoMemBenchmark, } @@ -21,4 +23,4 @@ export function getAvailableBenchmarks(): BenchmarkName[] { return Object.keys(benchmarks) as BenchmarkName[] } -export { LoCoMoBenchmark, LongMemEvalBenchmark, ConvoMemBenchmark } +export { LoCoMoBenchmark, LongMemEvalBenchmark, LongMemEvalV2Benchmark, ConvoMemBenchmark } diff --git a/src/benchmarks/longmemeval-v2/index.ts b/src/benchmarks/longmemeval-v2/index.ts new file mode 100644 index 0000000..6687fe5 --- /dev/null +++ b/src/benchmarks/longmemeval-v2/index.ts @@ -0,0 +1,305 @@ +import { existsSync, readFileSync } from "fs" +import { join } from "path" +import type { Benchmark, BenchmarkConfig, QuestionFilter } from "../../types/benchmark" +import type { + QuestionTypeRegistry, + UnifiedMessage, + UnifiedQuestion, + UnifiedSession, +} from "../../types/unified" +import { logger } from "../../utils/logger" +import type { LongMemEvalV2Question, LongMemEvalV2State, LongMemEvalV2Trajectory } from "./types" + +const DEFAULT_DATA_PATH = "./data/benchmarks/longmemeval-v2" +const DEFAULT_TIER = "small" +const TREE_EXCERPT_CHAR_LIMIT = 12000 +const COMPACT_UI_CHAR_LIMIT = 12000 + +type HaystackMap = Record + +function readJsonl(path: string): T[] { + return readFileSync(path, "utf8") + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean) + .map((line) => JSON.parse(line) as T) +} + +function requireFile(path: string, description: string): void { + if (!existsSync(path)) { + throw new Error( + `Missing ${description}: ${path}\n` + + "Download LongMemEval-V2 data first, then set BenchmarkConfig.dataPath or place files under data/benchmarks/longmemeval-v2." + ) + } +} + +function truncate(value: string, limit: number): string { + if (value.length <= limit) return value + return `${value.slice(0, limit)}\n...[truncated ${value.length - limit} chars]` +} + +function uniquePush(items: string[], seen: Set, value: string): void { + const normalized = value.replace(/\s+/g, " ").trim() + if (!normalized || normalized.length < 2 || seen.has(normalized)) return + seen.add(normalized) + items.push(normalized) +} + +function extractQuotedLabels(line: string): string[] { + const labels: string[] = [] + const patterns = [ + /\b(?:button|link|menuitem|option|combobox|textbox|searchbox|checkbox|heading|gridcell|cell|rowheader|columnheader|StaticText)\s+'([^']+)'/g, + /\bvalue='([^']+)'/g, + /\bplaceholder='([^']+)'/g, + ] + + for (const pattern of patterns) { + for (const match of line.matchAll(pattern)) { + labels.push(match[1]) + } + } + + return labels +} + +function compactAccessibilityTree(tree?: string | null): string { + if (!tree) return "" + + const roleLinePattern = + /\b(button|link|menuitem|option|combobox|textbox|searchbox|checkbox|heading|gridcell|cell|rowheader|columnheader|StaticText|listitem)\b/ + const labels: string[] = [] + const roleLines: string[] = [] + const seenLabels = new Set() + const seenLines = new Set() + + for (const rawLine of tree.split(/\r?\n/)) { + const line = rawLine.trim() + if (!line) continue + + for (const label of extractQuotedLabels(line)) { + uniquePush(labels, seenLabels, label) + } + + if (roleLinePattern.test(line)) { + uniquePush(roleLines, seenLines, line) + } + + const compactLength = labels.join("\n").length + roleLines.join("\n").length + if (compactLength > COMPACT_UI_CHAR_LIMIT) break + } + + const sections: string[] = [] + if (labels.length) { + sections.push(`UI labels and values:\n${labels.map((label) => `- ${label}`).join("\n")}`) + } + if (roleLines.length) { + sections.push(`Relevant accessibility lines:\n${roleLines.join("\n")}`) + } + + return truncate(sections.join("\n\n"), COMPACT_UI_CHAR_LIMIT) +} + +function stateToSession( + trajectory: LongMemEvalV2Trajectory, + state: LongMemEvalV2State +): UnifiedSession { + const compactTree = compactAccessibilityTree(state.accessibility_tree) + const treeExcerpt = state.accessibility_tree + ? truncate(state.accessibility_tree, TREE_EXCERPT_CHAR_LIMIT) + : "" + + const content = [ + "LongMemEval-V2 agent trajectory state", + `Trajectory ID: ${trajectory.id}`, + `Domain: ${trajectory.domain}`, + `Environment: ${trajectory.environment}`, + `Outcome: ${trajectory.outcome || "unknown"}`, + `Goal: ${trajectory.goal}`, + `State index: ${state.state_index}`, + state.step !== undefined ? `Step: ${state.step}` : "", + state.url ? `URL: ${state.url}` : "", + state.action ? `Action: ${state.action}` : "Action: null", + state.thought ? `Agent thought: ${state.thought}` : "", + state.screenshot ? `Screenshot path: ${state.screenshot}` : "", + compactTree ? `\nCompact UI extraction:\n${compactTree}` : "", + treeExcerpt ? `\nAccessibility tree excerpt:\n${treeExcerpt}` : "", + ] + .filter(Boolean) + .join("\n") + + return { + sessionId: `lme-v2-${trajectory.id}-state-${state.state_index}`, + messages: [{ role: "assistant", content }], + metadata: { + benchmark: "longmemeval-v2", + trajectoryId: trajectory.id, + stateIndex: state.state_index, + domain: trajectory.domain, + environment: trajectory.environment, + outcome: trajectory.outcome, + url: state.url, + screenshot: state.screenshot, + }, + } +} + +function trajectoryOverviewToSession(trajectory: LongMemEvalV2Trajectory): UnifiedSession { + const actionTrace = trajectory.states + .map((state) => { + const action = state.action || "null" + const thought = state.thought ? ` | thought: ${state.thought}` : "" + return `- state ${state.state_index}: action=${action}${thought}` + }) + .join("\n") + + const messages: UnifiedMessage[] = [ + { + role: "user", + content: [ + "LongMemEval-V2 trajectory overview", + `Trajectory ID: ${trajectory.id}`, + `Domain: ${trajectory.domain}`, + `Environment: ${trajectory.environment}`, + `Outcome: ${trajectory.outcome || "unknown"}`, + `Start URL: ${trajectory.start_url || "unknown"}`, + `Goal: ${trajectory.goal}`, + ].join("\n"), + }, + { + role: "assistant", + content: `Action/thought trace:\n${truncate(actionTrace, TREE_EXCERPT_CHAR_LIMIT)}`, + }, + ] + + return { + sessionId: `lme-v2-${trajectory.id}-overview`, + messages, + metadata: { + benchmark: "longmemeval-v2", + trajectoryId: trajectory.id, + domain: trajectory.domain, + environment: trajectory.environment, + outcome: trajectory.outcome, + sessionType: "trajectory-overview", + }, + } +} + +export class LongMemEvalV2Benchmark implements Benchmark { + name = "longmemeval-v2" + private questions: UnifiedQuestion[] = [] + private sessionsMap: Map = new Map() + private questionTypes: QuestionTypeRegistry = {} + + async load(config?: BenchmarkConfig): Promise { + const dataPath = config?.dataPath || DEFAULT_DATA_PATH + const tier = process.env.LONGMEMEVAL_V2_TIER || process.env.LME_V2_TIER || DEFAULT_TIER + const fullPath = join(process.cwd(), dataPath) + const questionsPath = join(fullPath, "questions.jsonl") + const trajectoriesPath = join(fullPath, "trajectories.jsonl") + const haystackPath = join(fullPath, "haystacks", `lme_v2_${tier}.json`) + + requireFile(questionsPath, "LongMemEval-V2 questions.jsonl") + requireFile(trajectoriesPath, "LongMemEval-V2 trajectories.jsonl") + requireFile(haystackPath, `LongMemEval-V2 ${tier} haystack`) + + const rawQuestions = readJsonl(questionsPath) + const haystack = JSON.parse(readFileSync(haystackPath, "utf8")) as HaystackMap + const haystackQuestionIds = new Set(Object.keys(haystack)) + const selectedTrajectoryIds = new Set(Object.values(haystack).flat()) + + const trajectorySessions = this.loadTrajectorySessions(trajectoriesPath, selectedTrajectoryIds) + + for (const item of rawQuestions) { + if (!haystackQuestionIds.has(item.id)) continue + + const sessions = haystack[item.id].flatMap((trajectoryId) => { + const trajectorySessionList = trajectorySessions.get(trajectoryId) + if (!trajectorySessionList) { + logger.warn(`Missing LongMemEval-V2 trajectory ${trajectoryId} for question ${item.id}`) + return [] + } + return trajectorySessionList + }) + + this.questions.push({ + questionId: item.id, + question: item.question, + questionType: item.question_type, + groundTruth: item.answer, + haystackSessionIds: sessions.map((session) => session.sessionId), + metadata: { + domain: item.domain, + environment: item.environment, + image: item.image, + evalFunction: item.eval_function, + haystackTier: tier, + trajectoryCount: haystack[item.id].length, + }, + }) + + this.sessionsMap.set(item.id, sessions) + this.questionTypes[item.question_type] ||= { + id: item.question_type, + alias: item.question_type, + description: `${item.question_type} questions`, + } + } + + logger.info(`Loaded ${this.questions.length} LongMemEval-V2 ${tier} questions from ${dataPath}`) + } + + private loadTrajectorySessions( + trajectoriesPath: string, + selectedTrajectoryIds: Set + ): Map { + const sessions = new Map() + + for (const trajectory of readJsonl(trajectoriesPath)) { + if (!selectedTrajectoryIds.has(trajectory.id)) continue + + const trajectorySessions = [ + trajectoryOverviewToSession(trajectory), + ...trajectory.states.map((state) => stateToSession(trajectory, state)), + ] + sessions.set(trajectory.id, trajectorySessions) + } + + logger.info(`Prepared ${sessions.size} LongMemEval-V2 trajectories for ingestion`) + return sessions + } + + getQuestions(filter?: QuestionFilter): UnifiedQuestion[] { + let result = [...this.questions] + + if (filter?.questionTypes?.length) { + result = result.filter((question) => filter.questionTypes!.includes(question.questionType)) + } + + if (filter?.offset) { + result = result.slice(filter.offset) + } + + if (filter?.limit) { + result = result.slice(0, filter.limit) + } + + return result + } + + getHaystackSessions(questionId: string): UnifiedSession[] { + return this.sessionsMap.get(questionId) || [] + } + + getGroundTruth(questionId: string): string { + const question = this.questions.find((item) => item.questionId === questionId) + return question?.groundTruth || "" + } + + getQuestionTypes(): QuestionTypeRegistry { + return this.questionTypes + } +} + +export default LongMemEvalV2Benchmark diff --git a/src/benchmarks/longmemeval-v2/types.ts b/src/benchmarks/longmemeval-v2/types.ts new file mode 100644 index 0000000..7c0395e --- /dev/null +++ b/src/benchmarks/longmemeval-v2/types.ts @@ -0,0 +1,30 @@ +export interface LongMemEvalV2Question { + id: string + domain: string + environment: string + question_type: string + question: string + image?: string | null + answer: string + eval_function?: string +} + +export interface LongMemEvalV2State { + state_index: number + step?: number + url?: string + action?: string | null + thought?: string | null + accessibility_tree?: string | null + screenshot?: string | null +} + +export interface LongMemEvalV2Trajectory { + id: string + domain: string + environment: string + goal: string + outcome?: string + start_url?: string + states: LongMemEvalV2State[] +} diff --git a/src/cli/index.ts b/src/cli/index.ts index b3c29d6..b2cc4c1 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -151,10 +151,15 @@ Available benchmark datasets for evaluation: Tests: user facts, assistant facts, preferences, implicit connections Source: HuggingFace Salesforce/ConvoMem (downloaded on first use) + longmemeval-v2 LongMemEval-V2 - Memory over web-agent trajectories + Tests: static UI facts, dynamic environment state, workflows, gotchas + Source: HuggingFace xiaowu0162/longmemeval-v2 (local data required) + Usage: -b locomo Run LoCoMo benchmark -b longmemeval Run LongMemEval benchmark -b convomem Run ConvoMem benchmark + -b longmemeval-v2 Run LongMemEval-V2 benchmark `) } diff --git a/src/types/benchmark.ts b/src/types/benchmark.ts index 07961e4..dccf507 100644 --- a/src/types/benchmark.ts +++ b/src/types/benchmark.ts @@ -20,4 +20,4 @@ export interface Benchmark { getQuestionTypes(): QuestionTypeRegistry } -export type BenchmarkName = "locomo" | "longmemeval" | "convomem" +export type BenchmarkName = "locomo" | "longmemeval" | "longmemeval-v2" | "convomem" From 9bef51be94b46611aa0625a3625f9475805b54c1 Mon Sep 17 00:00:00 2001 From: Ishaan Gupta Date: Wed, 8 Jul 2026 13:59:07 +0530 Subject: [PATCH 2/3] Document LongMemEval-V2 ingest flow --- src/benchmarks/longmemeval-v2/README.md | 96 +++++++++++++++++++++++++ src/cli/commands/ingest.ts | 25 +++++-- 2 files changed, 116 insertions(+), 5 deletions(-) create mode 100644 src/benchmarks/longmemeval-v2/README.md diff --git a/src/benchmarks/longmemeval-v2/README.md b/src/benchmarks/longmemeval-v2/README.md new file mode 100644 index 0000000..cc7500b --- /dev/null +++ b/src/benchmarks/longmemeval-v2/README.md @@ -0,0 +1,96 @@ +# LongMemEval-V2 + +This benchmark adapter loads LongMemEval-V2 web-agent trajectory data from local files and feeds it into the normal MemoryBench pipeline. + +## Data layout + +Place the downloaded dataset here: + +```text +data/benchmarks/longmemeval-v2/ + questions.jsonl + trajectories.jsonl + haystacks/ + lme_v2_small.json + lme_v2_medium.json +``` + +If you already downloaded the dataset into the sibling scratch folder used during local testing, copy it from the MemoryBench repo root with: + +```powershell +Copy-Item -Recurse ..\memory-bench\data\longmemeval-v2 data\benchmarks\longmemeval-v2 +``` + +The adapter reads `LONGMEMEVAL_V2_TIER` or `LME_V2_TIER` to choose the haystack tier. If neither is set, it uses `small`. + +## Ingest one question into Supermemory + +PowerShell: + +```powershell +$env:SUPERMEMORY_API_KEY = "sm_xxx" +$env:LONGMEMEVAL_V2_TIER = "small" +bun run src/index.ts ingest -p supermemory -b longmemeval-v2 -r lme-v2-01307e07 -q 01307e07 --force +``` + +Bash: + +```bash +SUPERMEMORY_API_KEY=sm_xxx \ +LONGMEMEVAL_V2_TIER=small \ +bun run src/index.ts ingest \ + -p supermemory \ + -b longmemeval-v2 \ + -r lme-v2-01307e07 \ + -q 01307e07 \ + --force +``` + +That creates one MemoryBench checkpoint and a Supermemory container tag for the question. The container tag is stored in `data/runs/lme-v2-01307e07/checkpoint.json` and follows: + +```text +- +``` + +## Search after ingest + +PowerShell: + +```powershell +$env:SUPERMEMORY_API_KEY = "sm_xxx" +$env:LONGMEMEVAL_V2_TIER = "small" +bun run src/index.ts search -r lme-v2-01307e07 +``` + +Bash: + +```bash +SUPERMEMORY_API_KEY=sm_xxx \ +LONGMEMEVAL_V2_TIER=small \ +bun run src/index.ts search -r lme-v2-01307e07 +``` + +Search results are written under: + +```text +data/runs/lme-v2-01307e07/results/.json +``` + +## Ingest a smoke subset by count + +PowerShell: + +```powershell +$env:SUPERMEMORY_API_KEY = "sm_xxx" +$env:LONGMEMEVAL_V2_TIER = "small" +bun run src/index.ts ingest -p supermemory -b longmemeval-v2 -r lme-v2-small-1 -l 1 --force +``` + +## What gets ingested + +For every trajectory in a question's haystack, the adapter creates: + +- one trajectory overview session with the goal, outcome, start URL, and action/thought trace +- one state session per trajectory state with URL, action, thought, screenshot path, compact UI labels/options, and a bounded accessibility-tree excerpt + +This keeps high-signal UI labels such as dropdown options searchable without requiring MemoryBench to store screenshots in this text-first pipeline. diff --git a/src/cli/commands/ingest.ts b/src/cli/commands/ingest.ts index 3d4d1a9..a8174ff 100644 --- a/src/cli/commands/ingest.ts +++ b/src/cli/commands/ingest.ts @@ -10,6 +10,8 @@ interface IngestArgs { benchmark?: string runId: string force?: boolean + limit?: number + questionId?: string } function generateRunId(): string { @@ -30,6 +32,10 @@ export function parseIngestArgs(args: string[]): IngestArgs | null { parsed.benchmark = args[++i] } else if (arg === "-r" || arg === "--run-id") { parsed.runId = args[++i] + } else if (arg === "-l" || arg === "--limit") { + parsed.limit = parseInt(args[++i], 10) + } else if (arg === "-q" || arg === "--question-id") { + parsed.questionId = args[++i] } else if (arg === "--force") { parsed.force = true } @@ -53,15 +59,22 @@ export async function ingestCommand(args: string[]): Promise { if (!parsed) { console.log("Usage:") console.log( - " New run: bun run src/index.ts ingest -p -b [-r ] [--force]" + " New run: bun run src/index.ts ingest -p -b [-r ] [-l | -q ] [--force]" ) console.log(" Continue run: bun run src/index.ts ingest -r ") console.log("") console.log("Options:") - console.log(` -p, --provider Provider: ${getAvailableProviders().join(", ")}`) - console.log(` -b, --benchmark Benchmark: ${getAvailableBenchmarks().join(", ")}`) - console.log(" -r, --run-id Run identifier") - console.log(" --force Clear existing checkpoint and start fresh") + console.log(` -p, --provider Provider: ${getAvailableProviders().join(", ")}`) + console.log(` -b, --benchmark Benchmark: ${getAvailableBenchmarks().join(", ")}`) + console.log(" -r, --run-id Run identifier") + console.log(" -l, --limit Limit number of questions for a new run") + console.log(" -q, --question-id Ingest a specific question for a new run") + console.log(" --force Clear existing checkpoint and start fresh") + return + } + + if (parsed.limit && parsed.questionId) { + logger.error("Use either --limit or --question-id, not both") return } @@ -109,6 +122,8 @@ export async function ingestCommand(args: string[]): Promise { provider: parsed.provider as ProviderName, benchmark: parsed.benchmark as BenchmarkName, runId: parsed.runId, + limit: parsed.limit, + questionIds: parsed.questionId ? [parsed.questionId] : undefined, force: parsed.force, }) } From 94bf9a39f1311a12e568e65bb36ea7acb8093901 Mon Sep 17 00:00:00 2001 From: Vedant Mahajan Date: Fri, 17 Jul 2026 13:41:28 +0530 Subject: [PATCH 3/3] Add controlled LongMemEval-V2 ingestion --- src/benchmarks/longmemeval-v2/README.md | 74 +++++++++ src/benchmarks/longmemeval-v2/index.ts | 210 +++++++++++++++++++----- src/cli/commands/ingest.ts | 117 +++++++++++++ src/orchestrator/checkpoint.ts | 23 ++- src/orchestrator/index.ts | 96 ++++++++++- src/orchestrator/phases/ingest.ts | 39 ++++- src/orchestrator/phases/search.ts | 2 +- src/providers/supermemory/index.ts | 45 ++++- src/types/benchmark.ts | 3 + src/types/checkpoint.ts | 3 + src/types/provider.ts | 2 + 11 files changed, 556 insertions(+), 58 deletions(-) diff --git a/src/benchmarks/longmemeval-v2/README.md b/src/benchmarks/longmemeval-v2/README.md index cc7500b..757af92 100644 --- a/src/benchmarks/longmemeval-v2/README.md +++ b/src/benchmarks/longmemeval-v2/README.md @@ -52,6 +52,80 @@ That creates one MemoryBench checkpoint and a Supermemory container tag for the - ``` +## Limit trajectories while preserving haystack order + +Use the full official haystack while ingesting only the first ordered trajectories for each selected question: + +```bash +LONGMEMEVAL_V2_TIER=small \ +bun run src/index.ts ingest \ + -p supermemory \ + -b longmemeval-v2 \ + -r lme-v2-first-trajectory \ + -q 01307e07 \ + --trajectory-limit 1 +``` + +`--trajectory-limit` limits trajectories, while `--limit` limits questions. The trajectory limit is saved in the checkpoint so resumed ingestion keeps the same selection. + +## Ingest one trajectory document at a time + +Use a unique run ID for each document and reuse one explicit container tag. This keeps every document independently checkpointed while accumulating them in one provider container. + +Start with the first trajectory's overview only: + +```bash +LONGMEMEVAL_V2_TIER=small \ +bun run src/index.ts ingest \ + -p supermemory \ + -b longmemeval-v2 \ + -r lme-v2-f224a4eb-overview-v1 \ + -q 01307e07 \ + --trajectory-limit 1 \ + --document overview \ + --container-tag lme-v2-f224a4eb-sequential-v1 +``` + +Later, ingest a single state into the same container by changing the run ID and document selector while retaining the container tag: + +```bash +LONGMEMEVAL_V2_TIER=small \ +bun run src/index.ts ingest \ + -p supermemory \ + -b longmemeval-v2 \ + -r lme-v2-f224a4eb-state-0-v1 \ + -q 01307e07 \ + --trajectory-limit 1 \ + --document state:0 \ + --container-tag lme-v2-f224a4eb-sequential-v1 +``` + +`--document overview` selects only the overview session. `--document state:` selects only that state. With `--trajectory-limit 1` and one question, each command ingests exactly one document. + +## Clean trajectory payloads + +Use `--trajectory-format clean` to create a causally separated payload: + +- overview (`STATE_-1`): goal and starting context only +- states (`STATE_0` onward): URL, action, thought, and screenshot path, without the repeated goal, final outcome, compact accessibility extraction, or raw accessibility tree +- result (`RESULT`): the final trajectory outcome exactly once + +Every document carries numeric `stateIndex` metadata using its real position: overview `-1`, states `0` onward, and result after the last state. To retrieve information available before state `x`, apply a numeric metadata filter of `stateIndex <= x - 1`. + +Use `--trajectory-format clean-tree` for the same goal/outcome separation while including the accessibility information exactly like the raw format: a compact UI extraction plus the normally truncated accessibility-tree excerpt. This mode uses filtered writes so state `x` only receives memories sourced from state `x - 1`, waits for indexing, and then waits another 60 seconds before submitting the next document. + +```bash +LONGMEMEVAL_V2_TIER=small \ +bun run src/index.ts ingest \ + -p supermemory \ + -b longmemeval-v2 \ + -r lme-v2-f224a4eb-clean-v1 \ + -q 01307e07 \ + --trajectory-limit 1 \ + --trajectory-format clean \ + --container-tag test_unraw_16_07_26 +``` + ## Search after ingest PowerShell: diff --git a/src/benchmarks/longmemeval-v2/index.ts b/src/benchmarks/longmemeval-v2/index.ts index 6687fe5..8836c29 100644 --- a/src/benchmarks/longmemeval-v2/index.ts +++ b/src/benchmarks/longmemeval-v2/index.ts @@ -14,8 +14,37 @@ const DEFAULT_DATA_PATH = "./data/benchmarks/longmemeval-v2" const DEFAULT_TIER = "small" const TREE_EXCERPT_CHAR_LIMIT = 12000 const COMPACT_UI_CHAR_LIMIT = 12000 +const POST_INDEXING_DELAY_MS = 60_000 type HaystackMap = Record +type TrajectoryFormat = "raw" | "clean" | "clean-tree" +type TrajectoryDocumentSelection = + | { type: "all" } + | { type: "overview" } + | { type: "state"; stateIndex: number } + | { type: "result" } + +function parseTrajectoryDocument(value?: string): TrajectoryDocumentSelection { + if (value === undefined) return { type: "all" } + if (value === "overview") return { type: "overview" } + if (value === "result") return { type: "result" } + + const match = /^state:(0|[1-9]\d*)$/.exec(value) + if (match) return { type: "state", stateIndex: Number(match[1]) } + + throw new Error("LongMemEval-V2 trajectoryDocument must be overview, state:, or result") +} + +function parseTrajectoryFormat(value?: string): TrajectoryFormat { + if (value === undefined || value === "raw") return "raw" + if (value === "clean") return "clean" + if (value === "clean-tree") return "clean-tree" + throw new Error("LongMemEval-V2 trajectoryFormat must be raw, clean, or clean-tree") +} + +function isCleanFormat(format: TrajectoryFormat): boolean { + return format === "clean" || format === "clean-tree" +} function readJsonl(path: string): T[] { return readFileSync(path, "utf8") @@ -102,20 +131,25 @@ function compactAccessibilityTree(tree?: string | null): string { function stateToSession( trajectory: LongMemEvalV2Trajectory, - state: LongMemEvalV2State + state: LongMemEvalV2State, + format: TrajectoryFormat ): UnifiedSession { - const compactTree = compactAccessibilityTree(state.accessibility_tree) - const treeExcerpt = state.accessibility_tree - ? truncate(state.accessibility_tree, TREE_EXCERPT_CHAR_LIMIT) + const usesRawTreeRepresentation = format === "raw" || format === "clean-tree" + const compactTree = usesRawTreeRepresentation + ? compactAccessibilityTree(state.accessibility_tree) : "" + const accessibilityTree = + usesRawTreeRepresentation && state.accessibility_tree + ? truncate(state.accessibility_tree, TREE_EXCERPT_CHAR_LIMIT) + : "" const content = [ "LongMemEval-V2 agent trajectory state", `Trajectory ID: ${trajectory.id}`, `Domain: ${trajectory.domain}`, `Environment: ${trajectory.environment}`, - `Outcome: ${trajectory.outcome || "unknown"}`, - `Goal: ${trajectory.goal}`, + format === "raw" ? `Outcome: ${trajectory.outcome || "unknown"}` : "", + format === "raw" ? `Goal: ${trajectory.goal}` : "", `State index: ${state.state_index}`, state.step !== undefined ? `Step: ${state.step}` : "", state.url ? `URL: ${state.url}` : "", @@ -123,7 +157,7 @@ function stateToSession( state.thought ? `Agent thought: ${state.thought}` : "", state.screenshot ? `Screenshot path: ${state.screenshot}` : "", compactTree ? `\nCompact UI extraction:\n${compactTree}` : "", - treeExcerpt ? `\nAccessibility tree excerpt:\n${treeExcerpt}` : "", + accessibilityTree ? `\nAccessibility tree excerpt:\n${accessibilityTree}` : "", ] .filter(Boolean) .join("\n") @@ -133,18 +167,25 @@ function stateToSession( messages: [{ role: "assistant", content }], metadata: { benchmark: "longmemeval-v2", + documentTag: `STATE_${state.state_index}`, trajectoryId: trajectory.id, stateIndex: state.state_index, + documentType: "state", + ...(isCleanFormat(format) ? { filterByMetadata: { stateIndex: state.state_index - 1 } } : {}), + ...(format === "clean-tree" ? { postIndexingDelayMs: POST_INDEXING_DELAY_MS } : {}), domain: trajectory.domain, environment: trajectory.environment, - outcome: trajectory.outcome, + ...(format === "raw" ? { outcome: trajectory.outcome } : {}), url: state.url, screenshot: state.screenshot, }, } } -function trajectoryOverviewToSession(trajectory: LongMemEvalV2Trajectory): UnifiedSession { +function trajectoryOverviewToSession( + trajectory: LongMemEvalV2Trajectory, + format: TrajectoryFormat +): UnifiedSession { const actionTrace = trajectory.states .map((state) => { const action = state.action || "null" @@ -153,39 +194,80 @@ function trajectoryOverviewToSession(trajectory: LongMemEvalV2Trajectory): Unifi }) .join("\n") - const messages: UnifiedMessage[] = [ - { - role: "user", - content: [ - "LongMemEval-V2 trajectory overview", - `Trajectory ID: ${trajectory.id}`, - `Domain: ${trajectory.domain}`, - `Environment: ${trajectory.environment}`, - `Outcome: ${trajectory.outcome || "unknown"}`, - `Start URL: ${trajectory.start_url || "unknown"}`, - `Goal: ${trajectory.goal}`, - ].join("\n"), - }, - { + const overviewContent = [ + "LongMemEval-V2 trajectory overview", + `Trajectory ID: ${trajectory.id}`, + `Domain: ${trajectory.domain}`, + `Environment: ${trajectory.environment}`, + format === "raw" ? `Outcome: ${trajectory.outcome || "unknown"}` : "", + `Start URL: ${trajectory.start_url || "unknown"}`, + `Goal: ${trajectory.goal}`, + ] + .filter(Boolean) + .join("\n") + + const messages: UnifiedMessage[] = [{ role: "user", content: overviewContent }] + if (format === "raw") { + messages.push({ role: "assistant", content: `Action/thought trace:\n${truncate(actionTrace, TREE_EXCERPT_CHAR_LIMIT)}`, - }, - ] + }) + } return { sessionId: `lme-v2-${trajectory.id}-overview`, messages, metadata: { benchmark: "longmemeval-v2", + documentTag: "STATE_-1", trajectoryId: trajectory.id, + stateIndex: -1, + documentType: "overview", domain: trajectory.domain, environment: trajectory.environment, - outcome: trajectory.outcome, + ...(format === "raw" ? { outcome: trajectory.outcome } : {}), + ...(format === "clean-tree" ? { postIndexingDelayMs: POST_INDEXING_DELAY_MS } : {}), sessionType: "trajectory-overview", }, } } +function trajectoryResultToSession( + trajectory: LongMemEvalV2Trajectory, + format: TrajectoryFormat +): UnifiedSession { + const resultStateIndex = + trajectory.states.reduce((max, state) => Math.max(max, state.state_index), -1) + 1 + + return { + sessionId: `lme-v2-${trajectory.id}-result`, + messages: [ + { + role: "assistant", + content: [ + "LongMemEval-V2 trajectory result", + `Trajectory ID: ${trajectory.id}`, + `Domain: ${trajectory.domain}`, + `Environment: ${trajectory.environment}`, + `Final outcome: ${trajectory.outcome || "unknown"}`, + ].join("\n"), + }, + ], + metadata: { + benchmark: "longmemeval-v2", + documentTag: "RESULT", + trajectoryId: trajectory.id, + stateIndex: resultStateIndex, + documentType: "result", + filterByMetadata: { stateIndex: resultStateIndex - 1 }, + domain: trajectory.domain, + environment: trajectory.environment, + ...(format === "clean-tree" ? { postIndexingDelayMs: POST_INDEXING_DELAY_MS } : {}), + sessionType: "trajectory-result", + }, + } +} + export class LongMemEvalV2Benchmark implements Benchmark { name = "longmemeval-v2" private questions: UnifiedQuestion[] = [] @@ -206,15 +288,36 @@ export class LongMemEvalV2Benchmark implements Benchmark { const rawQuestions = readJsonl(questionsPath) const haystack = JSON.parse(readFileSync(haystackPath, "utf8")) as HaystackMap - const haystackQuestionIds = new Set(Object.keys(haystack)) - const selectedTrajectoryIds = new Set(Object.values(haystack).flat()) + const trajectoryLimit = config?.trajectoryLimit + const trajectoryDocument = parseTrajectoryDocument(config?.trajectoryDocument) + const trajectoryFormat = parseTrajectoryFormat(config?.trajectoryFormat) + if ( + trajectoryLimit !== undefined && + (!Number.isInteger(trajectoryLimit) || trajectoryLimit < 1) + ) { + throw new Error("LongMemEval-V2 trajectoryLimit must be a positive integer") + } - const trajectorySessions = this.loadTrajectorySessions(trajectoriesPath, selectedTrajectoryIds) + const selectedHaystack = Object.fromEntries( + Object.entries(haystack).map(([questionId, trajectoryIds]) => [ + questionId, + trajectoryLimit === undefined ? trajectoryIds : trajectoryIds.slice(0, trajectoryLimit), + ]) + ) as HaystackMap + const haystackQuestionIds = new Set(Object.keys(selectedHaystack)) + const selectedTrajectoryIds = new Set(Object.values(selectedHaystack).flat()) + + const trajectorySessions = this.loadTrajectorySessions( + trajectoriesPath, + selectedTrajectoryIds, + trajectoryDocument, + trajectoryFormat + ) for (const item of rawQuestions) { if (!haystackQuestionIds.has(item.id)) continue - const sessions = haystack[item.id].flatMap((trajectoryId) => { + const sessions = selectedHaystack[item.id].flatMap((trajectoryId) => { const trajectorySessionList = trajectorySessions.get(trajectoryId) if (!trajectorySessionList) { logger.warn(`Missing LongMemEval-V2 trajectory ${trajectoryId} for question ${item.id}`) @@ -235,7 +338,11 @@ export class LongMemEvalV2Benchmark implements Benchmark { image: item.image, evalFunction: item.eval_function, haystackTier: tier, - trajectoryCount: haystack[item.id].length, + trajectoryCount: selectedHaystack[item.id].length, + fullTrajectoryCount: haystack[item.id].length, + trajectoryLimit, + trajectoryDocument: config?.trajectoryDocument, + trajectoryFormat, }, }) @@ -247,22 +354,51 @@ export class LongMemEvalV2Benchmark implements Benchmark { } } - logger.info(`Loaded ${this.questions.length} LongMemEval-V2 ${tier} questions from ${dataPath}`) + logger.info( + `Loaded ${this.questions.length} LongMemEval-V2 ${tier} questions from ${dataPath}` + + (trajectoryLimit === undefined + ? "" + : ` (first ${trajectoryLimit} ordered trajectories per question)`) + + (config?.trajectoryDocument === undefined + ? "" + : ` (document ${config.trajectoryDocument})`) + + ` (format ${trajectoryFormat})` + ) } private loadTrajectorySessions( trajectoriesPath: string, - selectedTrajectoryIds: Set + selectedTrajectoryIds: Set, + documentSelection: TrajectoryDocumentSelection, + format: TrajectoryFormat ): Map { const sessions = new Map() for (const trajectory of readJsonl(trajectoriesPath)) { if (!selectedTrajectoryIds.has(trajectory.id)) continue - const trajectorySessions = [ - trajectoryOverviewToSession(trajectory), - ...trajectory.states.map((state) => stateToSession(trajectory, state)), - ] + const trajectorySessions = + documentSelection.type === "overview" + ? [trajectoryOverviewToSession(trajectory, format)] + : documentSelection.type === "state" + ? trajectory.states + .filter((state) => state.state_index === documentSelection.stateIndex) + .map((state) => stateToSession(trajectory, state, format)) + : documentSelection.type === "result" + ? isCleanFormat(format) + ? [trajectoryResultToSession(trajectory, format)] + : [] + : [ + trajectoryOverviewToSession(trajectory, format), + ...trajectory.states.map((state) => stateToSession(trajectory, state, format)), + ...(isCleanFormat(format) ? [trajectoryResultToSession(trajectory, format)] : []), + ] + + if (trajectorySessions.length === 0) { + logger.warn( + `Trajectory ${trajectory.id} has no selected ${documentSelection.type === "state" ? `state ${documentSelection.stateIndex}` : "document"}` + ) + } sessions.set(trajectory.id, trajectorySessions) } diff --git a/src/cli/commands/ingest.ts b/src/cli/commands/ingest.ts index a8174ff..028ce79 100644 --- a/src/cli/commands/ingest.ts +++ b/src/cli/commands/ingest.ts @@ -11,6 +11,10 @@ interface IngestArgs { runId: string force?: boolean limit?: number + trajectoryLimit?: number + trajectoryDocument?: string + trajectoryFormat?: string + containerTag?: string questionId?: string } @@ -34,6 +38,14 @@ export function parseIngestArgs(args: string[]): IngestArgs | null { parsed.runId = args[++i] } else if (arg === "-l" || arg === "--limit") { parsed.limit = parseInt(args[++i], 10) + } else if (arg === "--trajectory-limit") { + parsed.trajectoryLimit = parseInt(args[++i], 10) + } else if (arg === "--document") { + parsed.trajectoryDocument = args[++i] + } else if (arg === "--trajectory-format") { + parsed.trajectoryFormat = args[++i] + } else if (arg === "--container-tag") { + parsed.containerTag = args[++i] } else if (arg === "-q" || arg === "--question-id") { parsed.questionId = args[++i] } else if (arg === "--force") { @@ -68,6 +80,12 @@ export async function ingestCommand(args: string[]): Promise { console.log(` -b, --benchmark Benchmark: ${getAvailableBenchmarks().join(", ")}`) console.log(" -r, --run-id Run identifier") console.log(" -l, --limit Limit number of questions for a new run") + console.log( + " --trajectory-limit Limit ordered trajectories per selected LongMemEval-V2 question" + ) + console.log(" --document LongMemEval-V2 document: overview, state:, or result") + console.log(" --trajectory-format LongMemEval-V2 payload format: raw, clean, or clean-tree") + console.log(" --container-tag Explicit provider container tag (requires one question)") console.log(" -q, --question-id Ingest a specific question for a new run") console.log(" --force Clear existing checkpoint and start fresh") return @@ -78,6 +96,35 @@ export async function ingestCommand(args: string[]): Promise { return } + if ( + parsed.trajectoryLimit !== undefined && + (!Number.isInteger(parsed.trajectoryLimit) || parsed.trajectoryLimit < 1) + ) { + logger.error("--trajectory-limit must be a positive integer") + return + } + + if ( + parsed.trajectoryDocument !== undefined && + !/^(overview|state:(0|[1-9]\d*)|result)$/.test(parsed.trajectoryDocument) + ) { + logger.error("--document must be overview, state:, or result") + return + } + + if ( + parsed.trajectoryFormat !== undefined && + !/^(raw|clean|clean-tree)$/.test(parsed.trajectoryFormat) + ) { + logger.error("--trajectory-format must be raw, clean, or clean-tree") + return + } + + if (parsed.containerTag !== undefined && !parsed.containerTag.trim()) { + logger.error("--container-tag cannot be empty") + return + } + const checkpointManager = new CheckpointManager() if (checkpointManager.exists(parsed.runId)) { @@ -95,9 +142,51 @@ export async function ingestCommand(args: string[]): Promise { ) return } + if ( + parsed.trajectoryLimit !== undefined && + checkpoint.trajectoryLimit !== undefined && + parsed.trajectoryLimit !== checkpoint.trajectoryLimit + ) { + logger.error( + `Run ${parsed.runId} uses trajectory limit ${checkpoint.trajectoryLimit}, not ${parsed.trajectoryLimit}` + ) + return + } + if ( + parsed.trajectoryDocument !== undefined && + checkpoint.trajectoryDocument !== undefined && + parsed.trajectoryDocument !== checkpoint.trajectoryDocument + ) { + logger.error( + `Run ${parsed.runId} uses trajectory document ${checkpoint.trajectoryDocument}, not ${parsed.trajectoryDocument}` + ) + return + } + if ( + parsed.trajectoryFormat !== undefined && + checkpoint.trajectoryFormat !== undefined && + parsed.trajectoryFormat !== checkpoint.trajectoryFormat + ) { + logger.error( + `Run ${parsed.runId} uses trajectory format ${checkpoint.trajectoryFormat}, not ${parsed.trajectoryFormat}` + ) + return + } + if (parsed.containerTag !== undefined) { + const existingTags = new Set( + Object.values(checkpoint.questions).map((question) => question.containerTag) + ) + if (existingTags.size > 0 && !existingTags.has(parsed.containerTag)) { + logger.error(`Run ${parsed.runId} does not use container tag ${parsed.containerTag}`) + return + } + } parsed.provider = checkpoint.provider parsed.benchmark = checkpoint.benchmark + parsed.trajectoryLimit = checkpoint.trajectoryLimit ?? parsed.trajectoryLimit + parsed.trajectoryDocument = checkpoint.trajectoryDocument ?? parsed.trajectoryDocument + parsed.trajectoryFormat = checkpoint.trajectoryFormat ?? parsed.trajectoryFormat logger.info( `Continuing ingest for ${parsed.runId} (${checkpoint.provider}/${checkpoint.benchmark})` ) @@ -118,11 +207,39 @@ export async function ingestCommand(args: string[]): Promise { } } + if (parsed.trajectoryLimit !== undefined && parsed.benchmark !== "longmemeval-v2") { + logger.error("--trajectory-limit is currently supported only for longmemeval-v2") + return + } + + if (parsed.trajectoryDocument !== undefined && parsed.benchmark !== "longmemeval-v2") { + logger.error("--document is currently supported only for longmemeval-v2") + return + } + + if (parsed.trajectoryFormat !== undefined && parsed.benchmark !== "longmemeval-v2") { + logger.error("--trajectory-format is currently supported only for longmemeval-v2") + return + } + + if ( + parsed.containerTag !== undefined && + !parsed.questionId && + !checkpointManager.exists(parsed.runId) + ) { + logger.error("--container-tag requires -q/--question-id for a new run") + return + } + await orchestrator.ingest({ provider: parsed.provider as ProviderName, benchmark: parsed.benchmark as BenchmarkName, runId: parsed.runId, limit: parsed.limit, + trajectoryLimit: parsed.trajectoryLimit, + trajectoryDocument: parsed.trajectoryDocument, + trajectoryFormat: parsed.trajectoryFormat, + containerTag: parsed.containerTag, questionIds: parsed.questionId ? [parsed.questionId] : undefined, force: parsed.force, }) diff --git a/src/orchestrator/checkpoint.ts b/src/orchestrator/checkpoint.ts index aa00835..42ee3da 100644 --- a/src/orchestrator/checkpoint.ts +++ b/src/orchestrator/checkpoint.ts @@ -105,7 +105,7 @@ export class CheckpointManager { // If we get here, all retries failed or it was a non-retriable error try { unlinkSync(tempPath) - } catch { } + } catch {} throw lastError } @@ -125,6 +125,9 @@ export class CheckpointManager { answeringModel: string, options?: { limit?: number + trajectoryLimit?: number + trajectoryDocument?: string + trajectoryFormat?: string sampling?: SamplingConfig targetQuestionIds?: string[] dataSourceRunId?: string @@ -143,6 +146,9 @@ export class CheckpointManager { createdAt: new Date().toISOString(), updatedAt: new Date().toISOString(), limit: options?.limit, + trajectoryLimit: options?.trajectoryLimit, + trajectoryDocument: options?.trajectoryDocument, + trajectoryFormat: options?.trajectoryFormat, sampling: options?.sampling, targetQuestionIds: options?.targetQuestionIds, concurrency: options?.concurrency, @@ -284,12 +290,12 @@ export class CheckpointManager { evaluated: questions.filter((q) => q.phases.evaluate.status === "completed").length, ...(episodesTotal > 0 ? { - indexingEpisodes: { - total: episodesTotal, - completed: episodesCompleted, - failed: episodesFailed, - }, - } + indexingEpisodes: { + total: episodesTotal, + completed: episodesCompleted, + failed: episodesFailed, + }, + } : {}), } } @@ -359,6 +365,9 @@ export class CheckpointManager { createdAt: new Date().toISOString(), updatedAt: new Date().toISOString(), limit: source.limit, + trajectoryLimit: source.trajectoryLimit, + trajectoryDocument: source.trajectoryDocument, + trajectoryFormat: source.trajectoryFormat, sampling: source.sampling, targetQuestionIds: source.targetQuestionIds, concurrency: source.concurrency, diff --git a/src/orchestrator/index.ts b/src/orchestrator/index.ts index 64578bb..5e62a8c 100644 --- a/src/orchestrator/index.ts +++ b/src/orchestrator/index.ts @@ -24,6 +24,10 @@ export interface OrchestratorOptions { runId: string answeringModel?: string limit?: number + trajectoryLimit?: number + trajectoryDocument?: string + trajectoryFormat?: string + containerTag?: string sampling?: SamplingConfig concurrency?: ConcurrencyConfig force?: boolean @@ -80,6 +84,10 @@ export class Orchestrator { runId, answeringModel = "gpt-4o", limit, + trajectoryLimit, + trajectoryDocument, + trajectoryFormat, + containerTag, sampling, concurrency, force = false, @@ -112,6 +120,18 @@ export class Orchestrator { } else { logger.info(`No sampling or limit provided`) } + if (trajectoryLimit !== undefined) { + logger.info(`Trajectory limit: ${trajectoryLimit} per selected question`) + } + if (trajectoryDocument !== undefined) { + logger.info(`Trajectory document: ${trajectoryDocument}`) + } + if (trajectoryFormat !== undefined) { + logger.info(`Trajectory format: ${trajectoryFormat}`) + } + if (containerTag !== undefined) { + logger.info(`Container tag: ${containerTag}`) + } if (force && this.checkpointManager.exists(runId)) { this.checkpointManager.delete(runId) @@ -131,17 +151,81 @@ export class Orchestrator { benchmarkName, judgeModel, answeringModel, - { limit, sampling, concurrency, status: "initializing" } + { + limit, + trajectoryLimit, + trajectoryDocument, + trajectoryFormat, + sampling, + concurrency, + status: "initializing", + } ) logger.info("Created checkpoint (initializing)") } + let effectiveTrajectoryLimit = trajectoryLimit + let effectiveTrajectoryDocument = trajectoryDocument + let effectiveTrajectoryFormat = trajectoryFormat + if (!isNewRun) { + const existingCheckpoint = this.checkpointManager.load(runId)! + if ( + trajectoryLimit !== undefined && + existingCheckpoint.trajectoryLimit !== undefined && + trajectoryLimit !== existingCheckpoint.trajectoryLimit + ) { + throw new Error( + `Run ${runId} uses trajectory limit ${existingCheckpoint.trajectoryLimit}, not ${trajectoryLimit}` + ) + } + effectiveTrajectoryLimit = existingCheckpoint.trajectoryLimit ?? trajectoryLimit + if ( + trajectoryDocument !== undefined && + existingCheckpoint.trajectoryDocument !== undefined && + trajectoryDocument !== existingCheckpoint.trajectoryDocument + ) { + throw new Error( + `Run ${runId} uses trajectory document ${existingCheckpoint.trajectoryDocument}, not ${trajectoryDocument}` + ) + } + effectiveTrajectoryDocument = existingCheckpoint.trajectoryDocument ?? trajectoryDocument + if ( + trajectoryFormat !== undefined && + existingCheckpoint.trajectoryFormat !== undefined && + trajectoryFormat !== existingCheckpoint.trajectoryFormat + ) { + throw new Error( + `Run ${runId} uses trajectory format ${existingCheckpoint.trajectoryFormat}, not ${trajectoryFormat}` + ) + } + effectiveTrajectoryFormat = existingCheckpoint.trajectoryFormat ?? trajectoryFormat + } + const benchmark = createBenchmark(benchmarkName) - await benchmark.load() + await benchmark.load({ + trajectoryLimit: effectiveTrajectoryLimit, + trajectoryDocument: effectiveTrajectoryDocument, + trajectoryFormat: effectiveTrajectoryFormat, + }) const allQuestions = benchmark.getQuestions() if (this.checkpointManager.exists(runId) && !isNewRun) { checkpoint = this.checkpointManager.load(runId)! + if (checkpoint.trajectoryLimit === undefined && effectiveTrajectoryLimit !== undefined) { + checkpoint.trajectoryLimit = effectiveTrajectoryLimit + this.checkpointManager.save(checkpoint) + } + if ( + checkpoint.trajectoryDocument === undefined && + effectiveTrajectoryDocument !== undefined + ) { + checkpoint.trajectoryDocument = effectiveTrajectoryDocument + this.checkpointManager.save(checkpoint) + } + if (checkpoint.trajectoryFormat === undefined && effectiveTrajectoryFormat !== undefined) { + checkpoint.trajectoryFormat = effectiveTrajectoryFormat + this.checkpointManager.save(checkpoint) + } effectiveLimit = checkpoint.limit targetQuestionIds = checkpoint.targetQuestionIds @@ -236,9 +320,13 @@ export class Orchestrator { ? allQuestions.filter((q) => targetQuestionIds!.includes(q.questionId)) : allQuestions + if (containerTag && questionsToInit.length !== 1) { + throw new Error("A custom container tag requires exactly one selected question") + } + for (const q of questionsToInit) { - const containerTag = `${q.questionId}-${checkpoint.dataSourceRunId}` - this.checkpointManager.initQuestion(checkpoint, q.questionId, containerTag, { + const questionContainerTag = containerTag || `${q.questionId}-${checkpoint.dataSourceRunId}` + this.checkpointManager.initQuestion(checkpoint, q.questionId, questionContainerTag, { question: q.question, groundTruth: q.groundTruth, questionType: q.questionType, diff --git a/src/orchestrator/phases/ingest.ts b/src/orchestrator/phases/ingest.ts index e38815d..9e75c19 100644 --- a/src/orchestrator/phases/ingest.ts +++ b/src/orchestrator/phases/ingest.ts @@ -41,8 +41,17 @@ export async function runIngestPhase( runId: checkpoint.runId, phaseName: "ingest", executeTask: async ({ item: question, index, total }) => { - const containerTag = `${question.questionId}-${checkpoint.dataSourceRunId}` + const containerTag = checkpoint.questions[question.questionId].containerTag const sessions = benchmark.getHaystackSessions(question.questionId) + const requiresSequentialIndexing = sessions.some( + (session) => session.metadata?.filterByMetadata !== undefined + ) + + if (requiresSequentialIndexing) { + logger.info( + `Using sequential filtered-write ingestion for ${question.questionId}; each document will finish indexing before the next is submitted` + ) + } const sessionsMetadata = sessions.map((s) => ({ sessionId: s.sessionId, @@ -74,6 +83,34 @@ export async function runIngestPhase( combinedResult.taskIds!.push(...result.taskIds) } + if (requiresSequentialIndexing) { + let failedIds: string[] = [] + await provider.awaitIndexing(result, containerTag, (progress) => { + failedIds = progress.failedIds + }) + + if (failedIds.length > 0) { + throw new Error( + `Indexing failed for session ${session.sessionId}: ${failedIds.join(", ")}` + ) + } + + logger.info(`Indexed ${session.sessionId}`) + + const postIndexingDelayMs = session.metadata?.postIndexingDelayMs + if ( + typeof postIndexingDelayMs === "number" && + Number.isFinite(postIndexingDelayMs) && + postIndexingDelayMs > 0 + ) { + logger.info( + `Waiting ${Math.round(postIndexingDelayMs / 1000)} seconds after ${session.sessionId} before continuing` + ) + await new Promise((resolve) => setTimeout(resolve, postIndexingDelayMs)) + logger.info(`Post-indexing wait complete for ${session.sessionId}`) + } + } + completedSessions.push(session.sessionId) checkpointManager.updatePhase(checkpoint, question.questionId, "ingest", { completedSessions, diff --git a/src/orchestrator/phases/search.ts b/src/orchestrator/phases/search.ts index 65e4ac7..c774ab8 100644 --- a/src/orchestrator/phases/search.ts +++ b/src/orchestrator/phases/search.ts @@ -46,7 +46,7 @@ export async function runSearchPhase( checkpoint.runId, "search", async ({ item: question, index, total }) => { - const containerTag = `${question.questionId}-${checkpoint.dataSourceRunId}` + const containerTag = checkpoint.questions[question.questionId].containerTag const startTime = Date.now() checkpointManager.updatePhase(checkpoint, question.questionId, "search", { diff --git a/src/providers/supermemory/index.ts b/src/providers/supermemory/index.ts index 027bc32..1dbea0c 100644 --- a/src/providers/supermemory/index.ts +++ b/src/providers/supermemory/index.ts @@ -40,18 +40,32 @@ export class SupermemoryProvider implements Provider { const formattedDate = session.metadata?.formattedDate as string const isoDate = session.metadata?.date as string - const content = formattedDate + const documentTag = session.metadata?.documentTag as string | undefined + const stateIndex = session.metadata?.stateIndex as number | undefined + const documentType = session.metadata?.documentType as string | undefined + const trajectoryId = session.metadata?.trajectoryId as string | undefined + const filterByMetadata = session.metadata?.filterByMetadata as + | Record + | undefined + const sessionContent = formattedDate ? `Here is the date the following session took place: ${formattedDate}\n\nHere is the session as a stringified JSON:\n${sessionStr}` : `Here is the session as a stringified JSON:\n${sessionStr}` + const content = documentTag ? `${documentTag}\n\n${sessionContent}` : sessionContent - const response = await this.client.add({ + const addParams = { content, containerTag: options.containerTag, metadata: { sessionId: session.sessionId, ...(isoDate ? { date: isoDate } : {}), + ...(documentTag ? { documentTag } : {}), + ...(stateIndex !== undefined ? { stateIndex } : {}), + ...(documentType ? { documentType } : {}), + ...(trajectoryId ? { trajectoryId } : {}), }, - }) + ...(filterByMetadata ? { filterByMetadata } : {}), + } + const response = await this.client.add(addParams) documentIds.push(response.id) logger.debug(`Ingested session ${session.sessionId}`) } @@ -120,16 +134,31 @@ export class SupermemoryProvider implements Provider { async search(query: string, options: SearchOptions): Promise { if (!this.client) throw new Error("Provider not initialized") + const filters = + options.maxStateIndex === undefined + ? undefined + : { + AND: [ + { + key: "stateIndex", + value: String(options.maxStateIndex), + filterType: "numeric" as const, + numericOperator: "<=" as const, + }, + ], + } + const response = await this.client.search.memories({ q: query, containerTag: options.containerTag, + ...(filters ? { filters } : {}), limit: 30, threshold: options.threshold || 0.3, - searchMode: "hybrid", - include: { - summaries: true, - chunks: true - } + searchMode: "hybrid", + include: { + summaries: true, + chunks: true, + }, }) return response.results || [] diff --git a/src/types/benchmark.ts b/src/types/benchmark.ts index dccf507..c5cf119 100644 --- a/src/types/benchmark.ts +++ b/src/types/benchmark.ts @@ -2,6 +2,9 @@ import type { UnifiedQuestion, UnifiedSession, QuestionTypeRegistry } from "./un export interface BenchmarkConfig { dataPath?: string + trajectoryLimit?: number + trajectoryDocument?: string + trajectoryFormat?: string } export interface QuestionFilter { diff --git a/src/types/checkpoint.ts b/src/types/checkpoint.ts index f8f1180..9a2bda1 100644 --- a/src/types/checkpoint.ts +++ b/src/types/checkpoint.ts @@ -122,6 +122,9 @@ export interface RunCheckpoint { createdAt: string updatedAt: string limit?: number + trajectoryLimit?: number + trajectoryDocument?: string + trajectoryFormat?: string sampling?: SamplingConfig targetQuestionIds?: string[] concurrency?: ConcurrencyConfig diff --git a/src/types/provider.ts b/src/types/provider.ts index cdc0228..71ac089 100644 --- a/src/types/provider.ts +++ b/src/types/provider.ts @@ -17,6 +17,8 @@ export interface SearchOptions { containerTag: string limit?: number threshold?: number + /** Include only memories/documents whose numeric stateIndex is at or below this cutoff. */ + maxStateIndex?: number } export interface IngestResult {