diff --git a/scripts/csm-audit.ts b/scripts/csm-audit.ts index 7e72858..4e90fbe 100644 --- a/scripts/csm-audit.ts +++ b/scripts/csm-audit.ts @@ -3,6 +3,7 @@ import { MemoryManager } from '../src/memory-manager.js'; import { EmbeddingGenerator } from '../src/embeddings.js'; import { Redactor } from '../src/redactor.js'; import { validateAndReturnConfig } from '../src/config.js'; +import { auditMemoryProvenance, REQUIRED_MEMORY_PROVENANCE_FIELDS } from '../src/memory-provenance-audit.js'; const config = validateAndReturnConfig(); const database = new Database(config); @@ -62,13 +63,26 @@ async function main() { const arch = await pool.query(`SELECT COUNT(*) FILTER (WHERE superseded_by IS NOT NULL) as superseded, COUNT(*) FILTER (WHERE archived_at IS NOT NULL) as archived FROM memories`); console.log(` Superseded: ${arch.rows[0].superseded}, Archived: ${arch.rows[0].archived}`); - // 9. Search smoke test + // 9. Historical provenance coverage (read-only) + console.log('\n--- Memory Provenance ---'); + const provenance = await auditMemoryProvenance(pool, database.dialect); + console.log(` Complete effective provenance: ${provenance.completeProvenance} / ${provenance.totalMemories}`); + console.log(` Active complete provenance: ${provenance.activeCompleteProvenance} / ${provenance.activeMemories}`); + console.log(` Rows with any gap: ${provenance.rowsWithAnyGap} total, ${provenance.activeRowsWithAnyGap} active`); + for (const field of REQUIRED_MEMORY_PROVENANCE_FIELDS) { + const suffix = field === 'source_session_id' ? ' (after session_id fallback)' : ''; + console.log(` Missing ${field}${suffix}: ${provenance.missingByField[field]}`); + } + console.log(` Model ID placeholders: unknown=${provenance.unknownModelIdRows}, default=${provenance.defaultModelIdRows}`); + console.log(` Active model placeholders: unknown=${provenance.activeUnknownModelIdRows}, default=${provenance.activeDefaultModelIdRows}`); + + // 10. Search smoke test console.log('\n--- Search Smoke Test ---'); const searchResults = await memoryManager.searchMemories({ query: 'sqlite database', limit: 3 }); console.log(` Query "sqlite database": ${searchResults.length} results`); for (const r of searchResults.slice(0, 2)) console.log(` #${r.memory.id} score=${r.score.toFixed(2)} | ${r.memory.content.substring(0, 80).replace(/\n/g, ' ')}`); - // 10. Embedding generation test + // 11. Embedding generation test console.log('\n--- Embedding Generation Test ---'); try { const vec = await embeddings.generate('test embedding generation'); diff --git a/src/memory-provenance-audit.ts b/src/memory-provenance-audit.ts new file mode 100644 index 0000000..4e745b6 --- /dev/null +++ b/src/memory-provenance-audit.ts @@ -0,0 +1,113 @@ +import { jsonExtractText, type QueryDialect } from './db/query-dialect.js'; +import type { DatabasePool } from './types.js'; + +export const REQUIRED_MEMORY_PROVENANCE_FIELDS = [ + 'source_kind', + 'evidence_strength', + 'source_session_id', + 'source_agent_id', + 'source_model_id', + 'source_surface', +] as const; + +export type RequiredMemoryProvenanceField = + (typeof REQUIRED_MEMORY_PROVENANCE_FIELDS)[number]; + +export interface MemoryProvenanceAuditReport { + totalMemories: number; + activeMemories: number; + completeProvenance: number; + rowsWithAnyGap: number; + activeRowsWithAnyGap: number; + activeCompleteProvenance: number; + missingByField: Record; + unknownModelIdRows: number; + defaultModelIdRows: number; + activeUnknownModelIdRows: number; + activeDefaultModelIdRows: number; +} + +type QueryTarget = Pick; + +function blankToNull(expression: string): string { + return `NULLIF(TRIM(COALESCE(${expression}, '')), '')`; +} + +function effectiveFieldExpressions( + dialect: QueryDialect, +): Record { + const metadataField = (field: RequiredMemoryProvenanceField) => + blankToNull(jsonExtractText(dialect, 'metadata', field)); + + return { + source_kind: metadataField('source_kind'), + evidence_strength: metadataField('evidence_strength'), + source_session_id: `COALESCE(${metadataField('source_session_id')}, ${blankToNull('session_id')})`, + source_agent_id: metadataField('source_agent_id'), + source_model_id: metadataField('source_model_id'), + source_surface: metadataField('source_surface'), + }; +} + +export function buildMemoryProvenanceAuditSql(dialect: QueryDialect): string { + const fields = effectiveFieldExpressions(dialect); + const anyGap = REQUIRED_MEMORY_PROVENANCE_FIELDS + .map((field) => `${fields[field]} IS NULL`) + .join(' OR '); + const active = 'superseded_by IS NULL AND archived_at IS NULL'; + const modelId = fields.source_model_id; + + const missingCounts = REQUIRED_MEMORY_PROVENANCE_FIELDS + .map((field) => `COALESCE(SUM(CASE WHEN ${fields[field]} IS NULL THEN 1 ELSE 0 END), 0) AS missing_${field}`) + .join(',\n '); + + return ` + SELECT + COUNT(*) AS total_memories, + COALESCE(SUM(CASE WHEN ${active} THEN 1 ELSE 0 END), 0) AS active_memories, + COALESCE(SUM(CASE WHEN NOT (${anyGap}) THEN 1 ELSE 0 END), 0) AS complete_provenance, + COALESCE(SUM(CASE WHEN ${anyGap} THEN 1 ELSE 0 END), 0) AS rows_with_any_gap, + COALESCE(SUM(CASE WHEN ${active} AND (${anyGap}) THEN 1 ELSE 0 END), 0) AS active_rows_with_any_gap, + ${missingCounts}, + COALESCE(SUM(CASE WHEN LOWER(${modelId}) = 'unknown' THEN 1 ELSE 0 END), 0) AS unknown_model_id_rows, + COALESCE(SUM(CASE WHEN LOWER(${modelId}) = 'default' THEN 1 ELSE 0 END), 0) AS default_model_id_rows, + COALESCE(SUM(CASE WHEN ${active} AND LOWER(${modelId}) = 'unknown' THEN 1 ELSE 0 END), 0) AS active_unknown_model_id_rows, + COALESCE(SUM(CASE WHEN ${active} AND LOWER(${modelId}) = 'default' THEN 1 ELSE 0 END), 0) AS active_default_model_id_rows + FROM memories + `; +} + +function count(row: Record, key: string): number { + const value = Number(row[key] ?? 0); + return Number.isFinite(value) ? value : 0; +} + +export async function auditMemoryProvenance( + pool: QueryTarget, + dialect: QueryDialect, +): Promise { + const result = await pool.query(buildMemoryProvenanceAuditSql(dialect)); + const row = (result.rows[0] ?? {}) as Record; + + const missingByField = Object.fromEntries( + REQUIRED_MEMORY_PROVENANCE_FIELDS.map((field) => [ + field, + count(row, `missing_${field}`), + ]), + ) as Record; + + return { + totalMemories: count(row, 'total_memories'), + activeMemories: count(row, 'active_memories'), + completeProvenance: count(row, 'complete_provenance'), + rowsWithAnyGap: count(row, 'rows_with_any_gap'), + activeRowsWithAnyGap: count(row, 'active_rows_with_any_gap'), + activeCompleteProvenance: + count(row, 'active_memories') - count(row, 'active_rows_with_any_gap'), + missingByField, + unknownModelIdRows: count(row, 'unknown_model_id_rows'), + defaultModelIdRows: count(row, 'default_model_id_rows'), + activeUnknownModelIdRows: count(row, 'active_unknown_model_id_rows'), + activeDefaultModelIdRows: count(row, 'active_default_model_id_rows'), + }; +} diff --git a/test/memory-provenance-audit.test.ts b/test/memory-provenance-audit.test.ts new file mode 100644 index 0000000..38bc7ac --- /dev/null +++ b/test/memory-provenance-audit.test.ts @@ -0,0 +1,78 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { + auditMemoryProvenance, + buildMemoryProvenanceAuditSql, +} from '../src/memory-provenance-audit.js'; + +describe('memory provenance audit', () => { + it('builds PostgreSQL provenance checks with session fallback and active-state accounting', () => { + const sql = buildMemoryProvenanceAuditSql('pg'); + + assert.match(sql, /metadata->>'source_kind'/); + assert.match(sql, /metadata->>'source_session_id'/); + assert.match(sql, /session_id/); + assert.match(sql, /superseded_by IS NULL AND archived_at IS NULL/); + assert.match(sql, /LOWER\(.+source_model_id/s); + assert.match(sql, /default_model_id_rows/); + }); + + it('builds SQLite provenance checks with JSON extraction', () => { + const sql = buildMemoryProvenanceAuditSql('sqlite'); + + assert.match(sql, /json_extract\(metadata, '\$\.source_kind'\)/); + assert.match(sql, /json_extract\(metadata, '\$\.source_session_id'\)/); + assert.match(sql, /session_id/); + }); + + it('maps database counts without treating string counts as text', async () => { + let observedSql = ''; + const pool = { + query: async (sql: string) => { + observedSql = sql; + return { + rows: [{ + total_memories: '60', + active_memories: '44', + complete_provenance: '20', + rows_with_any_gap: '40', + active_rows_with_any_gap: '26', + missing_source_kind: '3', + missing_evidence_strength: '4', + missing_source_session_id: '5', + missing_source_agent_id: '6', + missing_source_model_id: '7', + missing_source_surface: '8', + unknown_model_id_rows: '9', + default_model_id_rows: '21', + active_unknown_model_id_rows: '2', + active_default_model_id_rows: '17', + }], + rowCount: 1, + }; + }, + }; + + const report = await auditMemoryProvenance(pool as never, 'pg'); + + assert.match(observedSql, /FROM memories/); + assert.equal(report.totalMemories, 60); + assert.equal(report.activeMemories, 44); + assert.equal(report.completeProvenance, 20); + assert.equal(report.rowsWithAnyGap, 40); + assert.equal(report.activeRowsWithAnyGap, 26); + assert.equal(report.activeCompleteProvenance, 18); + assert.deepEqual(report.missingByField, { + source_kind: 3, + evidence_strength: 4, + source_session_id: 5, + source_agent_id: 6, + source_model_id: 7, + source_surface: 8, + }); + assert.equal(report.unknownModelIdRows, 9); + assert.equal(report.defaultModelIdRows, 21); + assert.equal(report.activeUnknownModelIdRows, 2); + assert.equal(report.activeDefaultModelIdRows, 17); + }); +});