diff --git a/.taplo.toml b/.taplo.toml index 1324aa64..4ee9638d 100644 --- a/.taplo.toml +++ b/.taplo.toml @@ -45,6 +45,7 @@ exclude = [ "coverage/**", "dist", "dist/**", + "makefiles/*.toml", "model", "model/**", "node_modules", diff --git a/Makefile.toml b/Makefile.toml index e5a2ca8d..8c2436f2 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -1,1913 +1,19 @@ # Rust workspace tasks. - -# Benchmark -# | task | type | cwd | -# | ------------------------------------------ | --------- | --- | -# | baseline-backfill-100k-docker | command | | -# | baseline-backfill-10k-docker | command | | -# | baseline-backfill-docker | command | | -# | baseline-live-docker | command | | -# | baseline-live-report | command | | -# | baseline-production-private | command | | -# | baseline-production-private-addendum | command | | -# | baseline-production-synthetic | command | | -# | baseline-soak-docker | command | | -# | local-agent-loop | command | | -# | openmemory-ui-export-readback | command | | -# | parity-docker | command | | -# | real-world-first-generation-oss | composite | | -# | real-world-first-generation-oss-json | command | | -# | real-world-first-generation-oss-report | command | | -# | real-world-job-operator-ux | composite | | -# | real-world-job-operator-ux-json | command | | -# | real-world-job-operator-ux-live-adapters | command | | -# | real-world-job-operator-ux-report | command | | -# | real-world-memory | composite | | -# | real-world-memory-adversarial-quality | composite | | -# | real-world-memory-adversarial-quality-json | command | | -# | real-world-memory-adversarial-quality-report | command | | -# | real-world-memory-consolidation | composite | | -# | real-world-memory-consolidation-json | command | | -# | real-world-memory-consolidation-report | command | | -# | real-world-memory-p1-closeout | composite | | -# | real-world-memory-p1-closeout-json | command | | -# | real-world-memory-p1-closeout-report | command | | -# | real-world-memory-p4-production-readiness | composite | | -# | real-world-memory-p4-production-readiness-json | command | | -# | real-world-memory-p4-production-readiness-report | command | | -# | real-world-memory-p4-quality-hardening-closeout | composite | | -# | real-world-memory-p2-knowledge-closeout | composite | | -# | real-world-memory-core-archival | composite | | -# | real-world-memory-core-archival-json | command | | -# | real-world-memory-core-archival-report | command | | -# | real-world-memory-context-trajectory | composite | | -# | real-world-memory-context-trajectory-json | command | | -# | real-world-memory-context-trajectory-report | command | | -# | real-world-memory-evolution | composite | | -# | real-world-memory-evolution-json | command | | -# | real-world-memory-evolution-report | command | | -# | real-world-memory-graph-rag | composite | | -# | real-world-memory-graph-rag-json | command | | -# | real-world-memory-graph-rag-report | command | | -# | real-world-memory-json | command | | -# | real-world-memory-knowledge | composite | | -# | real-world-memory-knowledge-json | command | | -# | real-world-memory-knowledge-report | command | | -# | real-world-memory-live-adapters | command | | -# | real-world-memory-live-consolidation | command | | -# | real-world-memory-live-knowledge | command | | -# | real-world-memory-mem0-openmemory-letta | composite | | -# | real-world-memory-mem0-openmemory-letta-json | command | | -# | real-world-memory-mem0-openmemory-letta-report | command | | -# | real-world-memory-pageindex-openkb | composite | | -# | real-world-memory-pageindex-openkb-json | command | | -# | real-world-memory-pageindex-openkb-report | command | | -# | real-world-memory-proactive-brief | composite | | -# | real-world-memory-proactive-brief-json | command | | -# | real-world-memory-proactive-brief-report | command | | -# | real-world-memory-production-ops | composite | | -# | real-world-memory-production-ops-json | command | | -# | real-world-memory-production-ops-report | command | | -# | real-world-memory-project-decisions | composite | | -# | real-world-memory-project-decisions-json | command | | -# | real-world-memory-project-decisions-report | command | | -# | real-world-memory-quantitative-scoreboard | composite | | -# | real-world-memory-quantitative-scoreboard-json | command | | -# | real-world-memory-quantitative-scoreboard-report | command | | -# | real-world-memory-report | command | | -# | real-world-memory-retrieval | composite | | -# | real-world-memory-retrieval-json | command | | -# | real-world-memory-retrieval-report | command | | -# | real-world-memory-scheduled | composite | | -# | real-world-memory-scheduled-json | command | | -# | real-world-memory-scheduled-report | command | | -# | real-world-memory-service-native-dreaming | command | | -# | real-world-memory-summary | composite | | -# | real-world-memory-summary-json | command | | -# | real-world-memory-summary-report | command | | -# | real-world-memory-work-continuity | composite | | -# | real-world-memory-work-continuity-json | command | | -# | real-world-memory-work-continuity-report | command | | - -[tasks.baseline-backfill-100k-docker] -workspace = false -command = "bash" -args = [ - "scripts/baseline-docker.sh", - "backfill-100k", -] - -[tasks.baseline-backfill-10k-docker] -workspace = false -command = "bash" -args = [ - "scripts/baseline-docker.sh", - "backfill-10k", -] - -[tasks.baseline-backfill-docker] -workspace = false -command = "bash" -args = [ - "scripts/baseline-docker.sh", - "backfill", -] - -[tasks.baseline-live-docker] -workspace = false -command = "bash" -args = [ - "scripts/baseline-docker.sh", - "live", -] - -[tasks.baseline-live-report] -workspace = false -command = "bash" -args = [ - "scripts/live-baseline-report-to-md.sh", -] - -[tasks.baseline-production-private] -workspace = false -command = "bash" -args = [ - "scripts/baseline-docker.sh", - "production-private", -] - -[tasks.baseline-production-private-addendum] -workspace = false -command = "bash" -args = [ - "scripts/baseline-docker.sh", - "production-private-addendum", -] - -[tasks.baseline-production-synthetic] -workspace = false -command = "bash" -args = [ - "scripts/baseline-docker.sh", - "production-synthetic", -] - -[tasks.baseline-soak-docker] -workspace = false -command = "bash" -args = [ - "scripts/baseline-docker.sh", - "soak", -] - -[tasks.local-agent-loop] -workspace = false -command = "bash" -args = [ - "scripts/local-agent-loop.sh", -] - -[tasks.openmemory-ui-export-readback] -workspace = false -command = "bash" -args = [ - "scripts/baseline-docker.sh", - "openmemory-ui-export-readback", -] - -[tasks.parity-docker] -workspace = false -command = "docker" -args = [ - "compose", - "-f", - "docker-compose.parity.yml", - "run", - "--build", - "--rm", - "parity-runner", -] - -[tasks.real-world-first-generation-oss] -workspace = false -dependencies = [ - "real-world-first-generation-oss-report", -] - -[tasks.real-world-first-generation-oss-json] -workspace = false -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "run", - "--fixtures", - "apps/elf-eval/fixtures/real_world_external_adapters/first_generation_oss", - "--out", - "tmp/real-world-memory/first-generation-oss/report.json", - "--run-id", - "first-generation-oss-continuity-source-store", - "--adapter-id", - "fixture_first_generation_oss", - "--adapter-name", - "First-generation OSS fixture coverage", -] - -[tasks.real-world-first-generation-oss-report] -workspace = false -dependencies = [ - "real-world-first-generation-oss-json", -] -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "publish", - "--report", - "tmp/real-world-memory/first-generation-oss/report.json", - "--out", - "tmp/real-world-memory/first-generation-oss/report.md", -] - -[tasks.real-world-job-operator-ux] -workspace = false -dependencies = [ - "real-world-job-operator-ux-report", -] - -[tasks.real-world-job-operator-ux-json] -workspace = false -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "run", - "--fixtures", - "apps/elf-eval/fixtures/real_world_job/operator_debugging_ux", - "--out", - "tmp/real-world-job/real-world-job-operator-ux-report.json", - "--run-id", - "real-world-job-operator-ux", - "--adapter-id", - "fixture_operator_ux", - "--adapter-name", - "ELF operator UX fixture", -] - -[tasks.real-world-job-operator-ux-live-adapters] -workspace = false -command = "bash" -args = [ - "scripts/real-world-docker.sh", - "job-operator-ux-live-adapters", -] - -[tasks.real-world-job-operator-ux-report] -workspace = false -dependencies = [ - "real-world-job-operator-ux-json", -] -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "publish", - "--report", - "tmp/real-world-job/real-world-job-operator-ux-report.json", - "--out", - "tmp/real-world-job/real-world-job-operator-ux-report.md", -] - -[tasks.real-world-memory] -workspace = false -dependencies = [ - "real-world-memory-report", -] - -[tasks.real-world-memory-adversarial-quality] -workspace = false -dependencies = [ - "real-world-memory-adversarial-quality-report", -] - -[tasks.real-world-memory-adversarial-quality-json] -workspace = false -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "run", - "--fixtures", - "apps/elf-eval/fixtures/real_world_memory/adversarial_quality", - "--out", - "tmp/real-world-memory/adversarial-quality/report.json", - "--run-id", - "real-world-memory-adversarial-quality", - "--adapter-id", - "fixture_adversarial_quality", - "--adapter-name", - "ELF adversarial quality fixture", -] - -[tasks.real-world-memory-adversarial-quality-report] -workspace = false -dependencies = [ - "real-world-memory-adversarial-quality-json", -] -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "publish", - "--report", - "tmp/real-world-memory/adversarial-quality/report.json", - "--out", - "tmp/real-world-memory/adversarial-quality/report.md", -] - -[tasks.real-world-memory-consolidation] -workspace = false -dependencies = [ - "real-world-memory-consolidation-report", -] - -[tasks.real-world-memory-consolidation-json] -workspace = false -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "run", - "--fixtures", - "apps/elf-eval/fixtures/real_world_memory/consolidation", - "--out", - "tmp/real-world-memory/consolidation/report.json", - "--run-id", - "real-world-memory-consolidation", - "--adapter-id", - "fixture_consolidation", - "--adapter-name", - "ELF consolidation fixture", -] - -[tasks.real-world-memory-consolidation-report] -workspace = false -dependencies = [ - "real-world-memory-consolidation-json", -] -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "publish", - "--report", - "tmp/real-world-memory/consolidation/report.json", - "--out", - "tmp/real-world-memory/consolidation/report.md", -] - -[tasks.real-world-memory-p1-closeout] -workspace = false -dependencies = [ - "real-world-memory-p1-closeout-report", -] - -[tasks.real-world-memory-p1-closeout-json] -workspace = false -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "run", - "--fixtures", - "apps/elf-eval/fixtures/real_world_memory/p1_closeout", - "--out", - "tmp/real-world-memory/p1-closeout/report.json", - "--run-id", - "real-world-memory-p1-closeout", - "--adapter-id", - "fixture_p1_closeout", - "--adapter-name", - "ELF P1 closeout fixture", -] - -[tasks.real-world-memory-p1-closeout-report] -workspace = false -dependencies = [ - "real-world-memory-p1-closeout-json", -] -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "publish", - "--report", - "tmp/real-world-memory/p1-closeout/report.json", - "--out", - "tmp/real-world-memory/p1-closeout/report.md", -] - -[tasks.real-world-memory-p4-production-readiness] -workspace = false -dependencies = [ - "real-world-memory-p4-production-readiness-report", -] - -[tasks.real-world-memory-p4-production-readiness-json] -workspace = false -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "run", - "--fixtures", - "apps/elf-eval/fixtures/real_world_memory/production_ops", - "--out", - "tmp/real-world-memory/p4-production-readiness/report.json", - "--run-id", - "real-world-memory-p4-production-readiness", - "--adapter-id", - "fixture_production_ops", - "--adapter-name", - "ELF P4 production-readiness fixture", -] - -[tasks.real-world-memory-p4-production-readiness-report] -workspace = false -dependencies = [ - "real-world-memory-p4-production-readiness-json", -] -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "publish", - "--report", - "tmp/real-world-memory/p4-production-readiness/report.json", - "--out", - "tmp/real-world-memory/p4-production-readiness/report.md", -] - -[tasks.real-world-memory-p4-quality-hardening-closeout] -workspace = false -dependencies = [ - "real-world-memory-adversarial-quality-report", - "real-world-memory-p2-knowledge-closeout", - "real-world-memory-p4-production-readiness-report", -] - -[tasks.real-world-memory-p2-knowledge-closeout] -workspace = false -dependencies = [ - "real-world-memory-source-library-report", - "real-world-memory-knowledge-report", -] - -[tasks.real-world-memory-core-archival] -workspace = false -dependencies = [ - "real-world-memory-core-archival-report", -] - -[tasks.real-world-memory-core-archival-json] -workspace = false -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "run", - "--fixtures", - "apps/elf-eval/fixtures/real_world_memory/core_archival_memory", - "--out", - "tmp/real-world-memory/core-archival/report.json", - "--run-id", - "real-world-memory-core-archival", - "--adapter-id", - "fixture_core_archival_memory", - "--adapter-name", - "ELF core and archival memory fixture", -] - -[tasks.real-world-memory-core-archival-report] -workspace = false -dependencies = [ - "real-world-memory-core-archival-json", -] -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "publish", - "--report", - "tmp/real-world-memory/core-archival/report.json", - "--out", - "tmp/real-world-memory/core-archival/report.md", -] - -[tasks.real-world-memory-context-trajectory] -workspace = false -dependencies = [ - "real-world-memory-context-trajectory-report", -] - -[tasks.real-world-memory-context-trajectory-json] -workspace = false -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "run", - "--fixtures", - "apps/elf-eval/fixtures/real_world_memory/context_trajectory", - "--out", - "tmp/real-world-memory/context-trajectory/report.json", - "--run-id", - "real-world-memory-context-trajectory", - "--adapter-id", - "fixture_context_trajectory", - "--adapter-name", - "ELF context trajectory fixture", -] - -[tasks.real-world-memory-context-trajectory-report] -workspace = false -dependencies = [ - "real-world-memory-context-trajectory-json", -] -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "publish", - "--report", - "tmp/real-world-memory/context-trajectory/report.json", - "--out", - "tmp/real-world-memory/context-trajectory/report.md", -] - -[tasks.real-world-memory-evolution] -workspace = false -dependencies = [ - "real-world-memory-evolution-report", -] - -[tasks.real-world-memory-evolution-json] -workspace = false -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "run", - "--fixtures", - "apps/elf-eval/fixtures/real_world_memory/evolution", - "--out", - "tmp/real-world-memory/evolution-report.json", - "--run-id", - "real-world-memory-evolution", - "--adapter-id", - "fixture_memory_evolution", - "--adapter-name", - "ELF fixture memory evolution", -] - -[tasks.real-world-memory-evolution-report] -workspace = false -dependencies = [ - "real-world-memory-evolution-json", -] -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "publish", - "--report", - "tmp/real-world-memory/evolution-report.json", - "--out", - "tmp/real-world-memory/evolution-report.md", -] - -[tasks.real-world-memory-graph-rag] -workspace = false -dependencies = [ - "real-world-memory-graph-rag-report", -] - -[tasks.real-world-memory-graph-rag-json] -workspace = false -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "run", - "--fixtures", - "apps/elf-eval/fixtures/real_world_external_adapters/graph_rag", - "--out", - "tmp/real-world-memory/graph-rag/report.json", - "--run-id", - "real-world-memory-graph-rag", - "--adapter-id", - "fixture_graph_rag_external_adapters", - "--adapter-name", - "Graph/RAG representative external-adapter fixtures", -] - -[tasks.real-world-memory-graph-rag-report] -workspace = false -dependencies = [ - "real-world-memory-graph-rag-json", -] -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "publish", - "--report", - "tmp/real-world-memory/graph-rag/report.json", - "--out", - "tmp/real-world-memory/graph-rag/report.md", -] - -[tasks.real-world-memory-json] -workspace = false -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "run", - "--fixtures", - "apps/elf-eval/fixtures/real_world_memory", - "--out", - "tmp/real-world-memory/real-world-memory-report.json", - "--run-id", - "real-world-memory", - "--adapter-id", - "elf_real_world_memory_fixture", - "--adapter-name", - "ELF real-world memory fixture", -] - -[tasks.real-world-memory-knowledge] -workspace = false -dependencies = [ - "real-world-memory-knowledge-report", -] - -[tasks.real-world-memory-knowledge-json] -workspace = false -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "run", - "--fixtures", - "apps/elf-eval/fixtures/real_world_memory/knowledge", - "--out", - "tmp/real-world-memory/knowledge-report.json", - "--run-id", - "real-world-memory-knowledge", - "--adapter-id", - "fixture_knowledge", - "--adapter-name", - "ELF knowledge fixture", -] - -[tasks.real-world-memory-knowledge-report] -workspace = false -dependencies = [ - "real-world-memory-knowledge-json", -] -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "publish", - "--report", - "tmp/real-world-memory/knowledge-report.json", - "--out", - "tmp/real-world-memory/knowledge-report.md", -] - -[tasks.real-world-memory-mem0-openmemory-letta] -workspace = false -dependencies = [ - "real-world-memory-mem0-openmemory-letta-report", -] - -[tasks.real-world-memory-mem0-openmemory-letta-json] -workspace = false -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "run", - "--fixtures", - "apps/elf-eval/fixtures/real_world_external_adapters/mem0_openmemory_letta", - "--out", - "tmp/real-world-memory/mem0-openmemory-letta/report.json", - "--run-id", - "real-world-memory-mem0-openmemory-letta", - "--adapter-id", - "fixture_mem0_openmemory_letta", - "--adapter-name", - "mem0/OpenMemory and Letta memory-history/core-archive adapters", - "--adapter-behavior", - "same_corpus_adapter_fixture", - "--adapter-storage-status", - "blocked", - "--adapter-runtime-status", - "blocked", - "--adapter-notes", - "Offline fixtures map mem0 SDK history/export outputs to source ids and preserve OpenMemory UI/export plus Letta core/archive blockers until contained product exports map source ids.", -] - -[tasks.real-world-memory-mem0-openmemory-letta-report] -workspace = false -dependencies = [ - "real-world-memory-mem0-openmemory-letta-json", -] -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "publish", - "--report", - "tmp/real-world-memory/mem0-openmemory-letta/report.json", - "--out", - "tmp/real-world-memory/mem0-openmemory-letta/report.md", -] - -[tasks.real-world-memory-pageindex-openkb] -workspace = false -dependencies = [ - "real-world-memory-pageindex-openkb-report", -] - -[tasks.real-world-memory-pageindex-openkb-json] -workspace = false -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "run", - "--fixtures", - "apps/elf-eval/fixtures/real_world_external_adapters/pageindex_openkb", - "--out", - "tmp/real-world-memory/pageindex-openkb/report.json", - "--run-id", - "real-world-memory-pageindex-openkb", - "--adapter-id", - "fixture_pageindex_openkb_same_corpus", - "--adapter-name", - "PageIndex/OpenKB same-corpus adapter blockers", - "--adapter-behavior", - "typed_setup_blocker", - "--adapter-storage-status", - "blocked", - "--adapter-runtime-status", - "blocked", - "--adapter-notes", - "Offline fixtures preserve same-corpus PageIndex/OpenKB setup blockers until contained product artifacts map to ELF source ids.", -] - -[tasks.real-world-memory-pageindex-openkb-report] -workspace = false -dependencies = [ - "real-world-memory-pageindex-openkb-json", -] -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "publish", - "--report", - "tmp/real-world-memory/pageindex-openkb/report.json", - "--out", - "tmp/real-world-memory/pageindex-openkb/report.md", -] - -[tasks.real-world-memory-source-library] -workspace = false -dependencies = [ - "real-world-memory-source-library-report", -] - -[tasks.real-world-memory-source-library-json] -workspace = false -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "run", - "--fixtures", - "apps/elf-eval/fixtures/real_world_memory/source_library", - "--out", - "tmp/real-world-memory/source-library-report.json", - "--run-id", - "real-world-memory-source-library", - "--adapter-id", - "fixture_source_library", - "--adapter-name", - "ELF source library fixture", -] - -[tasks.real-world-memory-source-library-report] -workspace = false -dependencies = [ - "real-world-memory-source-library-json", -] -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "publish", - "--report", - "tmp/real-world-memory/source-library-report.json", - "--out", - "tmp/real-world-memory/source-library-report.md", -] - -[tasks.real-world-memory-live-adapters] -workspace = false -command = "bash" -args = [ - "scripts/real-world-docker.sh", - "memory-live-adapters", -] - -[tasks.real-world-memory-live-consolidation] -workspace = false -command = "bash" -args = [ - "scripts/real-world-docker.sh", - "memory-live-consolidation", -] - -[tasks.real-world-memory-live-knowledge] -workspace = false -command = "bash" -args = [ - "scripts/real-world-docker.sh", - "memory-live-knowledge", -] - -[tasks.real-world-memory-proactive-brief] -workspace = false -dependencies = [ - "real-world-memory-proactive-brief-report", -] - -[tasks.real-world-memory-proactive-brief-json] -workspace = false -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "run", - "--fixtures", - "apps/elf-eval/fixtures/real_world_memory/proactive_brief", - "--out", - "tmp/real-world-memory/proactive-brief/report.json", - "--run-id", - "real-world-memory-proactive-brief", - "--adapter-id", - "fixture_proactive_brief", - "--adapter-name", - "ELF proactive brief fixture", -] - -[tasks.real-world-memory-proactive-brief-report] -workspace = false -dependencies = [ - "real-world-memory-proactive-brief-json", -] -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "publish", - "--report", - "tmp/real-world-memory/proactive-brief/report.json", - "--out", - "tmp/real-world-memory/proactive-brief/report.md", -] - -[tasks.real-world-memory-production-ops] -workspace = false -dependencies = [ - "real-world-memory-production-ops-report", -] - -[tasks.real-world-memory-production-ops-json] -workspace = false -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "run", - "--fixtures", - "apps/elf-eval/fixtures/real_world_memory/production_ops", - "--run-id", - "real-world-memory-production-ops", - "--adapter-id", - "fixture_production_ops", - "--adapter-name", - "ELF production-ops fixture", - "--out", - "tmp/real-world-memory/production-ops-report.json", -] - -[tasks.real-world-memory-production-ops-report] -workspace = false -dependencies = [ - "real-world-memory-production-ops-json", -] -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "publish", - "--report", - "tmp/real-world-memory/production-ops-report.json", - "--out", - "tmp/real-world-memory/production-ops-report.md", -] - -[tasks.real-world-memory-project-decisions] -workspace = false -dependencies = [ - "real-world-memory-project-decisions-report", -] - -[tasks.real-world-memory-project-decisions-json] -workspace = false -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "run", - "--fixtures", - "apps/elf-eval/fixtures/real_world_memory/project_decisions", - "--out", - "tmp/real-world-memory/project-decisions/report.json", - "--run-id", - "real-world-memory-project-decisions", - "--adapter-id", - "fixture_project_decisions", - "--adapter-name", - "ELF project decision fixture", -] - -[tasks.real-world-memory-project-decisions-report] -workspace = false -dependencies = [ - "real-world-memory-project-decisions-json", -] -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "publish", - "--report", - "tmp/real-world-memory/project-decisions/report.json", - "--out", - "tmp/real-world-memory/project-decisions/report.md", -] - -[tasks.real-world-memory-quantitative-scoreboard] -workspace = false -dependencies = [ - "real-world-memory-quantitative-scoreboard-report", -] - -[tasks.real-world-memory-quantitative-scoreboard-json] -workspace = false -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "run", - "--fixtures", - "apps/elf-eval/fixtures/real_world_memory", - "--out", - "tmp/real-world-memory/quantitative-scoreboard/report.json", - "--run-id", - "public-quantitative-competitor-scoreboard", - "--adapter-id", - "elf_real_world_memory_fixture", - "--adapter-name", - "ELF real-world memory fixture", -] - -[tasks.real-world-memory-quantitative-scoreboard-report] -workspace = false -dependencies = [ - "real-world-memory-quantitative-scoreboard-json", -] -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "publish", - "--report", - "tmp/real-world-memory/quantitative-scoreboard/report.json", - "--out", - "tmp/real-world-memory/quantitative-scoreboard/report.md", -] - -[tasks.real-world-memory-report] -workspace = false -dependencies = [ - "real-world-memory-json", -] -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "publish", - "--report", - "tmp/real-world-memory/real-world-memory-report.json", - "--out", - "tmp/real-world-memory/real-world-memory-report.md", -] - -[tasks.real-world-memory-retrieval] -workspace = false -dependencies = [ - "real-world-memory-retrieval-report", -] - -[tasks.real-world-memory-retrieval-json] -workspace = false -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "run", - "--fixtures", - "apps/elf-eval/fixtures/real_world_memory/retrieval", - "--run-id", - "real-world-memory-retrieval", - "--adapter-id", - "fixture_retrieval", - "--adapter-name", - "ELF fixture retrieval cases", - "--out", - "tmp/real-world-memory/retrieval-report.json", -] - -[tasks.real-world-memory-retrieval-report] -workspace = false -dependencies = [ - "real-world-memory-retrieval-json", -] -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "publish", - "--report", - "tmp/real-world-memory/retrieval-report.json", - "--out", - "tmp/real-world-memory/retrieval-report.md", -] - -[tasks.real-world-memory-scheduled] -workspace = false -dependencies = [ - "real-world-memory-scheduled-report", -] - -[tasks.real-world-memory-scheduled-json] -workspace = false -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "run", - "--fixtures", - "apps/elf-eval/fixtures/real_world_memory/scheduled_memory", - "--out", - "tmp/real-world-memory/scheduled/report.json", - "--run-id", - "real-world-memory-scheduled", - "--adapter-id", - "fixture_scheduled_memory", - "--adapter-name", - "ELF scheduled memory fixture", -] - -[tasks.real-world-memory-scheduled-report] -workspace = false -dependencies = [ - "real-world-memory-scheduled-json", -] -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "publish", - "--report", - "tmp/real-world-memory/scheduled/report.json", - "--out", - "tmp/real-world-memory/scheduled/report.md", -] - -[tasks.real-world-memory-work-continuity] -workspace = false -dependencies = [ - "real-world-memory-work-continuity-report", -] - -[tasks.real-world-memory-work-continuity-json] -workspace = false -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "run", - "--fixtures", - "apps/elf-eval/fixtures/real_world_memory/work_continuity", - "--out", - "tmp/real-world-memory/work-continuity/report.json", - "--run-id", - "real-world-memory-work-continuity", - "--adapter-id", - "fixture_work_continuity", - "--adapter-name", - "ELF Work Continuity fixture", -] - -[tasks.real-world-memory-work-continuity-report] -workspace = false -dependencies = [ - "real-world-memory-work-continuity-json", -] -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "publish", - "--report", - "tmp/real-world-memory/work-continuity/report.json", - "--out", - "tmp/real-world-memory/work-continuity/report.md", -] - -[tasks.real-world-memory-service-native-dreaming] -workspace = false -command = "bash" -args = [ - "scripts/real-world-docker.sh", - "memory-service-native-dreaming", -] - -[tasks.real-world-memory-summary] -workspace = false -dependencies = [ - "real-world-memory-summary-report", -] - -[tasks.real-world-memory-summary-json] -workspace = false -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "run", - "--fixtures", - "apps/elf-eval/fixtures/real_world_memory/memory_summary", - "--out", - "tmp/real-world-memory/memory-summary/report.json", - "--run-id", - "real-world-memory-summary", - "--adapter-id", - "fixture_memory_summary", - "--adapter-name", - "ELF memory summary fixture", -] - -[tasks.real-world-memory-summary-report] -workspace = false -dependencies = [ - "real-world-memory-summary-json", -] -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "publish", - "--report", - "tmp/real-world-memory/memory-summary/report.json", - "--out", - "tmp/real-world-memory/memory-summary/report.md", -] - -# Check -# | task | type | cwd | -# | ---------------- | --------- | --- | -# | check | composite | | -# | check-docs | command | | -# | check-rust | command | | -# | check-trace-gate | command | | -# | checks | composite | | - -[tasks.check] -clear = true -workspace = false -dependencies = [ - "fmt-check", - "check-docs", - "check-rust", - "lint", - "test", -] - -[tasks.check-docs] -workspace = false -command = "python3" -args = [ - "scripts/check-docs.py", -] - -[tasks.check-rust] -workspace = false -command = "cargo" -args = [ - "check", - "--workspace", - "--all-targets", - "--all-features", -] - -[tasks.check-trace-gate] -workspace = false -command = "bash" -args = [ - "scripts/trace-gate.sh", -] - -[tasks.checks] -workspace = false -dependencies = [ - "check", -] - -# Clean -# | task | type | cwd | -# | -------------------------- | ------- | --- | -# | clean-baseline-live-docker | command | | -# | clean-parity-docker | command | | - -[tasks.clean-baseline-live-docker] -workspace = false -command = "docker" -args = [ - "compose", - "-f", - "docker-compose.baseline.yml", - "down", - "-v", - "--remove-orphans", -] - -[tasks.clean-parity-docker] -workspace = false -command = "docker" -args = [ - "compose", - "-f", - "docker-compose.parity.yml", - "down", - "-v", - "--remove-orphans", -] - -# Format -# | task | type | cwd | -# | -------------- | --------- | --- | -# | fmt | composite | | -# | fmt-check | composite | | -# | fmt-rust | command | | -# | fmt-rust-check | extend | | -# | fmt-toml | command | | -# | fmt-toml-check | extend | | - -[tasks.fmt] -workspace = false -dependencies = [ - "fmt-rust", - "fmt-toml", -] - -[tasks.fmt-check] -workspace = false -dependencies = [ - "fmt-rust-check", - "fmt-toml-check", -] - -[tasks.fmt-rust] -workspace = false -script = "cargo +nightly fmt --all" - -[tasks.fmt-rust-check] -extend = "fmt-rust" -script = "cargo +nightly fmt --all -- --check" - -[tasks.fmt-toml] -workspace = false -command = "taplo" -args = [ - "fmt", -] - -[tasks.fmt-toml-check] -extend = "fmt-toml" -args = [ - "fmt", - "--check", -] - -# Lint -# | task | type | cwd | -# | ----------- | --------- | --- | -# | lint | composite | | -# | lint-rust | command | | -# | lint-vstyle | command | | - -[tasks.lint] -workspace = false -dependencies = [ - "lint-rust", - "lint-vstyle", -] - -[tasks.lint-rust] -workspace = false -command = "cargo" -args = [ - "clippy", - "--all-features", - "--all-targets", - "--workspace", - "--", - "-D", - "clippy::all", - "-D", - "clippy::too_many_lines", - "-D", - "clippy::unwrap_used", - "-D", - "clippy::use_self", - "-D", - "clippy::wildcard_imports", - "-D", - "missing-docs", - "-D", - "unused-crate-dependencies", - "-D", - "warnings", -] - -[tasks.lint-vstyle] -workspace = false -command = "cargo" -args = [ - "vstyle", - "curate", - "--language", - "rust", - "--workspace", - "--all-features", -] - -# Lint Fix -# | task | type | cwd | -# | --------------- | --------- | --- | -# | lint-fix | composite | | -# | lint-fix-rust | command | | -# | lint-fix-vstyle | command | | - -[tasks.lint-fix] -workspace = false -dependencies = [ - "lint-fix-rust", - "lint-fix-vstyle", -] - -[tasks.lint-fix-rust] -workspace = false -command = "cargo" -args = [ - "clippy", - "--fix", - "--allow-dirty", - "--all-features", - "--all-targets", - "--workspace", - "--", - "-D", - "clippy::all", - "-D", - "clippy::too_many_lines", - "-D", - "clippy::unwrap_used", - "-D", - "clippy::use_self", - "-D", - "clippy::wildcard_imports", - "-D", - "missing-docs", - "-D", - "unused-crate-dependencies", - "-D", - "warnings", -] - -[tasks.lint-fix-vstyle] -workspace = false -command = "cargo" -args = [ - "vstyle", - "tune", - "--language", - "rust", - "--workspace", - "--all-features", - "--strict", -] - -# Research -# | task | type | cwd | -# | --------------------------------------- | --------- | --- | -# | external-memory-radar | command | | -# | external-memory-radar-artifact | composite | | -# | external-memory-radar-artifact-json | command | | -# | external-memory-radar-artifact-validate | command | | -# | external-memory-radar-dry-run | composite | | -# | external-memory-radar-dry-run-json | command | | -# | external-memory-radar-dry-run-validate | command | | -# | external-memory-radar-validate | command | | - -[tasks.external-memory-radar] -workspace = false -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "external_memory_pattern_radar", - "--", - "run", - "--cursor", - "apps/elf-eval/fixtures/external_memory_pattern_radar/cursor.json", - "--summary", - "docs/evidence/external_memory_pattern_radar_latest.md", -] - -[tasks.external-memory-radar-artifact] -workspace = false -dependencies = [ - "external-memory-radar-artifact-json", - "external-memory-radar-artifact-validate", -] - -[tasks.external-memory-radar-artifact-json] -workspace = false -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "external_memory_pattern_radar", - "--", - "run", - "--cursor", - "apps/elf-eval/fixtures/external_memory_pattern_radar/cursor.json", - "--out-cursor", - "tmp/external-memory-pattern-radar/cursor.json", - "--summary", - "tmp/external-memory-pattern-radar/latest.md", -] - -[tasks.external-memory-radar-artifact-validate] -workspace = false -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "external_memory_pattern_radar", - "--", - "validate", - "--cursor", - "tmp/external-memory-pattern-radar/cursor.json", -] - -[tasks.external-memory-radar-dry-run] -workspace = false -dependencies = [ - "external-memory-radar-dry-run-json", - "external-memory-radar-dry-run-validate", -] - -[tasks.external-memory-radar-dry-run-json] -workspace = false -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "external_memory_pattern_radar", - "--", - "run", - "--mode", - "offline", - "--cursor", - "apps/elf-eval/fixtures/external_memory_pattern_radar/cursor.json", - "--out-cursor", - "tmp/external-memory-pattern-radar/cursor.json", - "--summary", - "tmp/external-memory-pattern-radar/latest.md", -] - -[tasks.external-memory-radar-dry-run-validate] -workspace = false -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "external_memory_pattern_radar", - "--", - "validate", - "--cursor", - "tmp/external-memory-pattern-radar/cursor.json", -] - -[tasks.external-memory-radar-validate] -workspace = false -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "external_memory_pattern_radar", - "--", - "validate", - "--cursor", - "apps/elf-eval/fixtures/external_memory_pattern_radar/cursor.json", -] - -# Smoke -# | task | type | cwd | -# | ---------------------------------- | --------- | --- | -# | smoke-graphify-docker-graph-report | command | | -# | smoke-graphiti-zep-docker-temporal | command | | -# | smoke-graphrag-docker | command | | -# | smoke-letta-core-archive-export-readback | command | | -# | smoke-lightrag-docker-context | command | | -# | smoke-ragflow-docker | command | | -# | smoke-real-world-job | composite | | -# | smoke-real-world-job-json | command | | -# | smoke-real-world-job-report | command | | - -[tasks.smoke-graphify-docker-graph-report] -workspace = false -command = "bash" -args = [ - "scripts/smoke-docker.sh", - "graphify-docker-graph-report", -] - -[tasks.smoke-graphiti-zep-docker-temporal] -workspace = false -command = "bash" -args = [ - "scripts/smoke-docker.sh", - "graphiti-zep-docker-temporal", -] - -[tasks.smoke-graphrag-docker] -workspace = false -command = "bash" -args = [ - "scripts/smoke-docker.sh", - "graphrag-docker", -] - -[tasks.smoke-letta-core-archive-export-readback] -workspace = false -command = "bash" -args = [ - "scripts/smoke-docker.sh", - "letta-core-archive-export-readback", -] - -[tasks.smoke-lightrag-docker-context] -workspace = false -command = "bash" -args = [ - "scripts/smoke-docker.sh", - "lightrag-docker-context", -] - -[tasks.smoke-ragflow-docker] -workspace = false -command = "bash" -args = [ - "scripts/ragflow-docker-evidence-smoke.sh", -] - -[tasks.smoke-real-world-job] -workspace = false -dependencies = [ - "smoke-real-world-job-report", -] - -[tasks.smoke-real-world-job-json] -workspace = false -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "run", - "--fixtures", - "apps/elf-eval/fixtures/real_world_memory/work_resume", - "--out", - "tmp/real-world-job/real-world-job-smoke-report.json", -] - -[tasks.smoke-real-world-job-report] -workspace = false -dependencies = [ - "smoke-real-world-job-json", -] -command = "cargo" -args = [ - "run", - "-p", - "elf-eval", - "--bin", - "real_world_job_benchmark", - "--", - "publish", - "--report", - "tmp/real-world-job/real-world-job-smoke-report.json", - "--out", - "tmp/real-world-job/real-world-job-smoke-report.md", -] - -# Test -# | task | type | cwd | -# | --------------------- | --------- | --- | -# | test | composite | | -# | test-e2e | command | | -# | test-rust | command | | -# | test-rust-all | command | | -# | test-rust-integration | command | | - -[tasks.test] -clear = true -workspace = false -dependencies = [ - "test-rust", -] - -[tasks.test-e2e] -workspace = false -command = "bash" -args = [ - "scripts/context-misranking-harness.sh", -] - -[tasks.test-rust] -workspace = false -command = "cargo" -args = [ - "nextest", - "run", - "--workspace", - "--all-targets", - "--all-features", -] - -[tasks.test-rust-all] -workspace = false -command = "cargo" -args = [ - "nextest", - "run", - "--workspace", - "--all-targets", - "--all-features", - "--run-ignored", - "all", -] - -[tasks.test-rust-integration] -workspace = false -command = "cargo" -args = [ - "nextest", - "run", - "--workspace", - "--all-targets", - "--all-features", - "--run-ignored", - "only", +# +# `Makefile.toml` is the cargo-make entrypoint. Task definitions live in the +# focused makefiles below so command ownership stays navigable while this file +# remains the public task-runner authority. + +extend = [ + { path = "makefiles/benchmark-core.toml" }, + { path = "makefiles/benchmark-memory-a.toml" }, + { path = "makefiles/benchmark-memory-b.toml" }, + { path = "makefiles/check.toml" }, + { path = "makefiles/clean.toml" }, + { path = "makefiles/format.toml" }, + { path = "makefiles/lint.toml" }, + { path = "makefiles/lint-fix.toml" }, + { path = "makefiles/research.toml" }, + { path = "makefiles/smoke.toml" }, + { path = "makefiles/test.toml" }, ] diff --git a/apps/elf-eval/tests/real_world_job_benchmark/closeout_reports_workspace.rs b/apps/elf-eval/tests/real_world_job_benchmark/closeout_reports_workspace.rs index 382c8ba6..092693cb 100644 --- a/apps/elf-eval/tests/real_world_job_benchmark/closeout_reports_workspace.rs +++ b/apps/elf-eval/tests/real_world_job_benchmark/closeout_reports_workspace.rs @@ -13,7 +13,7 @@ fn p2_knowledge_workspace_closeout_preserves_pageindex_openkb_boundaries() -> Re let markdown = fs::read_to_string( support::p2_knowledge_workspace_pageindex_openkb_closeout_report_markdown_path()?, )?; - let makefile = fs::read_to_string(support::workspace_root()?.join("Makefile.toml"))?; + let makefile = support::make_task_catalog()?; let benchmarking_index = fs::read_to_string(support::benchmarking_index_path()?)?; let readme = fs::read_to_string(support::readme_path()?)?; let benchmark_runbook = fs::read_to_string( diff --git a/apps/elf-eval/tests/real_world_job_benchmark/consolidation_knowledge_tests_live.rs b/apps/elf-eval/tests/real_world_job_benchmark/consolidation_knowledge_tests_live.rs index 77af72d6..f0e90a96 100644 --- a/apps/elf-eval/tests/real_world_job_benchmark/consolidation_knowledge_tests_live.rs +++ b/apps/elf-eval/tests/real_world_job_benchmark/consolidation_knowledge_tests_live.rs @@ -22,7 +22,7 @@ fn live_consolidation_report_preserves_reviewable_output_boundaries() -> Result< .join("benchmarking") .join("real_world_agent_memory_benchmark.md"), )?; - let makefile = fs::read_to_string(workspace.join("Makefile.toml"))?; + let makefile = support::make_task_catalog()?; let live_script = fs::read_to_string(workspace.join("scripts/real-world-consolidation-live-adapter.sh"))?; let live_adapter = @@ -136,7 +136,7 @@ fn assert_live_consolidation_scripts(docker_script: &str, live_script: &str, liv #[test] fn live_knowledge_page_rebuild_lint_has_dedicated_docker_task() -> Result<()> { let workspace = support::workspace_root()?; - let makefile = fs::read_to_string(workspace.join("Makefile.toml"))?; + let makefile = support::make_task_catalog()?; let docker_script = fs::read_to_string(workspace.join("scripts/real-world-docker.sh"))?; let live_script = fs::read_to_string(workspace.join("scripts/real-world-knowledge-live-adapter.sh"))?; diff --git a/apps/elf-eval/tests/real_world_job_benchmark/live_adapter_tasks.rs b/apps/elf-eval/tests/real_world_job_benchmark/live_adapter_tasks.rs index 096a9263..9974f5d9 100644 --- a/apps/elf-eval/tests/real_world_job_benchmark/live_adapter_tasks.rs +++ b/apps/elf-eval/tests/real_world_job_benchmark/live_adapter_tasks.rs @@ -55,7 +55,7 @@ fn append_rust_sources(dir: &Path, source: &mut String) -> Result<()> { #[test] fn live_adapter_aggregate_forwards_graph_rag_smoke_controls() -> Result<()> { let workspace = support::workspace_root()?; - let makefile = fs::read_to_string(workspace.join("Makefile.toml"))?; + let makefile = support::make_task_catalog()?; let docker_script = fs::read_to_string(workspace.join("scripts/real-world-docker.sh"))?; assert!( @@ -101,7 +101,7 @@ fn live_adapter_aggregate_forwards_graph_rag_smoke_controls() -> Result<()> { #[test] fn openmemory_ui_export_probe_has_dedicated_docker_task() -> Result<()> { let workspace_root = support::workspace_root()?; - let makefile = fs::read_to_string(workspace_root.join("Makefile.toml"))?; + let makefile = support::make_task_catalog()?; let docker_script = fs::read_to_string(workspace_root.join("scripts/baseline-docker.sh"))?; let compose = fs::read_to_string(workspace_root.join("docker-compose.baseline.yml"))?; let script = [ @@ -154,7 +154,7 @@ fn openmemory_ui_export_probe_has_dedicated_docker_task() -> Result<()> { #[test] fn operator_debug_live_adapter_task_is_docker_scoped() -> Result<()> { let workspace = support::workspace_root()?; - let makefile = fs::read_to_string(workspace.join("Makefile.toml"))?; + let makefile = support::make_task_catalog()?; let docker_script = fs::read_to_string(workspace.join("scripts/real-world-docker.sh"))?; let script = fs::read_to_string( workspace.join("scripts").join("real-world-operator-debug-live-adapters.sh"), diff --git a/apps/elf-eval/tests/real_world_job_benchmark/support.rs b/apps/elf-eval/tests/real_world_job_benchmark/support.rs index 8cf52509..0283e761 100644 --- a/apps/elf-eval/tests/real_world_job_benchmark/support.rs +++ b/apps/elf-eval/tests/real_world_job_benchmark/support.rs @@ -1,5 +1,6 @@ #[path = "support/contracts.rs"] mod contracts; #[path = "support/json.rs"] mod json; +#[path = "support/makefiles.rs"] mod makefiles; #[path = "support/paths.rs"] mod paths; #[path = "support/report_paths.rs"] mod report_paths; #[path = "support/runners.rs"] mod runners; @@ -9,6 +10,7 @@ pub(super) use self::{ json::{ array_at, array_contains_str, find_by_field, load_json, set_json_pointer, string_array_at, }, + makefiles::make_task_catalog, paths::{ adversarial_quality_fixture_dir, capture_fixture_dir, collapse_whitespace, consolidation_fixture_dir, context_trajectory_fixture_dir, diff --git a/apps/elf-eval/tests/real_world_job_benchmark/support/makefiles.rs b/apps/elf-eval/tests/real_world_job_benchmark/support/makefiles.rs new file mode 100644 index 00000000..8fd08a16 --- /dev/null +++ b/apps/elf-eval/tests/real_world_job_benchmark/support/makefiles.rs @@ -0,0 +1,28 @@ +use std::fs; + +use color_eyre::Result; + +pub(crate) fn make_task_catalog() -> Result { + let workspace = super::workspace_root()?; + let makefiles_dir = workspace.join("makefiles"); + let mut catalog = fs::read_to_string(workspace.join("Makefile.toml"))?; + + if makefiles_dir.is_dir() { + let mut paths = Vec::new(); + + for entry in fs::read_dir(makefiles_dir)? { + paths.push(entry?.path()); + } + + paths.sort(); + + for path in paths { + if path.extension().and_then(|ext| ext.to_str()) == Some("toml") { + catalog.push('\n'); + catalog.push_str(fs::read_to_string(path)?.as_str()); + } + } + } + + Ok(catalog) +} diff --git a/makefiles/benchmark-core.toml b/makefiles/benchmark-core.toml new file mode 100644 index 00000000..02c94349 --- /dev/null +++ b/makefiles/benchmark-core.toml @@ -0,0 +1,297 @@ +# Rust workspace tasks: Benchmark core, baseline, and operator tasks. + +# Rust workspace tasks: Benchmark. + +# Benchmark +# | task | type | cwd | +# | ------------------------------------------ | --------- | --- | +# | baseline-backfill-100k-docker | command | | +# | baseline-backfill-10k-docker | command | | +# | baseline-backfill-docker | command | | +# | baseline-live-docker | command | | +# | baseline-live-report | command | | +# | baseline-production-private | command | | +# | baseline-production-private-addendum | command | | +# | baseline-production-synthetic | command | | +# | baseline-soak-docker | command | | +# | local-agent-loop | command | | +# | openmemory-ui-export-readback | command | | +# | parity-docker | command | | +# | real-world-first-generation-oss | composite | | +# | real-world-first-generation-oss-json | command | | +# | real-world-first-generation-oss-report | command | | +# | real-world-job-operator-ux | composite | | +# | real-world-job-operator-ux-json | command | | +# | real-world-job-operator-ux-live-adapters | command | | +# | real-world-job-operator-ux-report | command | | +# | real-world-memory | composite | | +# | real-world-memory-adversarial-quality | composite | | +# | real-world-memory-adversarial-quality-json | command | | +# | real-world-memory-adversarial-quality-report | command | | +# | real-world-memory-consolidation | composite | | +# | real-world-memory-consolidation-json | command | | +# | real-world-memory-consolidation-report | command | | +# | real-world-memory-p1-closeout | composite | | +# | real-world-memory-p1-closeout-json | command | | +# | real-world-memory-p1-closeout-report | command | | +# | real-world-memory-p4-production-readiness | composite | | +# | real-world-memory-p4-production-readiness-json | command | | +# | real-world-memory-p4-production-readiness-report | command | | +# | real-world-memory-p4-quality-hardening-closeout | composite | | +# | real-world-memory-p2-knowledge-closeout | composite | | +# | real-world-memory-core-archival | composite | | +# | real-world-memory-core-archival-json | command | | +# | real-world-memory-core-archival-report | command | | +# | real-world-memory-context-trajectory | composite | | +# | real-world-memory-context-trajectory-json | command | | +# | real-world-memory-context-trajectory-report | command | | +# | real-world-memory-evolution | composite | | +# | real-world-memory-evolution-json | command | | +# | real-world-memory-evolution-report | command | | +# | real-world-memory-graph-rag | composite | | +# | real-world-memory-graph-rag-json | command | | +# | real-world-memory-graph-rag-report | command | | +# | real-world-memory-json | command | | +# | real-world-memory-knowledge | composite | | +# | real-world-memory-knowledge-json | command | | +# | real-world-memory-knowledge-report | command | | +# | real-world-memory-live-adapters | command | | +# | real-world-memory-live-consolidation | command | | +# | real-world-memory-live-knowledge | command | | +# | real-world-memory-mem0-openmemory-letta | composite | | +# | real-world-memory-mem0-openmemory-letta-json | command | | +# | real-world-memory-mem0-openmemory-letta-report | command | | +# | real-world-memory-pageindex-openkb | composite | | +# | real-world-memory-pageindex-openkb-json | command | | +# | real-world-memory-pageindex-openkb-report | command | | +# | real-world-memory-proactive-brief | composite | | +# | real-world-memory-proactive-brief-json | command | | +# | real-world-memory-proactive-brief-report | command | | +# | real-world-memory-production-ops | composite | | +# | real-world-memory-production-ops-json | command | | +# | real-world-memory-production-ops-report | command | | +# | real-world-memory-project-decisions | composite | | +# | real-world-memory-project-decisions-json | command | | +# | real-world-memory-project-decisions-report | command | | +# | real-world-memory-quantitative-scoreboard | composite | | +# | real-world-memory-quantitative-scoreboard-json | command | | +# | real-world-memory-quantitative-scoreboard-report | command | | +# | real-world-memory-report | command | | +# | real-world-memory-retrieval | composite | | +# | real-world-memory-retrieval-json | command | | +# | real-world-memory-retrieval-report | command | | +# | real-world-memory-scheduled | composite | | +# | real-world-memory-scheduled-json | command | | +# | real-world-memory-scheduled-report | command | | +# | real-world-memory-service-native-dreaming | command | | +# | real-world-memory-summary | composite | | +# | real-world-memory-summary-json | command | | +# | real-world-memory-summary-report | command | | +# | real-world-memory-work-continuity | composite | | +# | real-world-memory-work-continuity-json | command | | +# | real-world-memory-work-continuity-report | command | | + +[tasks.baseline-backfill-100k-docker] +workspace = false +command = "bash" +args = [ + "scripts/baseline-docker.sh", + "backfill-100k", +] + +[tasks.baseline-backfill-10k-docker] +workspace = false +command = "bash" +args = [ + "scripts/baseline-docker.sh", + "backfill-10k", +] + +[tasks.baseline-backfill-docker] +workspace = false +command = "bash" +args = [ + "scripts/baseline-docker.sh", + "backfill", +] + +[tasks.baseline-live-docker] +workspace = false +command = "bash" +args = [ + "scripts/baseline-docker.sh", + "live", +] + +[tasks.baseline-live-report] +workspace = false +command = "bash" +args = [ + "scripts/live-baseline-report-to-md.sh", +] + +[tasks.baseline-production-private] +workspace = false +command = "bash" +args = [ + "scripts/baseline-docker.sh", + "production-private", +] + +[tasks.baseline-production-private-addendum] +workspace = false +command = "bash" +args = [ + "scripts/baseline-docker.sh", + "production-private-addendum", +] + +[tasks.baseline-production-synthetic] +workspace = false +command = "bash" +args = [ + "scripts/baseline-docker.sh", + "production-synthetic", +] + +[tasks.baseline-soak-docker] +workspace = false +command = "bash" +args = [ + "scripts/baseline-docker.sh", + "soak", +] + +[tasks.local-agent-loop] +workspace = false +command = "bash" +args = [ + "scripts/local-agent-loop.sh", +] + +[tasks.openmemory-ui-export-readback] +workspace = false +command = "bash" +args = [ + "scripts/baseline-docker.sh", + "openmemory-ui-export-readback", +] + +[tasks.parity-docker] +workspace = false +command = "docker" +args = [ + "compose", + "-f", + "docker-compose.parity.yml", + "run", + "--build", + "--rm", + "parity-runner", +] + +[tasks.real-world-first-generation-oss] +workspace = false +dependencies = [ + "real-world-first-generation-oss-report", +] + +[tasks.real-world-first-generation-oss-json] +workspace = false +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "run", + "--fixtures", + "apps/elf-eval/fixtures/real_world_external_adapters/first_generation_oss", + "--out", + "tmp/real-world-memory/first-generation-oss/report.json", + "--run-id", + "first-generation-oss-continuity-source-store", + "--adapter-id", + "fixture_first_generation_oss", + "--adapter-name", + "First-generation OSS fixture coverage", +] + +[tasks.real-world-first-generation-oss-report] +workspace = false +dependencies = [ + "real-world-first-generation-oss-json", +] +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "publish", + "--report", + "tmp/real-world-memory/first-generation-oss/report.json", + "--out", + "tmp/real-world-memory/first-generation-oss/report.md", +] + +[tasks.real-world-job-operator-ux] +workspace = false +dependencies = [ + "real-world-job-operator-ux-report", +] + +[tasks.real-world-job-operator-ux-json] +workspace = false +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "run", + "--fixtures", + "apps/elf-eval/fixtures/real_world_job/operator_debugging_ux", + "--out", + "tmp/real-world-job/real-world-job-operator-ux-report.json", + "--run-id", + "real-world-job-operator-ux", + "--adapter-id", + "fixture_operator_ux", + "--adapter-name", + "ELF operator UX fixture", +] + +[tasks.real-world-job-operator-ux-live-adapters] +workspace = false +command = "bash" +args = [ + "scripts/real-world-docker.sh", + "job-operator-ux-live-adapters", +] + +[tasks.real-world-job-operator-ux-report] +workspace = false +dependencies = [ + "real-world-job-operator-ux-json", +] +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "publish", + "--report", + "tmp/real-world-job/real-world-job-operator-ux-report.json", + "--out", + "tmp/real-world-job/real-world-job-operator-ux-report.md", +] diff --git a/makefiles/benchmark-memory-a.toml b/makefiles/benchmark-memory-a.toml new file mode 100644 index 00000000..a7063ca4 --- /dev/null +++ b/makefiles/benchmark-memory-a.toml @@ -0,0 +1,414 @@ +# Rust workspace tasks: Benchmark real-world memory tasks, first half. + +[tasks.real-world-memory] +workspace = false +dependencies = [ + "real-world-memory-report", +] + +[tasks.real-world-memory-adversarial-quality] +workspace = false +dependencies = [ + "real-world-memory-adversarial-quality-report", +] + +[tasks.real-world-memory-adversarial-quality-json] +workspace = false +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "run", + "--fixtures", + "apps/elf-eval/fixtures/real_world_memory/adversarial_quality", + "--out", + "tmp/real-world-memory/adversarial-quality/report.json", + "--run-id", + "real-world-memory-adversarial-quality", + "--adapter-id", + "fixture_adversarial_quality", + "--adapter-name", + "ELF adversarial quality fixture", +] + +[tasks.real-world-memory-adversarial-quality-report] +workspace = false +dependencies = [ + "real-world-memory-adversarial-quality-json", +] +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "publish", + "--report", + "tmp/real-world-memory/adversarial-quality/report.json", + "--out", + "tmp/real-world-memory/adversarial-quality/report.md", +] + +[tasks.real-world-memory-consolidation] +workspace = false +dependencies = [ + "real-world-memory-consolidation-report", +] + +[tasks.real-world-memory-consolidation-json] +workspace = false +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "run", + "--fixtures", + "apps/elf-eval/fixtures/real_world_memory/consolidation", + "--out", + "tmp/real-world-memory/consolidation/report.json", + "--run-id", + "real-world-memory-consolidation", + "--adapter-id", + "fixture_consolidation", + "--adapter-name", + "ELF consolidation fixture", +] + +[tasks.real-world-memory-consolidation-report] +workspace = false +dependencies = [ + "real-world-memory-consolidation-json", +] +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "publish", + "--report", + "tmp/real-world-memory/consolidation/report.json", + "--out", + "tmp/real-world-memory/consolidation/report.md", +] + +[tasks.real-world-memory-p1-closeout] +workspace = false +dependencies = [ + "real-world-memory-p1-closeout-report", +] + +[tasks.real-world-memory-p1-closeout-json] +workspace = false +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "run", + "--fixtures", + "apps/elf-eval/fixtures/real_world_memory/p1_closeout", + "--out", + "tmp/real-world-memory/p1-closeout/report.json", + "--run-id", + "real-world-memory-p1-closeout", + "--adapter-id", + "fixture_p1_closeout", + "--adapter-name", + "ELF P1 closeout fixture", +] + +[tasks.real-world-memory-p1-closeout-report] +workspace = false +dependencies = [ + "real-world-memory-p1-closeout-json", +] +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "publish", + "--report", + "tmp/real-world-memory/p1-closeout/report.json", + "--out", + "tmp/real-world-memory/p1-closeout/report.md", +] + +[tasks.real-world-memory-p4-production-readiness] +workspace = false +dependencies = [ + "real-world-memory-p4-production-readiness-report", +] + +[tasks.real-world-memory-p4-production-readiness-json] +workspace = false +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "run", + "--fixtures", + "apps/elf-eval/fixtures/real_world_memory/production_ops", + "--out", + "tmp/real-world-memory/p4-production-readiness/report.json", + "--run-id", + "real-world-memory-p4-production-readiness", + "--adapter-id", + "fixture_production_ops", + "--adapter-name", + "ELF P4 production-readiness fixture", +] + +[tasks.real-world-memory-p4-production-readiness-report] +workspace = false +dependencies = [ + "real-world-memory-p4-production-readiness-json", +] +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "publish", + "--report", + "tmp/real-world-memory/p4-production-readiness/report.json", + "--out", + "tmp/real-world-memory/p4-production-readiness/report.md", +] + +[tasks.real-world-memory-p4-quality-hardening-closeout] +workspace = false +dependencies = [ + "real-world-memory-adversarial-quality-report", + "real-world-memory-p2-knowledge-closeout", + "real-world-memory-p4-production-readiness-report", +] + +[tasks.real-world-memory-p2-knowledge-closeout] +workspace = false +dependencies = [ + "real-world-memory-source-library-report", + "real-world-memory-knowledge-report", +] + +[tasks.real-world-memory-core-archival] +workspace = false +dependencies = [ + "real-world-memory-core-archival-report", +] + +[tasks.real-world-memory-core-archival-json] +workspace = false +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "run", + "--fixtures", + "apps/elf-eval/fixtures/real_world_memory/core_archival_memory", + "--out", + "tmp/real-world-memory/core-archival/report.json", + "--run-id", + "real-world-memory-core-archival", + "--adapter-id", + "fixture_core_archival_memory", + "--adapter-name", + "ELF core and archival memory fixture", +] + +[tasks.real-world-memory-core-archival-report] +workspace = false +dependencies = [ + "real-world-memory-core-archival-json", +] +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "publish", + "--report", + "tmp/real-world-memory/core-archival/report.json", + "--out", + "tmp/real-world-memory/core-archival/report.md", +] + +[tasks.real-world-memory-context-trajectory] +workspace = false +dependencies = [ + "real-world-memory-context-trajectory-report", +] + +[tasks.real-world-memory-context-trajectory-json] +workspace = false +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "run", + "--fixtures", + "apps/elf-eval/fixtures/real_world_memory/context_trajectory", + "--out", + "tmp/real-world-memory/context-trajectory/report.json", + "--run-id", + "real-world-memory-context-trajectory", + "--adapter-id", + "fixture_context_trajectory", + "--adapter-name", + "ELF context trajectory fixture", +] + +[tasks.real-world-memory-context-trajectory-report] +workspace = false +dependencies = [ + "real-world-memory-context-trajectory-json", +] +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "publish", + "--report", + "tmp/real-world-memory/context-trajectory/report.json", + "--out", + "tmp/real-world-memory/context-trajectory/report.md", +] + +[tasks.real-world-memory-evolution] +workspace = false +dependencies = [ + "real-world-memory-evolution-report", +] + +[tasks.real-world-memory-evolution-json] +workspace = false +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "run", + "--fixtures", + "apps/elf-eval/fixtures/real_world_memory/evolution", + "--out", + "tmp/real-world-memory/evolution-report.json", + "--run-id", + "real-world-memory-evolution", + "--adapter-id", + "fixture_memory_evolution", + "--adapter-name", + "ELF fixture memory evolution", +] + +[tasks.real-world-memory-evolution-report] +workspace = false +dependencies = [ + "real-world-memory-evolution-json", +] +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "publish", + "--report", + "tmp/real-world-memory/evolution-report.json", + "--out", + "tmp/real-world-memory/evolution-report.md", +] + +[tasks.real-world-memory-graph-rag] +workspace = false +dependencies = [ + "real-world-memory-graph-rag-report", +] + +[tasks.real-world-memory-graph-rag-json] +workspace = false +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "run", + "--fixtures", + "apps/elf-eval/fixtures/real_world_external_adapters/graph_rag", + "--out", + "tmp/real-world-memory/graph-rag/report.json", + "--run-id", + "real-world-memory-graph-rag", + "--adapter-id", + "fixture_graph_rag_external_adapters", + "--adapter-name", + "Graph/RAG representative external-adapter fixtures", +] + +[tasks.real-world-memory-graph-rag-report] +workspace = false +dependencies = [ + "real-world-memory-graph-rag-json", +] +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "publish", + "--report", + "tmp/real-world-memory/graph-rag/report.json", + "--out", + "tmp/real-world-memory/graph-rag/report.md", +] diff --git a/makefiles/benchmark-memory-b.toml b/makefiles/benchmark-memory-b.toml new file mode 100644 index 00000000..8657bb36 --- /dev/null +++ b/makefiles/benchmark-memory-b.toml @@ -0,0 +1,689 @@ +# Rust workspace tasks: Benchmark real-world memory tasks, second half. + +[tasks.real-world-memory-json] +workspace = false +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "run", + "--fixtures", + "apps/elf-eval/fixtures/real_world_memory", + "--out", + "tmp/real-world-memory/real-world-memory-report.json", + "--run-id", + "real-world-memory", + "--adapter-id", + "elf_real_world_memory_fixture", + "--adapter-name", + "ELF real-world memory fixture", +] + +[tasks.real-world-memory-knowledge] +workspace = false +dependencies = [ + "real-world-memory-knowledge-report", +] + +[tasks.real-world-memory-knowledge-json] +workspace = false +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "run", + "--fixtures", + "apps/elf-eval/fixtures/real_world_memory/knowledge", + "--out", + "tmp/real-world-memory/knowledge-report.json", + "--run-id", + "real-world-memory-knowledge", + "--adapter-id", + "fixture_knowledge", + "--adapter-name", + "ELF knowledge fixture", +] + +[tasks.real-world-memory-knowledge-report] +workspace = false +dependencies = [ + "real-world-memory-knowledge-json", +] +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "publish", + "--report", + "tmp/real-world-memory/knowledge-report.json", + "--out", + "tmp/real-world-memory/knowledge-report.md", +] + +[tasks.real-world-memory-mem0-openmemory-letta] +workspace = false +dependencies = [ + "real-world-memory-mem0-openmemory-letta-report", +] + +[tasks.real-world-memory-mem0-openmemory-letta-json] +workspace = false +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "run", + "--fixtures", + "apps/elf-eval/fixtures/real_world_external_adapters/mem0_openmemory_letta", + "--out", + "tmp/real-world-memory/mem0-openmemory-letta/report.json", + "--run-id", + "real-world-memory-mem0-openmemory-letta", + "--adapter-id", + "fixture_mem0_openmemory_letta", + "--adapter-name", + "mem0/OpenMemory and Letta memory-history/core-archive adapters", + "--adapter-behavior", + "same_corpus_adapter_fixture", + "--adapter-storage-status", + "blocked", + "--adapter-runtime-status", + "blocked", + "--adapter-notes", + "Offline fixtures map mem0 SDK history/export outputs to source ids and preserve OpenMemory UI/export plus Letta core/archive blockers until contained product exports map source ids.", +] + +[tasks.real-world-memory-mem0-openmemory-letta-report] +workspace = false +dependencies = [ + "real-world-memory-mem0-openmemory-letta-json", +] +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "publish", + "--report", + "tmp/real-world-memory/mem0-openmemory-letta/report.json", + "--out", + "tmp/real-world-memory/mem0-openmemory-letta/report.md", +] + +[tasks.real-world-memory-pageindex-openkb] +workspace = false +dependencies = [ + "real-world-memory-pageindex-openkb-report", +] + +[tasks.real-world-memory-pageindex-openkb-json] +workspace = false +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "run", + "--fixtures", + "apps/elf-eval/fixtures/real_world_external_adapters/pageindex_openkb", + "--out", + "tmp/real-world-memory/pageindex-openkb/report.json", + "--run-id", + "real-world-memory-pageindex-openkb", + "--adapter-id", + "fixture_pageindex_openkb_same_corpus", + "--adapter-name", + "PageIndex/OpenKB same-corpus adapter blockers", + "--adapter-behavior", + "typed_setup_blocker", + "--adapter-storage-status", + "blocked", + "--adapter-runtime-status", + "blocked", + "--adapter-notes", + "Offline fixtures preserve same-corpus PageIndex/OpenKB setup blockers until contained product artifacts map to ELF source ids.", +] + +[tasks.real-world-memory-pageindex-openkb-report] +workspace = false +dependencies = [ + "real-world-memory-pageindex-openkb-json", +] +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "publish", + "--report", + "tmp/real-world-memory/pageindex-openkb/report.json", + "--out", + "tmp/real-world-memory/pageindex-openkb/report.md", +] + +[tasks.real-world-memory-source-library] +workspace = false +dependencies = [ + "real-world-memory-source-library-report", +] + +[tasks.real-world-memory-source-library-json] +workspace = false +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "run", + "--fixtures", + "apps/elf-eval/fixtures/real_world_memory/source_library", + "--out", + "tmp/real-world-memory/source-library-report.json", + "--run-id", + "real-world-memory-source-library", + "--adapter-id", + "fixture_source_library", + "--adapter-name", + "ELF source library fixture", +] + +[tasks.real-world-memory-source-library-report] +workspace = false +dependencies = [ + "real-world-memory-source-library-json", +] +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "publish", + "--report", + "tmp/real-world-memory/source-library-report.json", + "--out", + "tmp/real-world-memory/source-library-report.md", +] + +[tasks.real-world-memory-live-adapters] +workspace = false +command = "bash" +args = [ + "scripts/real-world-docker.sh", + "memory-live-adapters", +] + +[tasks.real-world-memory-live-consolidation] +workspace = false +command = "bash" +args = [ + "scripts/real-world-docker.sh", + "memory-live-consolidation", +] + +[tasks.real-world-memory-live-knowledge] +workspace = false +command = "bash" +args = [ + "scripts/real-world-docker.sh", + "memory-live-knowledge", +] + +[tasks.real-world-memory-proactive-brief] +workspace = false +dependencies = [ + "real-world-memory-proactive-brief-report", +] + +[tasks.real-world-memory-proactive-brief-json] +workspace = false +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "run", + "--fixtures", + "apps/elf-eval/fixtures/real_world_memory/proactive_brief", + "--out", + "tmp/real-world-memory/proactive-brief/report.json", + "--run-id", + "real-world-memory-proactive-brief", + "--adapter-id", + "fixture_proactive_brief", + "--adapter-name", + "ELF proactive brief fixture", +] + +[tasks.real-world-memory-proactive-brief-report] +workspace = false +dependencies = [ + "real-world-memory-proactive-brief-json", +] +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "publish", + "--report", + "tmp/real-world-memory/proactive-brief/report.json", + "--out", + "tmp/real-world-memory/proactive-brief/report.md", +] + +[tasks.real-world-memory-production-ops] +workspace = false +dependencies = [ + "real-world-memory-production-ops-report", +] + +[tasks.real-world-memory-production-ops-json] +workspace = false +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "run", + "--fixtures", + "apps/elf-eval/fixtures/real_world_memory/production_ops", + "--run-id", + "real-world-memory-production-ops", + "--adapter-id", + "fixture_production_ops", + "--adapter-name", + "ELF production-ops fixture", + "--out", + "tmp/real-world-memory/production-ops-report.json", +] + +[tasks.real-world-memory-production-ops-report] +workspace = false +dependencies = [ + "real-world-memory-production-ops-json", +] +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "publish", + "--report", + "tmp/real-world-memory/production-ops-report.json", + "--out", + "tmp/real-world-memory/production-ops-report.md", +] + +[tasks.real-world-memory-project-decisions] +workspace = false +dependencies = [ + "real-world-memory-project-decisions-report", +] + +[tasks.real-world-memory-project-decisions-json] +workspace = false +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "run", + "--fixtures", + "apps/elf-eval/fixtures/real_world_memory/project_decisions", + "--out", + "tmp/real-world-memory/project-decisions/report.json", + "--run-id", + "real-world-memory-project-decisions", + "--adapter-id", + "fixture_project_decisions", + "--adapter-name", + "ELF project decision fixture", +] + +[tasks.real-world-memory-project-decisions-report] +workspace = false +dependencies = [ + "real-world-memory-project-decisions-json", +] +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "publish", + "--report", + "tmp/real-world-memory/project-decisions/report.json", + "--out", + "tmp/real-world-memory/project-decisions/report.md", +] + +[tasks.real-world-memory-quantitative-scoreboard] +workspace = false +dependencies = [ + "real-world-memory-quantitative-scoreboard-report", +] + +[tasks.real-world-memory-quantitative-scoreboard-json] +workspace = false +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "run", + "--fixtures", + "apps/elf-eval/fixtures/real_world_memory", + "--out", + "tmp/real-world-memory/quantitative-scoreboard/report.json", + "--run-id", + "public-quantitative-competitor-scoreboard", + "--adapter-id", + "elf_real_world_memory_fixture", + "--adapter-name", + "ELF real-world memory fixture", +] + +[tasks.real-world-memory-quantitative-scoreboard-report] +workspace = false +dependencies = [ + "real-world-memory-quantitative-scoreboard-json", +] +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "publish", + "--report", + "tmp/real-world-memory/quantitative-scoreboard/report.json", + "--out", + "tmp/real-world-memory/quantitative-scoreboard/report.md", +] + +[tasks.real-world-memory-report] +workspace = false +dependencies = [ + "real-world-memory-json", +] +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "publish", + "--report", + "tmp/real-world-memory/real-world-memory-report.json", + "--out", + "tmp/real-world-memory/real-world-memory-report.md", +] + +[tasks.real-world-memory-retrieval] +workspace = false +dependencies = [ + "real-world-memory-retrieval-report", +] + +[tasks.real-world-memory-retrieval-json] +workspace = false +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "run", + "--fixtures", + "apps/elf-eval/fixtures/real_world_memory/retrieval", + "--run-id", + "real-world-memory-retrieval", + "--adapter-id", + "fixture_retrieval", + "--adapter-name", + "ELF fixture retrieval cases", + "--out", + "tmp/real-world-memory/retrieval-report.json", +] + +[tasks.real-world-memory-retrieval-report] +workspace = false +dependencies = [ + "real-world-memory-retrieval-json", +] +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "publish", + "--report", + "tmp/real-world-memory/retrieval-report.json", + "--out", + "tmp/real-world-memory/retrieval-report.md", +] + +[tasks.real-world-memory-scheduled] +workspace = false +dependencies = [ + "real-world-memory-scheduled-report", +] + +[tasks.real-world-memory-scheduled-json] +workspace = false +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "run", + "--fixtures", + "apps/elf-eval/fixtures/real_world_memory/scheduled_memory", + "--out", + "tmp/real-world-memory/scheduled/report.json", + "--run-id", + "real-world-memory-scheduled", + "--adapter-id", + "fixture_scheduled_memory", + "--adapter-name", + "ELF scheduled memory fixture", +] + +[tasks.real-world-memory-scheduled-report] +workspace = false +dependencies = [ + "real-world-memory-scheduled-json", +] +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "publish", + "--report", + "tmp/real-world-memory/scheduled/report.json", + "--out", + "tmp/real-world-memory/scheduled/report.md", +] + +[tasks.real-world-memory-work-continuity] +workspace = false +dependencies = [ + "real-world-memory-work-continuity-report", +] + +[tasks.real-world-memory-work-continuity-json] +workspace = false +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "run", + "--fixtures", + "apps/elf-eval/fixtures/real_world_memory/work_continuity", + "--out", + "tmp/real-world-memory/work-continuity/report.json", + "--run-id", + "real-world-memory-work-continuity", + "--adapter-id", + "fixture_work_continuity", + "--adapter-name", + "ELF Work Continuity fixture", +] + +[tasks.real-world-memory-work-continuity-report] +workspace = false +dependencies = [ + "real-world-memory-work-continuity-json", +] +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "publish", + "--report", + "tmp/real-world-memory/work-continuity/report.json", + "--out", + "tmp/real-world-memory/work-continuity/report.md", +] + +[tasks.real-world-memory-service-native-dreaming] +workspace = false +command = "bash" +args = [ + "scripts/real-world-docker.sh", + "memory-service-native-dreaming", +] + +[tasks.real-world-memory-summary] +workspace = false +dependencies = [ + "real-world-memory-summary-report", +] + +[tasks.real-world-memory-summary-json] +workspace = false +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "run", + "--fixtures", + "apps/elf-eval/fixtures/real_world_memory/memory_summary", + "--out", + "tmp/real-world-memory/memory-summary/report.json", + "--run-id", + "real-world-memory-summary", + "--adapter-id", + "fixture_memory_summary", + "--adapter-name", + "ELF memory summary fixture", +] + +[tasks.real-world-memory-summary-report] +workspace = false +dependencies = [ + "real-world-memory-summary-json", +] +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "publish", + "--report", + "tmp/real-world-memory/memory-summary/report.json", + "--out", + "tmp/real-world-memory/memory-summary/report.md", +] + +# Check +# | task | type | cwd | +# | ---------------- | --------- | --- | +# | check | composite | | +# | check-docs | command | | +# | check-rust | command | | +# | check-trace-gate | command | | +# | checks | composite | | diff --git a/makefiles/check.toml b/makefiles/check.toml new file mode 100644 index 00000000..5756ac55 --- /dev/null +++ b/makefiles/check.toml @@ -0,0 +1,57 @@ +# Rust workspace tasks: Check. + +# Check +# | task | type | cwd | +# | ---------------- | --------- | --- | +# | check | composite | | +# | check-docs | command | | +# | check-rust | command | | +# | check-trace-gate | command | | +# | checks | composite | | + +[tasks.check] +clear = true +workspace = false +dependencies = [ + "fmt-check", + "check-docs", + "check-rust", + "lint", + "test", +] + +[tasks.check-docs] +workspace = false +command = "python3" +args = [ + "scripts/check-docs.py", +] + +[tasks.check-rust] +workspace = false +command = "cargo" +args = [ + "check", + "--workspace", + "--all-targets", + "--all-features", +] + +[tasks.check-trace-gate] +workspace = false +command = "bash" +args = [ + "scripts/trace-gate.sh", +] + +[tasks.checks] +workspace = false +dependencies = [ + "check", +] + +# Clean +# | task | type | cwd | +# | -------------------------- | ------- | --- | +# | clean-baseline-live-docker | command | | +# | clean-parity-docker | command | | diff --git a/makefiles/clean.toml b/makefiles/clean.toml new file mode 100644 index 00000000..7fc71c62 --- /dev/null +++ b/makefiles/clean.toml @@ -0,0 +1,41 @@ +# Rust workspace tasks: Clean. + +# Clean +# | task | type | cwd | +# | -------------------------- | ------- | --- | +# | clean-baseline-live-docker | command | | +# | clean-parity-docker | command | | + +[tasks.clean-baseline-live-docker] +workspace = false +command = "docker" +args = [ + "compose", + "-f", + "docker-compose.baseline.yml", + "down", + "-v", + "--remove-orphans", +] + +[tasks.clean-parity-docker] +workspace = false +command = "docker" +args = [ + "compose", + "-f", + "docker-compose.parity.yml", + "down", + "-v", + "--remove-orphans", +] + +# Format +# | task | type | cwd | +# | -------------- | --------- | --- | +# | fmt | composite | | +# | fmt-check | composite | | +# | fmt-rust | command | | +# | fmt-rust-check | extend | | +# | fmt-toml | command | | +# | fmt-toml-check | extend | | diff --git a/makefiles/format.toml b/makefiles/format.toml new file mode 100644 index 00000000..e214c216 --- /dev/null +++ b/makefiles/format.toml @@ -0,0 +1,54 @@ +# Rust workspace tasks: Format. + +# Format +# | task | type | cwd | +# | -------------- | --------- | --- | +# | fmt | composite | | +# | fmt-check | composite | | +# | fmt-rust | command | | +# | fmt-rust-check | extend | | +# | fmt-toml | command | | +# | fmt-toml-check | extend | | + +[tasks.fmt] +workspace = false +dependencies = [ + "fmt-rust", + "fmt-toml", +] + +[tasks.fmt-check] +workspace = false +dependencies = [ + "fmt-rust-check", + "fmt-toml-check", +] + +[tasks.fmt-rust] +workspace = false +script = "cargo +nightly fmt --all" + +[tasks.fmt-rust-check] +extend = "fmt-rust" +script = "cargo +nightly fmt --all -- --check" + +[tasks.fmt-toml] +workspace = false +command = "taplo" +args = [ + "fmt", +] + +[tasks.fmt-toml-check] +extend = "fmt-toml" +args = [ + "fmt", + "--check", +] + +# Lint +# | task | type | cwd | +# | ----------- | --------- | --- | +# | lint | composite | | +# | lint-rust | command | | +# | lint-vstyle | command | | diff --git a/makefiles/lint-fix.toml b/makefiles/lint-fix.toml new file mode 100644 index 00000000..5aada462 --- /dev/null +++ b/makefiles/lint-fix.toml @@ -0,0 +1,69 @@ +# Rust workspace tasks: Lint Fix. + +# Lint Fix +# | task | type | cwd | +# | --------------- | --------- | --- | +# | lint-fix | composite | | +# | lint-fix-rust | command | | +# | lint-fix-vstyle | command | | + +[tasks.lint-fix] +workspace = false +dependencies = [ + "lint-fix-rust", + "lint-fix-vstyle", +] + +[tasks.lint-fix-rust] +workspace = false +command = "cargo" +args = [ + "clippy", + "--fix", + "--allow-dirty", + "--all-features", + "--all-targets", + "--workspace", + "--", + "-D", + "clippy::all", + "-D", + "clippy::too_many_lines", + "-D", + "clippy::unwrap_used", + "-D", + "clippy::use_self", + "-D", + "clippy::wildcard_imports", + "-D", + "missing-docs", + "-D", + "unused-crate-dependencies", + "-D", + "warnings", +] + +[tasks.lint-fix-vstyle] +workspace = false +command = "cargo" +args = [ + "vstyle", + "tune", + "--language", + "rust", + "--workspace", + "--all-features", + "--strict", +] + +# Research +# | task | type | cwd | +# | --------------------------------------- | --------- | --- | +# | external-memory-radar | command | | +# | external-memory-radar-artifact | composite | | +# | external-memory-radar-artifact-json | command | | +# | external-memory-radar-artifact-validate | command | | +# | external-memory-radar-dry-run | composite | | +# | external-memory-radar-dry-run-json | command | | +# | external-memory-radar-dry-run-validate | command | | +# | external-memory-radar-validate | command | | diff --git a/makefiles/lint.toml b/makefiles/lint.toml new file mode 100644 index 00000000..1cedd668 --- /dev/null +++ b/makefiles/lint.toml @@ -0,0 +1,61 @@ +# Rust workspace tasks: Lint. + +# Lint +# | task | type | cwd | +# | ----------- | --------- | --- | +# | lint | composite | | +# | lint-rust | command | | +# | lint-vstyle | command | | + +[tasks.lint] +workspace = false +dependencies = [ + "lint-rust", + "lint-vstyle", +] + +[tasks.lint-rust] +workspace = false +command = "cargo" +args = [ + "clippy", + "--all-features", + "--all-targets", + "--workspace", + "--", + "-D", + "clippy::all", + "-D", + "clippy::too_many_lines", + "-D", + "clippy::unwrap_used", + "-D", + "clippy::use_self", + "-D", + "clippy::wildcard_imports", + "-D", + "missing-docs", + "-D", + "unused-crate-dependencies", + "-D", + "warnings", +] + +[tasks.lint-vstyle] +workspace = false +command = "cargo" +args = [ + "vstyle", + "curate", + "--language", + "rust", + "--workspace", + "--all-features", +] + +# Lint Fix +# | task | type | cwd | +# | --------------- | --------- | --- | +# | lint-fix | composite | | +# | lint-fix-rust | command | | +# | lint-fix-vstyle | command | | diff --git a/makefiles/research.toml b/makefiles/research.toml new file mode 100644 index 00000000..1c9db279 --- /dev/null +++ b/makefiles/research.toml @@ -0,0 +1,142 @@ +# Rust workspace tasks: Research. + +# Research +# | task | type | cwd | +# | --------------------------------------- | --------- | --- | +# | external-memory-radar | command | | +# | external-memory-radar-artifact | composite | | +# | external-memory-radar-artifact-json | command | | +# | external-memory-radar-artifact-validate | command | | +# | external-memory-radar-dry-run | composite | | +# | external-memory-radar-dry-run-json | command | | +# | external-memory-radar-dry-run-validate | command | | +# | external-memory-radar-validate | command | | + +[tasks.external-memory-radar] +workspace = false +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "external_memory_pattern_radar", + "--", + "run", + "--cursor", + "apps/elf-eval/fixtures/external_memory_pattern_radar/cursor.json", + "--summary", + "docs/evidence/external_memory_pattern_radar_latest.md", +] + +[tasks.external-memory-radar-artifact] +workspace = false +dependencies = [ + "external-memory-radar-artifact-json", + "external-memory-radar-artifact-validate", +] + +[tasks.external-memory-radar-artifact-json] +workspace = false +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "external_memory_pattern_radar", + "--", + "run", + "--cursor", + "apps/elf-eval/fixtures/external_memory_pattern_radar/cursor.json", + "--out-cursor", + "tmp/external-memory-pattern-radar/cursor.json", + "--summary", + "tmp/external-memory-pattern-radar/latest.md", +] + +[tasks.external-memory-radar-artifact-validate] +workspace = false +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "external_memory_pattern_radar", + "--", + "validate", + "--cursor", + "tmp/external-memory-pattern-radar/cursor.json", +] + +[tasks.external-memory-radar-dry-run] +workspace = false +dependencies = [ + "external-memory-radar-dry-run-json", + "external-memory-radar-dry-run-validate", +] + +[tasks.external-memory-radar-dry-run-json] +workspace = false +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "external_memory_pattern_radar", + "--", + "run", + "--mode", + "offline", + "--cursor", + "apps/elf-eval/fixtures/external_memory_pattern_radar/cursor.json", + "--out-cursor", + "tmp/external-memory-pattern-radar/cursor.json", + "--summary", + "tmp/external-memory-pattern-radar/latest.md", +] + +[tasks.external-memory-radar-dry-run-validate] +workspace = false +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "external_memory_pattern_radar", + "--", + "validate", + "--cursor", + "tmp/external-memory-pattern-radar/cursor.json", +] + +[tasks.external-memory-radar-validate] +workspace = false +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "external_memory_pattern_radar", + "--", + "validate", + "--cursor", + "apps/elf-eval/fixtures/external_memory_pattern_radar/cursor.json", +] + +# Smoke +# | task | type | cwd | +# | ---------------------------------- | --------- | --- | +# | smoke-graphify-docker-graph-report | command | | +# | smoke-graphiti-zep-docker-temporal | command | | +# | smoke-graphrag-docker | command | | +# | smoke-letta-core-archive-export-readback | command | | +# | smoke-lightrag-docker-context | command | | +# | smoke-ragflow-docker | command | | +# | smoke-real-world-job | composite | | +# | smoke-real-world-job-json | command | | +# | smoke-real-world-job-report | command | | diff --git a/makefiles/smoke.toml b/makefiles/smoke.toml new file mode 100644 index 00000000..88c4e494 --- /dev/null +++ b/makefiles/smoke.toml @@ -0,0 +1,113 @@ +# Rust workspace tasks: Smoke. + +# Smoke +# | task | type | cwd | +# | ---------------------------------- | --------- | --- | +# | smoke-graphify-docker-graph-report | command | | +# | smoke-graphiti-zep-docker-temporal | command | | +# | smoke-graphrag-docker | command | | +# | smoke-letta-core-archive-export-readback | command | | +# | smoke-lightrag-docker-context | command | | +# | smoke-ragflow-docker | command | | +# | smoke-real-world-job | composite | | +# | smoke-real-world-job-json | command | | +# | smoke-real-world-job-report | command | | + +[tasks.smoke-graphify-docker-graph-report] +workspace = false +command = "bash" +args = [ + "scripts/smoke-docker.sh", + "graphify-docker-graph-report", +] + +[tasks.smoke-graphiti-zep-docker-temporal] +workspace = false +command = "bash" +args = [ + "scripts/smoke-docker.sh", + "graphiti-zep-docker-temporal", +] + +[tasks.smoke-graphrag-docker] +workspace = false +command = "bash" +args = [ + "scripts/smoke-docker.sh", + "graphrag-docker", +] + +[tasks.smoke-letta-core-archive-export-readback] +workspace = false +command = "bash" +args = [ + "scripts/smoke-docker.sh", + "letta-core-archive-export-readback", +] + +[tasks.smoke-lightrag-docker-context] +workspace = false +command = "bash" +args = [ + "scripts/smoke-docker.sh", + "lightrag-docker-context", +] + +[tasks.smoke-ragflow-docker] +workspace = false +command = "bash" +args = [ + "scripts/ragflow-docker-evidence-smoke.sh", +] + +[tasks.smoke-real-world-job] +workspace = false +dependencies = [ + "smoke-real-world-job-report", +] + +[tasks.smoke-real-world-job-json] +workspace = false +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "run", + "--fixtures", + "apps/elf-eval/fixtures/real_world_memory/work_resume", + "--out", + "tmp/real-world-job/real-world-job-smoke-report.json", +] + +[tasks.smoke-real-world-job-report] +workspace = false +dependencies = [ + "smoke-real-world-job-json", +] +command = "cargo" +args = [ + "run", + "-p", + "elf-eval", + "--bin", + "real_world_job_benchmark", + "--", + "publish", + "--report", + "tmp/real-world-job/real-world-job-smoke-report.json", + "--out", + "tmp/real-world-job/real-world-job-smoke-report.md", +] + +# Test +# | task | type | cwd | +# | --------------------- | --------- | --- | +# | test | composite | | +# | test-e2e | command | | +# | test-rust | command | | +# | test-rust-all | command | | +# | test-rust-integration | command | | diff --git a/makefiles/test.toml b/makefiles/test.toml new file mode 100644 index 00000000..4245ab58 --- /dev/null +++ b/makefiles/test.toml @@ -0,0 +1,61 @@ +# Rust workspace tasks: Test. + +# Test +# | task | type | cwd | +# | --------------------- | --------- | --- | +# | test | composite | | +# | test-e2e | command | | +# | test-rust | command | | +# | test-rust-all | command | | +# | test-rust-integration | command | | + +[tasks.test] +clear = true +workspace = false +dependencies = [ + "test-rust", +] + +[tasks.test-e2e] +workspace = false +command = "bash" +args = [ + "scripts/context-misranking-harness.sh", +] + +[tasks.test-rust] +workspace = false +command = "cargo" +args = [ + "nextest", + "run", + "--workspace", + "--all-targets", + "--all-features", +] + +[tasks.test-rust-all] +workspace = false +command = "cargo" +args = [ + "nextest", + "run", + "--workspace", + "--all-targets", + "--all-features", + "--run-ignored", + "all", +] + +[tasks.test-rust-integration] +workspace = false +command = "cargo" +args = [ + "nextest", + "run", + "--workspace", + "--all-targets", + "--all-features", + "--run-ignored", + "only", +] diff --git a/scripts/check-docs.py b/scripts/check-docs.py index 9f64d34e..ba14f1f8 100755 --- a/scripts/check-docs.py +++ b/scripts/check-docs.py @@ -3,6 +3,7 @@ import re import sys +import tomllib from pathlib import Path @@ -16,7 +17,26 @@ def read_text(path: Path) -> str: return path.read_text(encoding="utf-8") +def makefile_task_names(path: Path, seen: set[Path] | None = None) -> set[str]: + seen = seen or set() + path = path.resolve() + if path in seen: + return set() + seen.add(path) + + data = tomllib.loads(read_text(path)) + tasks = set(data.get("tasks", {})) + for item in data.get("extend", []): + if not isinstance(item, dict) or not item.get("path"): + continue + tasks.update(makefile_task_names(path.parent / item["path"], seen)) + return tasks + + def cargo_make_tasks() -> set[str]: + tasks = makefile_task_names(ROOT / "Makefile.toml") + if tasks: + return tasks return set(TASK_RE.findall(read_text(ROOT / "Makefile.toml")))