From 1021572af97d7c9ca9f747c32f084fed9c9c57a7 Mon Sep 17 00:00:00 2001 From: initializ-mk Date: Sun, 27 Sep 2026 00:28:28 -0400 Subject: [PATCH] feat(memory): project identity and global namespace Introduce the foundation for a unified global memory store under ~/.forge/memory with two ownership namespaces: projects/ coding-session memory keyed by git remote agents/ deployed-agent operational memory keyed by agent_id - ProjectID: sha256(normalized git remote)[:12] with a local- fallback so equivalent remote URLs collapse and distinct local checkouts never do. AgentID normalizes forge.yaml agent_id to a filesystem-safe segment. - Registry: JSON map (projects.json / agents.json) id -> entry, unioning local paths across checkouts; atomic writes. - NewGlobalFileStore(root, namespace, id) with a single-segment path guard; NewFileStore retained for repo-local use. - MigrateRepoLocal: copies repo-local .forge/memory into agents//, mapping daily logs to sessions/, leaving the source intact with a .migrated sentinel (idempotent). - forge memory {migrate,projects,agents} CLI. - Add atomic-write helper (temp -> fsync -> rename) for all new writes. - Fix FileVectorStore.Search silently clamping k<=0 to 10, which capped keyword-only full scans. Tests: TestProjectID, TestAgentID, TestRegistry, TestMigrate, TestNormalizeRemote. --- forge-cli/cmd/memory.go | 140 +++++++++++++++++++++++ forge-core/memory/atomicwrite.go | 57 +++++++++ forge-core/memory/filestore.go | 33 +++++- forge-core/memory/migrate.go | 145 +++++++++++++++++++++++ forge-core/memory/migrate_test.go | 93 +++++++++++++++ forge-core/memory/project.go | 117 +++++++++++++++++++ forge-core/memory/project_test.go | 81 +++++++++++++ forge-core/memory/registry.go | 178 +++++++++++++++++++++++++++++ forge-core/memory/registry_test.go | 66 +++++++++++ forge-core/memory/vectorstore.go | 10 +- 10 files changed, 917 insertions(+), 3 deletions(-) create mode 100644 forge-cli/cmd/memory.go create mode 100644 forge-core/memory/atomicwrite.go create mode 100644 forge-core/memory/migrate.go create mode 100644 forge-core/memory/migrate_test.go create mode 100644 forge-core/memory/project.go create mode 100644 forge-core/memory/project_test.go create mode 100644 forge-core/memory/registry.go create mode 100644 forge-core/memory/registry_test.go diff --git a/forge-cli/cmd/memory.go b/forge-cli/cmd/memory.go new file mode 100644 index 00000000..4a8ce669 --- /dev/null +++ b/forge-cli/cmd/memory.go @@ -0,0 +1,140 @@ +package cmd + +import ( + "fmt" + "os" + "path/filepath" + "text/tabwriter" + + "github.com/spf13/cobra" + "gopkg.in/yaml.v3" + + "github.com/initializ/forge/forge-core/memory" +) + +var ( + memoryMigrateDryRun bool + memoryMigrateDir string +) + +// memoryRoot is the global memory root under ~/.forge. +func memoryRoot() string { return filepath.Join(forgeHome(), "memory") } + +var memoryCmd = &cobra.Command{ + Use: "memory", + Short: "Manage forge long-term memory", + Long: `Manage forge's global long-term memory store (~/.forge/memory). + +Memory is organized into two namespaces: + projects/ coding-session memory keyed by git remote + agents/ deployed-agent operational memory keyed by agent_id`, +} + +var memoryMigrateCmd = &cobra.Command{ + Use: "migrate", + Short: "Copy repo-local .forge/memory into the global agents namespace", + Long: `Copy a repo-local .forge/memory directory into the global agents +namespace at ~/.forge/memory/agents//. + +Daily logs (YYYY-MM-DD.md) move under sessions/. The source is left intact and +a .migrated sentinel is written so re-runs are no-ops. The agent id comes from +forge.yaml in the target directory.`, + RunE: func(cmd *cobra.Command, _ []string) error { + workDir := memoryMigrateDir + if workDir == "" { + workDir = "." + } + agentID := agentIDFromDir(workDir) + source := filepath.Join(workDir, ".forge", "memory") + + res, err := memory.MigrateRepoLocal(memoryRoot(), agentID, source, memoryMigrateDryRun) + if err != nil { + return err + } + + out := cmd.OutOrStdout() + if res.Skipped { + fmt.Fprintf(out, "skipped: %s (%s)\n", res.Source, res.SkipNote) + return nil + } + verb := "migrated" + if res.DryRun { + verb = "would migrate" + } + fmt.Fprintf(out, "%s %d file(s) from %s\n -> %s\n", verb, len(res.Files), res.Source, res.Dest) + for _, f := range res.Files { + fmt.Fprintf(out, " %s\n", f) + } + return nil + }, +} + +var memoryProjectsCmd = &cobra.Command{ + Use: "projects", + Short: "List coding projects tracked in global memory", + RunE: func(cmd *cobra.Command, _ []string) error { + return listRegistry(cmd, memory.NamespaceProjects) + }, +} + +var memoryAgentsCmd = &cobra.Command{ + Use: "agents", + Short: "List deployed agents tracked in global memory", + RunE: func(cmd *cobra.Command, _ []string) error { + return listRegistry(cmd, memory.NamespaceAgents) + }, +} + +func listRegistry(cmd *cobra.Command, namespace string) error { + reg, err := memory.OpenRegistry(memoryRoot(), namespace) + if err != nil { + return err + } + entries, err := reg.List() + if err != nil { + return err + } + if len(entries) == 0 { + fmt.Fprintf(cmd.OutOrStdout(), "no %s tracked in %s\n", namespace, memoryRoot()) + return nil + } + tw := tabwriter.NewWriter(cmd.OutOrStdout(), 0, 4, 2, ' ', 0) + fmt.Fprintln(tw, "ID\tSOURCE\tPATHS\tLAST SEEN") + for _, e := range entries { + last := "" + if !e.LastSeen.IsZero() { + last = e.LastSeen.Format("2006-01-02") + } + fmt.Fprintf(tw, "%s\t%s\t%d\t%s\n", e.ID, e.Source, len(e.LocalPaths), last) + } + return tw.Flush() +} + +// agentIDFromDir reads only the agent_id field from forge.yaml in dir, or "" +// when no config is found (the migrator then falls back to "unknown-agent"). +// It deliberately does a lightweight parse rather than a full ParseForgeConfig +// so a partial/invalid config still yields a usable agent id for migration. +func agentIDFromDir(dir string) string { + data, err := os.ReadFile(filepath.Join(dir, "forge.yaml")) + if err != nil { + return "" + } + var cfg struct { + AgentID string `yaml:"agent_id"` + } + if err := yaml.Unmarshal(data, &cfg); err != nil { + return "" + } + return cfg.AgentID +} + +func init() { + memoryMigrateCmd.Flags().BoolVar(&memoryMigrateDryRun, "dry-run", false, "show what would be migrated without writing") + memoryMigrateCmd.Flags().StringVar(&memoryMigrateDir, "dir", ".", "directory containing .forge/memory and forge.yaml") + + memoryCmd.AddCommand(memoryMigrateCmd) + memoryCmd.AddCommand(memoryProjectsCmd) + memoryCmd.AddCommand(memoryAgentsCmd) + + rootCmd.AddCommand(memoryCmd) +} diff --git a/forge-core/memory/atomicwrite.go b/forge-core/memory/atomicwrite.go new file mode 100644 index 00000000..82e2455a --- /dev/null +++ b/forge-core/memory/atomicwrite.go @@ -0,0 +1,57 @@ +package memory + +import ( + "fmt" + "os" + "path/filepath" +) + +// atomicWriteFile writes data to path atomically: it writes to a temp file in +// the same directory, fsyncs it, then renames it over the destination. The +// rename is atomic on POSIX filesystems, so a reader (or a crash mid-write) +// never observes a partially written file. The parent directory is fsynced +// best-effort so the rename itself is durable. +// +// All card, procedure, session, and registry writes go through this helper — +// the FileVectorStore flush is the one pre-existing temp→rename path (without +// fsync), and it is left as-is. +func atomicWriteFile(path string, data []byte, perm os.FileMode) error { + dir := filepath.Dir(path) + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("creating dir: %w", err) + } + + tmp, err := os.CreateTemp(dir, ".tmp-"+filepath.Base(path)+"-*") + if err != nil { + return fmt.Errorf("creating temp file: %w", err) + } + tmpName := tmp.Name() + // Best-effort cleanup if we bail before the rename succeeds. After a + // successful rename this simply fails silently (the name is gone). + defer func() { _ = os.Remove(tmpName) }() + + if _, err := tmp.Write(data); err != nil { + _ = tmp.Close() + return fmt.Errorf("writing temp file: %w", err) + } + if err := tmp.Sync(); err != nil { + _ = tmp.Close() + return fmt.Errorf("syncing temp file: %w", err) + } + if err := tmp.Close(); err != nil { + return fmt.Errorf("closing temp file: %w", err) + } + if err := os.Chmod(tmpName, perm); err != nil { + return fmt.Errorf("chmod temp file: %w", err) + } + if err := os.Rename(tmpName, path); err != nil { + return fmt.Errorf("renaming temp file: %w", err) + } + + // Best-effort: fsync the directory so the rename survives a crash. + if d, err := os.Open(dir); err == nil { + _ = d.Sync() + _ = d.Close() + } + return nil +} diff --git a/forge-core/memory/filestore.go b/forge-core/memory/filestore.go index f869bce1..df957f47 100644 --- a/forge-core/memory/filestore.go +++ b/forge-core/memory/filestore.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path/filepath" + "regexp" "strings" "time" ) @@ -15,7 +16,9 @@ type FileStore struct { dir string } -// NewFileStore creates a FileStore rooted at dir, creating it if needed. +// NewFileStore creates a FileStore rooted at dir, creating it if needed. This +// is the legacy / repo-local constructor; global-namespace callers should use +// NewGlobalFileStore so the id is validated as a single safe path segment. func NewFileStore(dir string) (*FileStore, error) { if err := os.MkdirAll(dir, 0o755); err != nil { return nil, fmt.Errorf("creating memory dir: %w", err) @@ -23,6 +26,34 @@ func NewFileStore(dir string) (*FileStore, error) { return &FileStore{dir: dir}, nil } +// safeSegmentRe matches a single filesystem-safe path segment: it must start +// with an alphanumeric and contain only lowercase alphanumerics, dot, dash, or +// underscore. ProjectID (hex or local-) and AgentID both satisfy it. +var safeSegmentRe = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]*$`) + +// safeSegment reports whether s is safe to use as a single path segment (no +// traversal, no separators). +func safeSegment(s string) bool { + if s == "" || len(s) > 128 || strings.Contains(s, "..") { + return false + } + return safeSegmentRe.MatchString(s) +} + +// NewGlobalFileStore creates a FileStore under the global memory root for a +// given namespace ("projects" or "agents") and id, i.e. rooted at +// ///. The id is validated to be a single safe path +// segment so a crafted id cannot escape the namespace. +func NewGlobalFileStore(root, namespace, id string) (*FileStore, error) { + if namespace != NamespaceProjects && namespace != NamespaceAgents { + return nil, fmt.Errorf("invalid namespace %q", namespace) + } + if !safeSegment(id) { + return nil, fmt.Errorf("invalid id segment %q", id) + } + return NewFileStore(filepath.Join(root, namespace, id)) +} + // Dir returns the root directory of the file store. func (fs *FileStore) Dir() string { return fs.dir } diff --git a/forge-core/memory/migrate.go b/forge-core/memory/migrate.go new file mode 100644 index 00000000..c6558afc --- /dev/null +++ b/forge-core/memory/migrate.go @@ -0,0 +1,145 @@ +package memory + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "time" +) + +// migratedSentinel is the marker written into a repo-local memory directory +// once its contents have been copied into the global namespace. Its presence +// makes migration idempotent and records where the copy went. +const migratedSentinel = ".migrated" + +// dailyLogRe matches the legacy daily-log filenames (YYYY-MM-DD.md) that the +// repo-local store wrote; these become episodic sessions under the global +// namespace. +var dailyLogRe = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}\.md$`) + +// MigrateResult reports what a migration did, or would do for a dry run. +type MigrateResult struct { + AgentID string `json:"agent_id"` + Source string `json:"source"` + Dest string `json:"dest"` + Files []string `json:"files"` // destination-relative paths (would be) copied + Skipped bool `json:"skipped"` + DryRun bool `json:"dry_run"` + SkipNote string `json:"skip_note,omitempty"` +} + +// sentinelData is the JSON payload written into the .migrated sentinel. +type sentinelData struct { + MigratedAt time.Time `json:"migrated_at"` + Dest string `json:"dest"` +} + +// MigrateRepoLocal copies a repo-local .forge/memory directory (sourceDir) into +// the global agents namespace at /agents//. Daily logs +// (YYYY-MM-DD.md) are placed under sessions/; MEMORY.md, index/, and any other +// .md files are copied at their relative position. The source is left intact — +// migration is a copy, not a move — and a .migrated sentinel recording the +// destination is written so a re-run is a no-op. +// +// When dryRun is true nothing is written; the returned result lists the files +// that would be copied. +func MigrateRepoLocal(root, agentID, sourceDir string, dryRun bool) (*MigrateResult, error) { + safeAgent := AgentID(agentID) + dest := filepath.Join(root, NamespaceAgents, safeAgent) + + res := &MigrateResult{ + AgentID: safeAgent, + Source: sourceDir, + Dest: dest, + DryRun: dryRun, + } + + info, err := os.Stat(sourceDir) + if err != nil || !info.IsDir() { + res.Skipped = true + res.SkipNote = "no repo-local memory directory" + return res, nil + } + + // Already migrated? The sentinel makes this idempotent. + if _, err := os.Stat(filepath.Join(sourceDir, migratedSentinel)); err == nil { + res.Skipped = true + res.SkipNote = "already migrated (.migrated sentinel present)" + return res, nil + } + + // Plan the copy: map each source file to its destination-relative path. + type copyOp struct{ src, relDest string } + var ops []copyOp + err = filepath.WalkDir(sourceDir, func(path string, d os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if d.IsDir() { + return nil + } + rel, err := filepath.Rel(sourceDir, path) + if err != nil { + return err + } + if rel == migratedSentinel { + return nil + } + relDest := rel + // Daily logs at the top level become episodic sessions. + if filepath.Dir(rel) == "." && dailyLogRe.MatchString(filepath.Base(rel)) { + relDest = filepath.Join("sessions", filepath.Base(rel)) + } + ops = append(ops, copyOp{src: path, relDest: relDest}) + return nil + }) + if err != nil { + return nil, fmt.Errorf("scanning source: %w", err) + } + + sort.Slice(ops, func(i, j int) bool { return ops[i].relDest < ops[j].relDest }) + for _, op := range ops { + res.Files = append(res.Files, op.relDest) + } + + if dryRun { + return res, nil + } + + for _, op := range ops { + data, err := os.ReadFile(op.src) + if err != nil { + return nil, fmt.Errorf("reading %s: %w", op.src, err) + } + if err := atomicWriteFile(filepath.Join(dest, op.relDest), data, 0o644); err != nil { + return nil, fmt.Errorf("writing %s: %w", op.relDest, err) + } + } + + // Record the migration in both the registry and a source-side sentinel. + reg, err := OpenRegistry(root, NamespaceAgents) + if err != nil { + return nil, err + } + if err := reg.Upsert(RegistryEntry{ + ID: safeAgent, + Source: agentID, + Name: agentID, + LocalPaths: []string{sourceDir}, + }); err != nil { + return nil, fmt.Errorf("updating registry: %w", err) + } + + sentinel, err := json.MarshalIndent(sentinelData{MigratedAt: time.Now().UTC(), Dest: dest}, "", " ") + if err != nil { + return nil, err + } + if err := atomicWriteFile(filepath.Join(sourceDir, migratedSentinel), sentinel, 0o644); err != nil { + return nil, fmt.Errorf("writing sentinel: %w", err) + } + + return res, nil +} diff --git a/forge-core/memory/migrate_test.go b/forge-core/memory/migrate_test.go new file mode 100644 index 00000000..416dfc58 --- /dev/null +++ b/forge-core/memory/migrate_test.go @@ -0,0 +1,93 @@ +package memory + +import ( + "os" + "path/filepath" + "testing" +) + +func mustWrite(t *testing.T, path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } +} + +func assertExists(t *testing.T, path string) { + t.Helper() + if _, err := os.Stat(path); err != nil { + t.Errorf("expected %s to exist: %v", path, err) + } +} + +func TestMigrate(t *testing.T) { + root := t.TempDir() + work := t.TempDir() + src := filepath.Join(work, ".forge", "memory") + mustWrite(t, filepath.Join(src, "MEMORY.md"), "# mem") + mustWrite(t, filepath.Join(src, "2026-09-01.md"), "log entry") + mustWrite(t, filepath.Join(src, "index", "index.json"), "[]") + + sessionRel := filepath.Join("sessions", "2026-09-01.md") + + // Dry run: reports files, writes nothing. + res, err := MigrateRepoLocal(root, "weather-agent", src, true) + if err != nil { + t.Fatal(err) + } + if !res.DryRun || len(res.Files) != 3 { + t.Fatalf("dry run: dryRun=%v files=%v", res.DryRun, res.Files) + } + if _, err := os.Stat(filepath.Join(root, NamespaceAgents)); err == nil { + t.Error("dry run wrote to destination") + } + found := false + for _, f := range res.Files { + if f == sessionRel { + found = true + } + } + if !found { + t.Errorf("daily log not mapped to sessions/: %v", res.Files) + } + + // Real run: copies files, maps daily log, leaves source intact. + if _, err := MigrateRepoLocal(root, "weather-agent", src, false); err != nil { + t.Fatal(err) + } + dest := filepath.Join(root, NamespaceAgents, "weather-agent") + assertExists(t, filepath.Join(dest, "MEMORY.md")) + assertExists(t, filepath.Join(dest, sessionRel)) + assertExists(t, filepath.Join(dest, "index", "index.json")) + assertExists(t, filepath.Join(src, migratedSentinel)) + assertExists(t, filepath.Join(src, "MEMORY.md")) // source preserved (copy, not move) + + reg, err := OpenRegistry(root, NamespaceAgents) + if err != nil { + t.Fatal(err) + } + if _, ok, _ := reg.Get("weather-agent"); !ok { + t.Error("registry not updated after migration") + } + + // Idempotent: the sentinel makes a second run a no-op. + res, err = MigrateRepoLocal(root, "weather-agent", src, false) + if err != nil { + t.Fatal(err) + } + if !res.Skipped { + t.Error("expected second run to be skipped") + } + + // Missing source directory is skipped, not an error. + res, err = MigrateRepoLocal(root, "x", filepath.Join(work, "does-not-exist"), false) + if err != nil { + t.Fatal(err) + } + if !res.Skipped { + t.Error("expected skip on missing source") + } +} diff --git a/forge-core/memory/project.go b/forge-core/memory/project.go new file mode 100644 index 00000000..abefd094 --- /dev/null +++ b/forge-core/memory/project.go @@ -0,0 +1,117 @@ +package memory + +import ( + "crypto/sha256" + "encoding/hex" + "os/exec" + "path/filepath" + "regexp" + "strings" +) + +// ProjectID returns the canonical identity for a coding project: the first 12 +// hex chars of sha256(normalized git remote). All the ways of naming the same +// repo (scp-style, https, ssh, with/without .git) collapse to one id, so two +// checkouts of the same remote share memory. +// +// When the worktree has no remote (or git is unavailable) it falls back to +// "local-" so unrelated local dirs never collapse. +func ProjectID(worktree string) string { + if remote := gitRemote(worktree); remote != "" { + return hashID(NormalizeRemote(remote)) + } + abs, err := filepath.Abs(worktree) + if err != nil || abs == "" { + abs = worktree + } + return "local-" + hashID(abs) +} + +// NormalizeRemote canonicalizes a git remote URL so equivalent forms map to one +// identity. Examples that all normalize to "github.com/initializ/forge": +// +// git@github.com:initializ/forge.git +// https://github.com/initializ/forge.git +// ssh://git@github.com:22/initializ/forge +// https://user:token@github.com/initializ/forge/ +func NormalizeRemote(remote string) string { + s := strings.ToLower(strings.TrimSpace(remote)) + + // Strip URL scheme (https://, ssh://, git://, ...). + if i := strings.Index(s, "://"); i >= 0 { + s = s[i+3:] + } + // Strip any user@ / user:token@ credentials prefix. + if i := strings.LastIndex(s, "@"); i >= 0 { + s = s[i+1:] + } + // Turn a host:path or host:port/path separator into a plain path separator. + if i := strings.IndexByte(s, ':'); i >= 0 { + host, rest := s[:i], s[i+1:] + if j := strings.IndexByte(rest, '/'); j >= 0 && isAllDigits(rest[:j]) { + // host:port/path → host/path + s = host + rest[j:] + } else { + // scp-style host:path → host/path + s = host + "/" + rest + } + } + + s = strings.TrimSuffix(s, "/") + s = strings.TrimSuffix(s, ".git") + s = strings.TrimSuffix(s, "/") + return s +} + +// agentIDUnsafe matches any run of characters that are not allowed in a +// filesystem-safe namespace segment. +var agentIDUnsafe = regexp.MustCompile(`[^a-z0-9._-]+`) + +// AgentID normalizes a forge.yaml agent_id into a filesystem-safe namespace +// segment (lowercase alphanumerics plus dot, dash, underscore). Empty or +// fully-invalid ids fall back to "unknown-agent". +func AgentID(raw string) string { + s := strings.ToLower(strings.TrimSpace(raw)) + s = agentIDUnsafe.ReplaceAllString(s, "-") + s = strings.Trim(s, "-._") + if s == "" { + return "unknown-agent" + } + if len(s) > 64 { + s = strings.Trim(s[:64], "-._") + if s == "" { + return "unknown-agent" + } + } + return s +} + +// gitRemote returns the origin remote URL of the git repo at worktree, or "" +// when there is none / git is unavailable. Best-effort. +func gitRemote(worktree string) string { + if worktree == "" { + return "" + } + out, err := exec.Command("git", "-C", worktree, "config", "--get", "remote.origin.url").Output() + if err != nil { + return "" + } + return strings.TrimSpace(string(out)) +} + +func hashID(s string) string { + sum := sha256.Sum256([]byte(s)) + return hex.EncodeToString(sum[:])[:12] +} + +func isAllDigits(s string) bool { + if s == "" { + return false + } + for i := 0; i < len(s); i++ { + if s[i] < '0' || s[i] > '9' { + return false + } + } + return true +} diff --git a/forge-core/memory/project_test.go b/forge-core/memory/project_test.go new file mode 100644 index 00000000..3e791a32 --- /dev/null +++ b/forge-core/memory/project_test.go @@ -0,0 +1,81 @@ +package memory + +import ( + "os/exec" + "strings" + "testing" +) + +func TestNormalizeRemote(t *testing.T) { + const want = "github.com/initializ/forge" + cases := []string{ + "git@github.com:initializ/forge.git", + "https://github.com/initializ/forge.git", + "https://github.com/initializ/forge", + "ssh://git@github.com/initializ/forge.git", + "ssh://git@github.com:22/initializ/forge", + "https://user:token@github.com/initializ/forge/", + "GIT@GitHub.com:initializ/Forge.git", + " git@github.com:initializ/forge.git ", + } + for _, in := range cases { + if got := NormalizeRemote(in); got != want { + t.Errorf("NormalizeRemote(%q) = %q, want %q", in, got, want) + } + } +} + +func TestProjectIDLocalFallback(t *testing.T) { + dir := t.TempDir() + id := ProjectID(dir) + if !strings.HasPrefix(id, "local-") { + t.Fatalf("expected local- prefix for non-git dir, got %q", id) + } + if got := ProjectID(dir); got != id { + t.Errorf("ProjectID not deterministic: %q vs %q", id, got) + } + if other := ProjectID(t.TempDir()); other == id { + t.Errorf("distinct dirs collapsed to same id %q", id) + } +} + +func TestProjectID(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not available") + } + dir := t.TempDir() + git := func(args ...string) { + t.Helper() + cmd := exec.Command("git", append([]string{"-C", dir}, args...)...) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } + } + git("init") + git("remote", "add", "origin", "git@github.com:initializ/forge.git") + idA := ProjectID(dir) + if len(idA) != 12 { + t.Errorf("expected 12-char id, got %q", idA) + } + // An equivalent remote URL must yield the same id. + git("remote", "set-url", "origin", "https://github.com/initializ/forge.git") + if idB := ProjectID(dir); idB != idA { + t.Errorf("equivalent remotes gave different ids: %q vs %q", idA, idB) + } +} + +func TestAgentID(t *testing.T) { + cases := map[string]string{ + "weather-agent": "weather-agent", + "Weather Agent": "weather-agent", + " My_Agent!! ": "my_agent", + "UPPER": "upper", + "": "unknown-agent", + "///": "unknown-agent", + } + for in, want := range cases { + if got := AgentID(in); got != want { + t.Errorf("AgentID(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/forge-core/memory/registry.go b/forge-core/memory/registry.go new file mode 100644 index 00000000..25edc566 --- /dev/null +++ b/forge-core/memory/registry.go @@ -0,0 +1,178 @@ +package memory + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sync" + "time" +) + +// Namespace names under the global memory root (~/.forge/memory). +const ( + // NamespaceProjects holds coding-session memory keyed by ProjectID + // (written by the optimizer proxy): the 3-tier card system. + NamespaceProjects = "projects" + // NamespaceAgents holds deployed-agent operational memory keyed by + // AgentID (written by the agent runtime). + NamespaceAgents = "agents" +) + +// RegistryEntry records the identity mapping for one project or agent. For +// projects, Source is the normalized git remote; for agents it is the raw +// forge.yaml agent_id. LocalPaths accumulates every worktree/workdir observed +// for this id so two checkouts of one remote collapse to a single entry. +type RegistryEntry struct { + ID string `json:"id"` + Source string `json:"source,omitempty"` + Name string `json:"name,omitempty"` + LocalPaths []string `json:"local_paths,omitempty"` + LastSeen time.Time `json:"last_seen"` +} + +// Registry is a JSON-file map of id → RegistryEntry, one file per namespace +// (projects.json / agents.json) under the global memory root. It is safe for +// concurrent use within a process; the underlying file is written atomically. +type Registry struct { + mu sync.Mutex + path string +} + +// OpenRegistry opens (creating on first write) the registry file for a +// namespace under root, e.g. /projects.json. +func OpenRegistry(root, namespace string) (*Registry, error) { + if namespace != NamespaceProjects && namespace != NamespaceAgents { + return nil, fmt.Errorf("invalid namespace %q", namespace) + } + if err := os.MkdirAll(root, 0o755); err != nil { + return nil, fmt.Errorf("creating memory root: %w", err) + } + return &Registry{path: filepath.Join(root, namespace+".json")}, nil +} + +// Upsert merges e into the registry: it unions LocalPaths with any existing +// entry, preserves a prior Name/Source when e leaves them blank, and stamps +// LastSeen with the current time when e leaves it zero. +func (r *Registry) Upsert(e RegistryEntry) error { + r.mu.Lock() + defer r.mu.Unlock() + + m, err := r.loadLocked() + if err != nil { + return err + } + + if existing, ok := m[e.ID]; ok { + e.LocalPaths = unionPaths(existing.LocalPaths, e.LocalPaths) + if e.Name == "" { + e.Name = existing.Name + } + if e.Source == "" { + e.Source = existing.Source + } + if e.LastSeen.IsZero() { + e.LastSeen = existing.LastSeen + } + } else { + e.LocalPaths = unionPaths(nil, e.LocalPaths) + } + if e.LastSeen.IsZero() { + e.LastSeen = time.Now().UTC() + } + + m[e.ID] = e + return r.saveLocked(m) +} + +// Get returns the entry for id and whether it was found. +func (r *Registry) Get(id string) (RegistryEntry, bool, error) { + r.mu.Lock() + defer r.mu.Unlock() + + m, err := r.loadLocked() + if err != nil { + return RegistryEntry{}, false, err + } + e, ok := m[id] + return e, ok, nil +} + +// List returns all entries, sorted by id for stable output. +func (r *Registry) List() ([]RegistryEntry, error) { + r.mu.Lock() + defer r.mu.Unlock() + + m, err := r.loadLocked() + if err != nil { + return nil, err + } + out := make([]RegistryEntry, 0, len(m)) + for _, e := range m { + out = append(out, e) + } + sortEntries(out) + return out, nil +} + +func (r *Registry) loadLocked() (map[string]RegistryEntry, error) { + data, err := os.ReadFile(r.path) + if os.IsNotExist(err) { + return map[string]RegistryEntry{}, nil + } + if err != nil { + return nil, fmt.Errorf("reading registry: %w", err) + } + m := map[string]RegistryEntry{} + if len(data) == 0 { + return m, nil + } + if err := json.Unmarshal(data, &m); err != nil { + return nil, fmt.Errorf("decoding registry %s: %w", r.path, err) + } + return m, nil +} + +func (r *Registry) saveLocked(m map[string]RegistryEntry) error { + // json.Marshal sorts string map keys, so output is deterministic. + data, err := json.MarshalIndent(m, "", " ") + if err != nil { + return fmt.Errorf("encoding registry: %w", err) + } + return atomicWriteFile(r.path, data, 0o644) +} + +// unionPaths returns the union of two path slices with order preserved +// (existing first, then any new paths) and duplicates removed. +func unionPaths(existing, add []string) []string { + seen := make(map[string]struct{}, len(existing)+len(add)) + var out []string + for _, p := range existing { + if p == "" { + continue + } + if _, ok := seen[p]; !ok { + seen[p] = struct{}{} + out = append(out, p) + } + } + for _, p := range add { + if p == "" { + continue + } + if _, ok := seen[p]; !ok { + seen[p] = struct{}{} + out = append(out, p) + } + } + return out +} + +// sortEntries sorts entries by ID in place. +func sortEntries(entries []RegistryEntry) { + for i := 1; i < len(entries); i++ { + for j := i; j > 0 && entries[j].ID < entries[j-1].ID; j-- { + entries[j], entries[j-1] = entries[j-1], entries[j] + } + } +} diff --git a/forge-core/memory/registry_test.go b/forge-core/memory/registry_test.go new file mode 100644 index 00000000..99e53237 --- /dev/null +++ b/forge-core/memory/registry_test.go @@ -0,0 +1,66 @@ +package memory + +import "testing" + +func TestRegistry(t *testing.T) { + root := t.TempDir() + + reg, err := OpenRegistry(root, NamespaceProjects) + if err != nil { + t.Fatal(err) + } + if list, err := reg.List(); err != nil || len(list) != 0 { + t.Fatalf("expected empty registry, got %v (err %v)", list, err) + } + + if err := reg.Upsert(RegistryEntry{ + ID: "abc123456789", + Source: "github.com/x/y", + Name: "y", + LocalPaths: []string{"/a"}, + }); err != nil { + t.Fatal(err) + } + + // Re-upsert with a blank source/name must preserve the originals and union paths. + if err := reg.Upsert(RegistryEntry{ + ID: "abc123456789", + LocalPaths: []string{"/b", "/a"}, + }); err != nil { + t.Fatal(err) + } + + e, ok, err := reg.Get("abc123456789") + if err != nil || !ok { + t.Fatalf("Get: ok=%v err=%v", ok, err) + } + if e.Source != "github.com/x/y" { + t.Errorf("source lost on merge: %q", e.Source) + } + if e.Name != "y" { + t.Errorf("name lost on merge: %q", e.Name) + } + if len(e.LocalPaths) != 2 { + t.Errorf("LocalPaths = %v, want 2 unioned", e.LocalPaths) + } + if e.LastSeen.IsZero() { + t.Error("LastSeen not stamped") + } + + // Reopen from disk: the entry must persist. + reg2, err := OpenRegistry(root, NamespaceProjects) + if err != nil { + t.Fatal(err) + } + list, err := reg2.List() + if err != nil { + t.Fatal(err) + } + if len(list) != 1 || list[0].ID != "abc123456789" { + t.Errorf("persistence failed: %v", list) + } + + if _, err := OpenRegistry(root, "bogus"); err == nil { + t.Error("expected error for invalid namespace") + } +} diff --git a/forge-core/memory/vectorstore.go b/forge-core/memory/vectorstore.go index af024640..89d81e66 100644 --- a/forge-core/memory/vectorstore.go +++ b/forge-core/memory/vectorstore.go @@ -82,8 +82,14 @@ func (s *FileVectorStore) Search(_ context.Context, queryVector []float32, k int s.mu.RLock() defer s.mu.RUnlock() - if k <= 0 { - k = 10 + // k <= 0 means "return every match" (a full scan), used by keyword-only + // callers that rank/filter downstream — previously this was silently + // clamped to 10, capping full scans. Cap to the corpus size instead. + if k <= 0 || k > len(s.chunks) { + k = len(s.chunks) + } + if k == 0 { + return nil, nil } // Compute similarities and maintain a top-k list.