From ecb0f44e4dca20aecc48a9d1a7eb51386c4a155e Mon Sep 17 00:00:00 2001 From: yujiezhang-ops Date: Fri, 9 Oct 2026 13:54:14 +0800 Subject: [PATCH 01/18] fix: stop reporting finished npm commands as stalled A command that exited 0 could still be reported as stalled: a process it left behind kept the output pipes open, so Wait blocked until the stall watchdog fired. Run now starts the command in its own process tree, stops reading the pipes a few seconds after the command exits, and treats a clean exit as success whatever the watchdog did meanwhile. npm prints nothing while it downloads, which the watchdog also read as a stall. Install and update now pass npm's cache and install directories as activity paths, and growth under them counts as progress. Co-Authored-By: Claude Opus 5 (1M context) --- internal/app/update.go | 4 +- internal/install/install.go | 4 +- internal/install/npm_activity.go | 247 ++++++++++++++++++ internal/install/npm_activity_test.go | 172 ++++++++++++ internal/install/npm_activity_windows_test.go | 29 ++ internal/process/activity.go | 96 +++++++ internal/process/activity_test.go | 66 +++++ internal/process/filestate_other.go | 16 ++ internal/process/filestate_windows.go | 46 ++++ internal/process/process.go | 60 ++++- internal/process/tree_unix.go | 55 ++++ internal/process/tree_unix_test.go | 160 ++++++++++++ internal/process/tree_windows.go | 91 +++++++ 13 files changed, 1040 insertions(+), 6 deletions(-) create mode 100644 internal/install/npm_activity.go create mode 100644 internal/install/npm_activity_test.go create mode 100644 internal/install/npm_activity_windows_test.go create mode 100644 internal/process/activity.go create mode 100644 internal/process/activity_test.go create mode 100644 internal/process/filestate_other.go create mode 100644 internal/process/filestate_windows.go create mode 100644 internal/process/tree_unix.go create mode 100644 internal/process/tree_unix_test.go create mode 100644 internal/process/tree_windows.go diff --git a/internal/app/update.go b/internal/app/update.go index 426c51cc..58c3e109 100644 --- a/internal/app/update.go +++ b/internal/app/update.go @@ -70,7 +70,9 @@ func (u *UseCases) UpdateAgent(ctx context.Context, agentID string, listeners .. } args = append(args, "--registry="+registry) } - result, err := runtime.Run(ctx, args, install.NPMEnvironment(runtime, npm, registry), install.DefaultCommandTimeout) + environment := install.NPMEnvironment(runtime, npm, registry) + runContext := process.WithActivityPaths(ctx, install.NPMActivityPaths(runtime, npm, agent.Package.Name, environment)...) + result, err := runtime.Run(runContext, args, environment, install.DefaultCommandTimeout) if err != nil { return AgentUpdateResult{}, oneerrors.New(oneerrors.InternalError, "Unable to update "+agent.Name, oneerrors.WithStatus(500), oneerrors.WithRetryable(true), oneerrors.WithCause(err)) } diff --git a/internal/install/install.go b/internal/install/install.go index a0f36a36..f18427ff 100644 --- a/internal/install/install.go +++ b/internal/install/install.go @@ -159,6 +159,7 @@ func InstallAgent(ctx context.Context, runtime Runtime, agent catalog.Agent, opt manager := packageInfo.Manager packageName := packageInfo.Name var argv []string + var activity []string environment := map[string]string{} switch manager { case "npm": @@ -189,6 +190,7 @@ func InstallAgent(ctx context.Context, runtime Runtime, agent catalog.Agent, opt argv = append(argv, "--registry="+registry) } argv = append(argv, spec) + activity = NPMActivityPaths(runtime, npm, packageName, environment) case "uv": uv, ok := runtime.Runner.LookPath("uv") if !ok || uv == "" { @@ -208,7 +210,7 @@ func InstallAgent(ctx context.Context, runtime Runtime, agent catalog.Agent, opt default: return Result{}, prerequisiteError(fmt.Sprintf("No allowlisted package manager for %s", agent.Name)) } - commandResult, runErr := runtime.command(ctx, argv, environment, runtime.timeout(options.Timeout)) + commandResult, runErr := runtime.command(process.WithActivityPaths(ctx, activity...), argv, environment, runtime.timeout(options.Timeout)) if runErr != nil { if isContextError(runErr) { return Result{}, oneerrors.New(oneerrors.Timeout, fmt.Sprintf("Installing %s timed out", agent.Name), oneerrors.WithRetryable(true), oneerrors.WithCause(runErr)) diff --git a/internal/install/npm_activity.go b/internal/install/npm_activity.go new file mode 100644 index 00000000..bdda1a97 --- /dev/null +++ b/internal/install/npm_activity.go @@ -0,0 +1,247 @@ +package install + +import ( + "bufio" + "io" + "os" + "path/filepath" + "regexp" + "strings" +) + +// NPMActivityPaths lists where a global npm install or update of packageName +// writes while it downloads, for the stall watchdog to treat as progress (see +// process.WithActivityPaths). +// +// Two places, both measured during a deliberately slow `npm update -g`: the +// package's directory under the global node_modules, which npm recreates after +// moving the old copy aside and fills as the tarball streams in, and the cache's +// tmp directory, where the tarball itself is staged byte by byte. npm printed +// nothing at all while both grew. +// +// Getting these right matters more than it looks: a stall now stops the whole +// install tree, so a download whose directories were missed would be cut off +// mid-way rather than merely misreported. The prefix and cache are therefore +// read from wherever npm reads them -- the environment and the npmrc files -- +// rather than assumed, and every plausible prefix is listed. Asking npm would +// cost a process per install; a candidate that does not exist is skipped by the +// watchdog for free, and either signal alone is enough. +func NPMActivityPaths(runtime Runtime, npm, packageName string, overrides map[string]string) []string { + packageName = strings.TrimSpace(packageName) + if !safePackagePath(packageName) { + return nil + } + lookup := func(name string) string { + for _, source := range []map[string]string{overrides, runtime.Env} { + for key, value := range source { + if strings.EqualFold(key, name) && strings.TrimSpace(value) != "" { + return strings.TrimSpace(value) + } + } + } + return "" + } + windows := runtime.Platform.OS == "windows" + home := runtime.Home + config := npmConfigFiles(npmrcLocations(runtime, npm, lookup), home, lookup) + + var prefixes []string + if prefix := lookup("npm_config_prefix"); prefix != "" { + prefixes = append(prefixes, prefix) + } + prefixes = append(prefixes, config.values("prefix")...) + if managedNPM(runtime, npm) { + prefixes = append(prefixes, GlobalPrefix(home)) + } + if root := npmNodeRoot(npm, windows); root != "" { + prefixes = append(prefixes, root) + } + if windows { + // The Node installer's default global prefix, which is not where npm.cmd + // lives (that is Program Files). + if appData := lookup("APPDATA"); appData != "" { + prefixes = append(prefixes, filepath.Join(appData, "npm")) + } + } + + paths := make([]string, 0, len(prefixes)+1) + seen := map[string]bool{} + add := func(path string) { + path = filepath.Clean(path) + // A relative candidate -- npm found on PATH as a bare name, an unexpanded + // variable -- would be walked from the working directory, which is not + // where npm writes. + if !filepath.IsAbs(path) { + return + } + if !seen[path] { + seen[path] = true + paths = append(paths, path) + } + } + for _, prefix := range prefixes { + root := filepath.Join(prefix, "lib", "node_modules") + if windows { + root = filepath.Join(prefix, "node_modules") + } + add(filepath.Join(root, filepath.FromSlash(packageName))) + } + + // Unlike the prefix, the cache is one place: the first source npm consults + // that sets it is the one npm writes to. + cache := lookup("npm_config_cache") + if cache == "" { + if values := config.values("cache"); len(values) > 0 { + cache = values[0] + } + } + if cache == "" { + if windows { + if local := lookup("LOCALAPPDATA"); local != "" { + cache = filepath.Join(local, "npm-cache") + } + } else if home != "" { + cache = filepath.Join(home, ".npm") + } + } + if cache != "" { + // Only tmp: the rest of the cache is every package ever fetched, which + // would make each sample slow and tell nothing about this download. + add(filepath.Join(cache, "_cacache", "tmp")) + } + return paths +} + +// npmNodeRoot is the directory npm runs from: /bin/npm on POSIX and +// \npm.cmd on Windows. A symlinked npm, as Homebrew installs it, is +// followed to the real installation. +func npmNodeRoot(npm string, windows bool) string { + if npm == "" { + return "" + } + if resolved, err := filepath.EvalSymlinks(npm); err == nil { + npm = resolved + } + dir := filepath.Dir(npm) + if windows { + return dir + } + // Following the link usually lands on npm's own script, in + // /lib/node_modules/npm/bin; checked first, because that directory is + // also called bin. + if packageBin := filepath.Join("lib", "node_modules", "npm", "bin"); strings.HasSuffix(dir, string(filepath.Separator)+packageBin) { + return strings.TrimSuffix(dir, string(filepath.Separator)+packageBin) + } + if filepath.Base(dir) == "bin" { + return filepath.Dir(dir) + } + return dir +} + +// npmrcLocations lists the config files npm reads for a global command, in +// the order it consults them: the user's, the global one under the Node +// install, and the npmrc shipped inside npm itself -- which is where the +// Windows installer puts its %APPDATA%\npm prefix. A project .npmrc is not +// read for a global command, so it is not listed. +func npmrcLocations(runtime Runtime, npm string, lookup func(string) string) []string { + windows := runtime.Platform.OS == "windows" + var files []string + if user := lookup("npm_config_userconfig"); user != "" { + files = append(files, user) + } else if runtime.Home != "" { + files = append(files, filepath.Join(runtime.Home, ".npmrc")) + } + root := npmNodeRoot(npm, windows) + if global := lookup("npm_config_globalconfig"); global != "" { + files = append(files, global) + } else if root != "" { + files = append(files, filepath.Join(root, "etc", "npmrc")) + } + if root != "" { + if windows { + files = append(files, filepath.Join(root, "node_modules", "npm", "npmrc")) + } else { + files = append(files, filepath.Join(root, "lib", "node_modules", "npm", "npmrc")) + } + } + return files +} + +type npmConfig [][2]string + +// values returns every setting of key across the files, in precedence order. +func (c npmConfig) values(key string) []string { + var found []string + for _, entry := range c { + if entry[0] == key { + found = append(found, entry[1]) + } + } + return found +} + +// maxNPMRCBytes bounds what is read of one npmrc. Real ones are a handful of +// lines; this only keeps a stray large file from being read on every install. +const maxNPMRCBytes = 64 << 10 + +var npmrcVariable = regexp.MustCompile(`\$\{([^}?]+)\??\}`) + +// npmConfigFiles reads prefix and cache out of the given npmrc files the way +// npm parses them: `key = value` lines, `;` and `#` comments, optional quotes, +// ${VAR} expansion and a leading ~ for the home directory. A file that is +// missing or unreadable contributes nothing. +func npmConfigFiles(files []string, home string, lookup func(string) string) npmConfig { + var config npmConfig + for _, path := range files { + file, err := os.Open(path) + if err != nil { + continue + } + scanner := bufio.NewScanner(io.LimitReader(file, maxNPMRCBytes)) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, ";") || strings.HasPrefix(line, "#") { + continue + } + key, value, ok := strings.Cut(line, "=") + if !ok { + continue + } + key = strings.ToLower(strings.TrimSpace(key)) + if key != "prefix" && key != "cache" { + continue + } + value = strings.TrimSpace(value) + if len(value) >= 2 && (value[0] == '"' || value[0] == '\'') && value[len(value)-1] == value[0] { + value = value[1 : len(value)-1] + } + value = npmrcVariable.ReplaceAllStringFunc(value, func(match string) string { + return lookup(npmrcVariable.FindStringSubmatch(match)[1]) + }) + if home != "" && (value == "~" || strings.HasPrefix(value, "~/") || strings.HasPrefix(value, `~\`)) { + value = filepath.Join(home, value[1:]) + } + if value != "" { + config = append(config, [2]string{key, value}) + } + } + _ = file.Close() + } + return config +} + +// safePackagePath accepts the name shapes npm allows, "name" and +// "@scope/name", so a manifest value can never turn the watched path into +// something outside the package's own directory. +func safePackagePath(name string) bool { + parts := strings.Split(name, "/") + if len(parts) > 2 || (len(parts) == 2 && !strings.HasPrefix(parts[0], "@")) { + return false + } + for _, part := range parts { + if part == "" || part == "." || part == ".." || part == "@" || strings.ContainsAny(part, `\:`) { + return false + } + } + return true +} diff --git a/internal/install/npm_activity_test.go b/internal/install/npm_activity_test.go new file mode 100644 index 00000000..3fd07f55 --- /dev/null +++ b/internal/install/npm_activity_test.go @@ -0,0 +1,172 @@ +package install + +import ( + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/MaimoryLab/BootAgent/internal/platform" +) + +// A managed npm's packages land under BootAgent's own prefix, and the cache is +// npm's default for the user. Both have to be watched: either one going quiet +// while the other grows is still a download in progress. +func TestNPMActivityPathsCoverTheManagedPrefixAndCache(t *testing.T) { + home := t.TempDir() + runtime := Runtime{Home: home, Platform: platform.For("darwin", "arm64"), Env: map[string]string{}} + npm := filepath.Join(RuntimeRoot(home), "node", "bin", "npm") + paths := NPMActivityPaths(runtime, npm, "@anthropic-ai/claude-code", NPMEnvironment(runtime, npm, "")) + for _, want := range []string{ + filepath.Join(GlobalPrefix(home), "lib", "node_modules", "@anthropic-ai", "claude-code"), + filepath.Join(home, ".npm", "_cacache", "tmp"), + } { + if !slices.Contains(paths, want) { + t.Errorf("paths = %v, want %s among them", paths, want) + } + } +} + +// On Windows the global node_modules sits directly under the prefix, the +// default prefix is %APPDATA%\npm rather than where npm.cmd lives, and the +// cache is under %LOCALAPPDATA%. An explicit npm_config_cache wins over the +// default, matched without regard to case as Windows environments are. +// +// Built from this host's absolute paths so it runs everywhere; what it pins is +// the Windows layout, which is keyed off Platform rather than the host. +func TestNPMActivityPathsFollowWindowsLayoutAndOverrides(t *testing.T) { + base := t.TempDir() + roaming := filepath.Join(base, "Roaming") + local := filepath.Join(base, "Local") + nodejs := filepath.Join(base, "nodejs") + runtime := Runtime{ + Home: base, + Platform: platform.For("windows", "amd64"), + Env: map[string]string{"APPDATA": roaming, "LOCALAPPDATA": local}, + } + npm := filepath.Join(nodejs, "npm.cmd") + paths := NPMActivityPaths(runtime, npm, "@anthropic-ai/claude-code", nil) + for _, want := range []string{ + filepath.Join(roaming, "npm", "node_modules", "@anthropic-ai", "claude-code"), + filepath.Join(nodejs, "node_modules", "@anthropic-ai", "claude-code"), + filepath.Join(local, "npm-cache", "_cacache", "tmp"), + } { + if !slices.Contains(paths, want) { + t.Errorf("paths = %v, want %s among them", paths, want) + } + } + if slices.Contains(paths, filepath.Join(nodejs, "lib", "node_modules", "@anthropic-ai", "claude-code")) { + t.Errorf("paths = %v: Windows has no lib/ under the prefix", paths) + } + + configured := filepath.Join(base, "custom-cache") + runtime.Env["NPM_CONFIG_CACHE"] = configured + paths = NPMActivityPaths(runtime, npm, "@anthropic-ai/claude-code", nil) + if !slices.Contains(paths, filepath.Join(configured, "_cacache", "tmp")) { + t.Errorf("paths = %v, want the configured cache", paths) + } + if slices.Contains(paths, filepath.Join(local, "npm-cache", "_cacache", "tmp")) { + t.Errorf("paths = %v: the default cache is not where an overridden npm writes", paths) + } +} + +// A prefix or cache set in an npmrc rather than the environment is where npm +// actually writes, so it has to be watched; missing it would let a stall stop a +// download that was progressing. Parsed as npm parses it: ${VAR}, a leading ~, +// quotes, and both comment styles. +func TestNPMActivityPathsReadThePrefixAndCacheFromNPMRC(t *testing.T) { + home := t.TempDir() + custom := filepath.Join(home, "elsewhere") + if err := os.WriteFile(filepath.Join(home, ".npmrc"), []byte(strings.Join([]string{ + "; npm's own comment style", + "# and the other one", + "registry=https://registry.example/", + "prefix = ${CUSTOM_ROOT}/npm-global", + `cache="~/my-cache"`, + }, "\n")), 0o600); err != nil { + t.Fatal(err) + } + runtime := Runtime{Home: home, Platform: platform.For("linux", "amd64"), Env: map[string]string{"CUSTOM_ROOT": custom}} + paths := NPMActivityPaths(runtime, "", "lodash", nil) + for _, want := range []string{ + filepath.Join(custom, "npm-global", "lib", "node_modules", "lodash"), + filepath.Join(home, "my-cache", "_cacache", "tmp"), + } { + if !slices.Contains(paths, want) { + t.Errorf("paths = %v, want %s among them", paths, want) + } + } + if slices.Contains(paths, filepath.Join(home, ".npm", "_cacache", "tmp")) { + t.Errorf("paths = %v: the default cache is not where an npmrc-configured npm writes", paths) + } +} + +// The environment outranks the npmrc for the cache, as it does in npm. +func TestNPMActivityPathsLetTheEnvironmentOutrankTheNPMRCCache(t *testing.T) { + home := t.TempDir() + if err := os.WriteFile(filepath.Join(home, ".npmrc"), []byte("cache=/from/npmrc\n"), 0o600); err != nil { + t.Fatal(err) + } + env := filepath.Join(home, "from-env") + runtime := Runtime{Home: home, Platform: platform.For("linux", "amd64"), Env: map[string]string{"npm_config_cache": env}} + paths := NPMActivityPaths(runtime, "", "lodash", nil) + if !slices.Contains(paths, filepath.Join(env, "_cacache", "tmp")) || slices.Contains(paths, filepath.Join("/from/npmrc", "_cacache", "tmp")) { + t.Errorf("paths = %v, want the environment's cache only", paths) + } +} + +// Homebrew links bin/npm to npm's own script in lib/node_modules/npm/bin. The +// link is followed, and that directory -- also named bin -- must not be taken +// for the prefix's bin, or the watched package directory would point inside +// npm's own package. +func TestNPMActivityPathsFollowASymlinkedNPMToItsInstallation(t *testing.T) { + root := t.TempDir() + script := filepath.Join(root, "lib", "node_modules", "npm", "bin", "npm-cli.js") + if err := os.MkdirAll(filepath.Dir(script), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(script, nil, 0o755); err != nil { + t.Fatal(err) + } + link := filepath.Join(root, "bin", "npm") + if err := os.MkdirAll(filepath.Dir(link), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(script, link); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + resolvedRoot, _ := filepath.EvalSymlinks(root) + runtime := Runtime{Home: t.TempDir(), Platform: platform.For("darwin", "arm64"), Env: map[string]string{}} + paths := NPMActivityPaths(runtime, link, "lodash", nil) + if want := filepath.Join(resolvedRoot, "lib", "node_modules", "lodash"); !slices.Contains(paths, want) { + t.Errorf("paths = %v, want %s", paths, want) + } + for _, path := range paths { + if strings.Contains(path, filepath.Join("node_modules", "npm", "lib")) { + t.Errorf("paths = %v: %s points inside npm's own package", paths, path) + } + } +} + +// npm found on PATH as a bare name, or a prefix variable that was never +// expanded, would otherwise be walked relative to the working directory. +func TestNPMActivityPathsDropRelativeCandidates(t *testing.T) { + runtime := Runtime{Home: "", Platform: platform.For("linux", "amd64"), Env: map[string]string{"npm_config_prefix": "relative/prefix"}} + for _, path := range NPMActivityPaths(runtime, "npm", "lodash", nil) { + if !filepath.IsAbs(path) { + t.Errorf("relative candidate %q was kept", path) + } + } +} + +// The package name comes from the manifest, but it becomes a path the watchdog +// walks, so anything that could leave the package's own directory is refused. +func TestNPMActivityPathsRefuseUnsafePackageNames(t *testing.T) { + runtime := Runtime{Home: t.TempDir(), Platform: platform.For("linux", "amd64"), Env: map[string]string{}} + for _, name := range []string{"", "..", "../etc", "@scope/../x", "a/b", "@scope/", "@/x", `@s\x`, "a/b/c"} { + if paths := NPMActivityPaths(runtime, "/usr/bin/npm", name, nil); paths != nil { + t.Errorf("NPMActivityPaths(%q) = %v, want nil", name, paths) + } + } +} diff --git a/internal/install/npm_activity_windows_test.go b/internal/install/npm_activity_windows_test.go new file mode 100644 index 00000000..f6972a14 --- /dev/null +++ b/internal/install/npm_activity_windows_test.go @@ -0,0 +1,29 @@ +package install + +import ( + "slices" + "testing" + + "github.com/MaimoryLab/BootAgent/internal/platform" +) + +// The same layout as TestNPMActivityPathsFollowWindowsLayoutAndOverrides, with +// the drive-letter paths a real Windows install has, which only a Windows host +// treats as absolute. +func TestNPMActivityPathsUseRealWindowsLocations(t *testing.T) { + runtime := Runtime{ + Home: `C:\Users\u`, + Platform: platform.For("windows", "amd64"), + Env: map[string]string{"APPDATA": `C:\Users\u\AppData\Roaming`, "LOCALAPPDATA": `C:\Users\u\AppData\Local`}, + } + paths := NPMActivityPaths(runtime, `C:\Program Files\nodejs\npm.cmd`, "@anthropic-ai/claude-code", nil) + for _, want := range []string{ + `C:\Users\u\AppData\Roaming\npm\node_modules\@anthropic-ai\claude-code`, + `C:\Program Files\nodejs\node_modules\@anthropic-ai\claude-code`, + `C:\Users\u\AppData\Local\npm-cache\_cacache\tmp`, + } { + if !slices.Contains(paths, want) { + t.Errorf("paths = %v, want %s among them", paths, want) + } + } +} diff --git a/internal/process/activity.go b/internal/process/activity.go new file mode 100644 index 00000000..1e9f6722 --- /dev/null +++ b/internal/process/activity.go @@ -0,0 +1,96 @@ +package process + +import ( + "context" + "io/fs" + "path/filepath" + "strings" +) + +// maxActivityEntries bounds one path's share of a sample. The directories a +// caller names are an install's own output and cache staging area, a few +// thousand entries at most; the cap only guards a mistaken path that points +// somewhere enormous. It is per path, so one large tree cannot stop the others +// from being sampled -- the cache staging area is usually listed last and is +// the signal that matters most. The walk order is lexical, so a capped sample +// still compares consistently. +const maxActivityEntries = 20000 + +type activityPathsKey struct{} + +// WithActivityPaths names directories whose growth counts as a command being +// alive, alongside its output. +// +// Output alone cannot tell a slow download from a hung one. npm prints nothing +// on a non-TTY while it fetches a tarball -- measured: zero bytes of output for +// the whole of a slow global install, while the package directory and npm's +// cache staging area grew the entire time. Claude Code's Windows binary is a +// 111 MB tarball, so a slow link spent minutes in that silence, past the stall +// timeout, and a download that was progressing was treated as hung. +// +// The paths travel on the context rather than through Runner so every decorator +// between the caller and the runner passes them along unchanged. +func WithActivityPaths(ctx context.Context, paths ...string) context.Context { + cleaned := make([]string, 0, len(paths)) + for _, path := range paths { + if path = strings.TrimSpace(path); path != "" { + cleaned = append(cleaned, filepath.Clean(path)) + } + } + if len(cleaned) == 0 { + return ctx + } + return context.WithValue(ctx, activityPathsKey{}, append(activityPaths(ctx), cleaned...)) +} + +func activityPaths(ctx context.Context) []string { + paths, _ := ctx.Value(activityPathsKey{}).([]string) + return append([]string(nil), paths...) +} + +// pathSignature summarises the watched trees. Any change between two samples is +// progress: a new entry, a file that grew, or a newer write time. A missing path +// contributes nothing, so a directory that only appears mid-install is itself a +// change. +type pathSignature struct { + entries int + bytes int64 + newest int64 +} + +func samplePaths(paths []string, done <-chan struct{}) pathSignature { + var signature pathSignature + for _, root := range paths { + walked := 0 + _ = filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error { + select { + case <-done: + return filepath.SkipAll + default: + } + if err != nil { + // A file renamed or removed between listing and stat is ordinary + // while npm moves packages around; skip it, keep walking. + if entry != nil && entry.IsDir() { + return filepath.SkipDir + } + return nil + } + if walked >= maxActivityEntries { + return filepath.SkipAll + } + walked++ + signature.entries++ + if entry.Type()&fs.ModeSymlink != 0 { + // Never followed: a link out of the tree could reach anything. + return nil + } + if size, modified, ok := liveFileState(path, entry); ok { + signature.bytes += size + signature.newest = max(signature.newest, modified) + } + return nil + }) + } + return signature +} diff --git a/internal/process/activity_test.go b/internal/process/activity_test.go new file mode 100644 index 00000000..c44bab3c --- /dev/null +++ b/internal/process/activity_test.go @@ -0,0 +1,66 @@ +package process + +import ( + "context" + "errors" + "os" + "path/filepath" + "runtime" + "testing" + "time" +) + +// A command that prints nothing but keeps writing under a watched path is +// working, not hung -- the shape of npm fetching a large tarball. With only +// output to go on, the watchdog stopped it at the stall timeout. +func TestOSRunnerTreatsGrowthUnderAWatchedPathAsActivity(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("uses a POSIX shell") + } + dir := t.TempDir() + target := filepath.Join(dir, "download.part") + ctx := WithActivityPaths(context.Background(), dir) + runner := OSRunner{StallTimeout: 400 * time.Millisecond} + // Silent for 1.5s -- several stall timeouts -- while the file grows. + script := `i=0; while [ $i -lt 15 ]; do printf x >> "$1"; sleep 0.1; i=$((i+1)); done` + result, err := runner.Run(ctx, []string{"/bin/sh", "-c", script, "sh", target}, nil, time.Minute) + if err != nil { + t.Fatalf("Run() error = %v, want nil: the file grew the whole time", err) + } + if result.ExitCode != 0 { + t.Fatalf("exit code = %d", result.ExitCode) + } +} + +// Watching a path must not keep a genuinely hung command alive. +func TestOSRunnerStillStopsASilentCommandWhoseWatchedPathIsIdle(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("uses a POSIX shell") + } + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "existing"), []byte("left by an earlier install"), 0o600); err != nil { + t.Fatal(err) + } + ctx := WithActivityPaths(context.Background(), dir, filepath.Join(dir, "does-not-exist")) + runner := OSRunner{StallTimeout: 300 * time.Millisecond} + started := time.Now() + _, err := runner.Run(ctx, []string{"/bin/sh", "-c", "sleep 30"}, nil, time.Minute) + if !errors.Is(err, ErrStalled) { + t.Fatalf("Run() error = %v, want ErrStalled", err) + } + if elapsed := time.Since(started); elapsed > 3*time.Second { + t.Errorf("stall took %s to detect", elapsed.Round(time.Millisecond)) + } +} + +func TestWithActivityPathsAccumulatesAndIgnoresBlanks(t *testing.T) { + ctx := WithActivityPaths(context.Background(), " ", "") + if got := activityPaths(ctx); len(got) != 0 { + t.Fatalf("blank paths = %v, want none", got) + } + ctx = WithActivityPaths(ctx, "/a") + ctx = WithActivityPaths(ctx, "/b/../c") + if got := activityPaths(ctx); len(got) != 2 || got[0] != filepath.Clean("/a") || got[1] != filepath.Clean("/c") { + t.Fatalf("paths = %v, want [/a /c]", got) + } +} diff --git a/internal/process/filestate_other.go b/internal/process/filestate_other.go new file mode 100644 index 00000000..3cd30a9e --- /dev/null +++ b/internal/process/filestate_other.go @@ -0,0 +1,16 @@ +//go:build !windows + +package process + +import "io/fs" + +// liveFileState reports a file's size and write time. Outside Windows the +// directory entry's metadata is the inode's, so it is current even while +// another process is still writing the file. +func liveFileState(_ string, entry fs.DirEntry) (int64, int64, bool) { + info, err := entry.Info() + if err != nil { + return 0, 0, false + } + return info.Size(), info.ModTime().UnixNano(), true +} diff --git a/internal/process/filestate_windows.go b/internal/process/filestate_windows.go new file mode 100644 index 00000000..2d50fdc7 --- /dev/null +++ b/internal/process/filestate_windows.go @@ -0,0 +1,46 @@ +package process + +import ( + "io/fs" + + "golang.org/x/sys/windows" +) + +// liveFileState reports a file's size and write time as the file system holds +// them now. +// +// Directory enumeration is not enough on Windows: NTFS updates the size and +// write time in a directory entry lazily, so a file another process is still +// writing can be listed at its old size until that process closes it. A +// tarball holding one large binary would then look idle for its whole +// download. Querying an open handle reads the live values instead. +// +// The handle asks for attributes only and shares read, write and delete, so it +// cannot cause a sharing violation for the installer or stop it renaming or +// removing the file -- npm moves whole package directories during an update. +func liveFileState(path string, entry fs.DirEntry) (int64, int64, bool) { + if entry.IsDir() { + info, err := entry.Info() + if err != nil { + return 0, 0, false + } + return 0, info.ModTime().UnixNano(), true + } + name, err := windows.UTF16PtrFromString(path) + if err != nil { + return 0, 0, false + } + handle, err := windows.CreateFile(name, windows.FILE_READ_ATTRIBUTES, + windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE|windows.FILE_SHARE_DELETE, + nil, windows.OPEN_EXISTING, windows.FILE_FLAG_OPEN_REPARSE_POINT, 0) + if err != nil { + return 0, 0, false + } + defer windows.CloseHandle(handle) + var info windows.ByHandleFileInformation + if err := windows.GetFileInformationByHandle(handle, &info); err != nil { + return 0, 0, false + } + size := int64(info.FileSizeHigh)<<32 | int64(info.FileSizeLow) + return size, info.LastWriteTime.Nanoseconds(), true +} diff --git a/internal/process/process.go b/internal/process/process.go index 0e284af0..2f8763ea 100644 --- a/internal/process/process.go +++ b/internal/process/process.go @@ -319,6 +319,23 @@ type OSRunner struct { // negative value disables stall detection. Tests set it so they can assert // the behaviour without waiting minutes. StallTimeout time.Duration + // PipeDrainDelay overrides DefaultPipeDrainDelay. Zero means the default. + PipeDrainDelay time.Duration +} + +// DefaultPipeDrainDelay is how long Run keeps reading a command's output after +// the command itself has exited. Output still in flight arrives in +// milliseconds; anything later is a process the command left behind holding +// the pipes, and waiting for it would tie the result to that process. It has to +// stay well under DefaultStallTimeout, or a finished command would still be +// reported as stalled. +const DefaultPipeDrainDelay = 5 * time.Second + +func (r OSRunner) pipeDrainDelay() time.Duration { + if r.PipeDrainDelay > 0 { + return r.PipeDrainDelay + } + return DefaultPipeDrainDelay } func Current() OSRunner { @@ -508,14 +525,36 @@ func (r OSRunner) runWithOutput(ctx context.Context, argv []string, overrides ma activity.touch() command.Stdout = &streamWriter{stream: "stdout", buffer: stdout, listener: listener, mu: &streamLock, activity: activity} command.Stderr = &streamWriter{stream: "stderr", buffer: stderr, listener: listener, mu: &streamLock, activity: activity} - stalled := watchForStall(runContext, activity, r.stallTimeout(), stopForStall) - err := command.Run() + tree := newProcessTree(command) + defer tree.release() + // Wait otherwise reads the pipes until every holder closes them. A process + // the command started and left running inherits the pipes, so a command + // that had already exited 0 would keep Run waiting on that process, and a + // long enough wait reached the stall watchdog, which then reported the + // finished command as stalled. + command.WaitDelay = r.pipeDrainDelay() + stalled := watchForStall(runContext, activity, r.stallTimeout(), stopForStall, activityPaths(ctx)) + err := command.Start() + if err == nil { + tree.attach() + err = command.Wait() + } stalled.stop() result.Stdout = stdout.String() result.Stderr = stderr.String() if command.ProcessState != nil { result.ExitCode = command.ProcessState.ExitCode() } + // A command that exited 0 before anything cancelled it succeeded, whatever + // the watchdog did while its leftover pipes drained: exec does not call + // Cancel once it has seen the exit, so nothing was killed and nothing is + // lost. Checked before the stall, which would otherwise call a finished + // npm update stalled -- the reported failure -- whenever the drain delay + // happened to straddle the stall timeout. Err is nil or ErrWaitDelay only + // when Cancel did not run, so a command that was stopped never reaches this. + if command.ProcessState != nil && command.ProcessState.Success() && (err == nil || errors.Is(err, exec.ErrWaitDelay)) { + return result, nil + } // Reported ahead of the generic "killed" error, because the caller cannot // otherwise distinguish a stall from a user cancellation: both arrive as a // cancelled context on the same ctx. The captured output goes back with it -- @@ -579,8 +618,9 @@ func (w *stallWatch) tripped() bool { } // watchForStall cancels a command that has produced nothing for timeout. A -// zero or negative timeout disables the watchdog. -func watchForStall(ctx context.Context, activity *activityClock, timeout time.Duration, cancel context.CancelFunc) *stallWatch { +// zero or negative timeout disables the watchdog. Growth under paths counts as +// activity too, for commands that work in silence; see WithActivityPaths. +func watchForStall(ctx context.Context, activity *activityClock, timeout time.Duration, cancel context.CancelFunc, paths []string) *stallWatch { watch := &stallWatch{done: make(chan struct{}), fired: make(chan struct{})} if timeout <= 0 { return watch @@ -589,6 +629,12 @@ func watchForStall(ctx context.Context, activity *activityClock, timeout time.Du go func() { ticker := time.NewTicker(interval) defer ticker.Stop() + // The baseline is what was on disk before the command started, so files + // a previous install left behind are not mistaken for progress. + var last pathSignature + if len(paths) > 0 { + last = samplePaths(paths, watch.done) + } for { select { case <-watch.done: @@ -596,6 +642,12 @@ func watchForStall(ctx context.Context, activity *activityClock, timeout time.Du case <-ctx.Done(): return case <-ticker.C: + if len(paths) > 0 { + if current := samplePaths(paths, watch.done); current != last { + last = current + activity.touch() + } + } if activity.idle() < timeout { continue } diff --git a/internal/process/tree_unix.go b/internal/process/tree_unix.go new file mode 100644 index 00000000..65ba30ea --- /dev/null +++ b/internal/process/tree_unix.go @@ -0,0 +1,55 @@ +//go:build unix + +package process + +import ( + "errors" + "os" + "os/exec" + "syscall" +) + +// processTree makes cancellation reach every process a command starts, not only +// the one exec started. The command leads its own process group, so the group +// ID is its PID, and a cancelled command is killed as a group. +// +// Without this a cancelled `sh -c 'x & wait'`, or an npm whose postinstall +// script forked, loses only its leader: the orphans keep running and keep the +// stdout pipe open, so Wait blocks until they finish on their own. +type processTree struct { + command *exec.Cmd +} + +func newProcessTree(command *exec.Cmd) *processTree { + if command.SysProcAttr == nil { + command.SysProcAttr = &syscall.SysProcAttr{} + } + command.SysProcAttr.Setpgid = true + tree := &processTree{command: command} + command.Cancel = tree.terminate + return tree +} + +// attach has nothing to do here: the group exists from the moment Start forks. +func (t *processTree) attach() {} + +func (t *processTree) release() {} + +func (t *processTree) terminate() error { + process := t.command.Process + if process == nil { + return os.ErrProcessDone + } + err := syscall.Kill(-process.Pid, syscall.SIGKILL) + if errors.Is(err, syscall.ESRCH) { + // The whole group is already gone, which Wait reads as "exited on its + // own" rather than as a failed cancellation. + return os.ErrProcessDone + } + if err != nil { + // A group the kernel would not signal still leaves the leader, which is + // what the default Cancel kills. + return process.Kill() + } + return nil +} diff --git a/internal/process/tree_unix_test.go b/internal/process/tree_unix_test.go new file mode 100644 index 00000000..b8e0b77f --- /dev/null +++ b/internal/process/tree_unix_test.go @@ -0,0 +1,160 @@ +//go:build unix + +package process + +import ( + "context" + "errors" + "os" + "path/filepath" + "strconv" + "strings" + "syscall" + "testing" + "time" +) + +// The shape behind the reported failure: the command exits 0 but leaves a +// process holding its stdout. Run used to read the pipe until that process +// ended, so a finished `npm update -g` sat until the stall watchdog called the +// successful install stalled. +func TestOSRunnerReturnsWhenACommandExitsButLeavesItsPipesOpen(t *testing.T) { + pidFile := leftoverPIDFile(t) + runner := OSRunner{StallTimeout: 3 * time.Second, PipeDrainDelay: 200 * time.Millisecond} + started := time.Now() + result, err := runner.Run(context.Background(), []string{"/bin/sh", "-c", `echo updated; sleep 30 & echo $! > "$1"; exit 0`, "sh", pidFile}, nil, time.Minute) + elapsed := time.Since(started) + if err != nil { + t.Fatalf("Run() error = %v, want nil: the command itself exited 0", err) + } + if result.ExitCode != 0 { + t.Errorf("exit code = %d, want 0", result.ExitCode) + } + if !strings.Contains(result.Stdout, "updated") { + t.Errorf("stdout = %q, want the output written before the command exited", result.Stdout) + } + if elapsed > 2*time.Second { + t.Errorf("Run() took %s; it waited on the process left behind instead of the command", elapsed.Round(time.Millisecond)) + } +} + +// The reported failure in its narrowest form: quiet for most of the stall +// timeout, then exit 0 with a leftover process holding the pipes. The stall +// timeout runs out during the drain delay, after the command has finished. A +// finished command is a success, not a stall. +func TestOSRunnerDoesNotCallAFinishedCommandStalledDuringTheDrain(t *testing.T) { + pidFile := leftoverPIDFile(t) + runner := OSRunner{StallTimeout: 1200 * time.Millisecond, PipeDrainDelay: 2 * time.Second} + result, err := runner.Run(context.Background(), []string{"/bin/sh", "-c", `sleep 0.9; sleep 30 & echo $! > "$1"; exit 0`, "sh", pidFile}, nil, time.Minute) + if err != nil { + t.Fatalf("Run() error = %v, want nil: the command exited 0 before the stall timeout", err) + } + if result.ExitCode != 0 { + t.Errorf("exit code = %d, want 0", result.ExitCode) + } +} + +// A failing command keeps its own exit code even when it, too, left something +// holding the pipes: the drain delay must not turn a failure into success. +func TestOSRunnerKeepsAFailureExitCodeWhenPipesAreLeftOpen(t *testing.T) { + pidFile := leftoverPIDFile(t) + runner := OSRunner{StallTimeout: 3 * time.Second, PipeDrainDelay: 200 * time.Millisecond} + result, err := runner.Run(context.Background(), []string{"/bin/sh", "-c", `sleep 30 & echo $! > "$1"; exit 7`, "sh", pidFile}, nil, time.Minute) + if err != nil { + t.Fatalf("Run() error = %v, want nil with the exit code reported", err) + } + if result.ExitCode != 7 { + t.Errorf("exit code = %d, want 7", result.ExitCode) + } +} + +// A stall has to stop everything the command started. Killing only the leader +// left `sleep` running with the pipe open, so Run returned when it finished -- +// for npm, when the install the user had already been told failed completed +// anyway, racing whatever they retried. +func TestOSRunnerStopsTheWholeTreeWhenACommandStalls(t *testing.T) { + pidFile := filepath.Join(t.TempDir(), "child.pid") + runner := OSRunner{StallTimeout: 300 * time.Millisecond, PipeDrainDelay: 200 * time.Millisecond} + started := time.Now() + _, err := runner.Run(context.Background(), []string{"/bin/sh", "-c", `sleep 30 & echo $! > "$1"; wait`, "sh", pidFile}, nil, time.Minute) + elapsed := time.Since(started) + if !errors.Is(err, ErrStalled) { + t.Fatalf("Run() error = %v, want ErrStalled", err) + } + if elapsed > 3*time.Second { + t.Errorf("Run() took %s after a 300ms stall; it waited on a process it should have stopped", elapsed.Round(time.Millisecond)) + } + data, readErr := os.ReadFile(pidFile) + if readErr != nil { + t.Fatalf("read child pid: %v", readErr) + } + pid, convErr := strconv.Atoi(strings.TrimSpace(string(data))) + if convErr != nil { + t.Fatalf("parse child pid %q: %v", data, convErr) + } + if !processGone(pid, 2*time.Second) { + _ = syscall.Kill(pid, syscall.SIGKILL) + t.Errorf("child %d outlived the stalled command", pid) + } +} + +// Cancellation by the user reaches the same tree as a stall does. +func TestOSRunnerStopsTheWholeTreeWhenCancelled(t *testing.T) { + pidFile := filepath.Join(t.TempDir(), "child.pid") + ctx, cancel := context.WithCancel(context.Background()) + go func() { + for range 100 { + if _, err := os.Stat(pidFile); err == nil { + break + } + time.Sleep(20 * time.Millisecond) + } + cancel() + }() + runner := OSRunner{StallTimeout: -1, PipeDrainDelay: 200 * time.Millisecond} + _, err := runner.Run(ctx, []string{"/bin/sh", "-c", `sleep 30 & echo $! > "$1"; wait`, "sh", pidFile}, nil, time.Minute) + if !errors.Is(err, context.Canceled) { + t.Fatalf("Run() error = %v, want context.Canceled", err) + } + data, readErr := os.ReadFile(pidFile) + if readErr != nil { + t.Fatalf("read child pid: %v", readErr) + } + pid, _ := strconv.Atoi(strings.TrimSpace(string(data))) + if !processGone(pid, 2*time.Second) { + _ = syscall.Kill(pid, syscall.SIGKILL) + t.Errorf("child %d outlived the cancelled command", pid) + } +} + +// leftoverPIDFile names a file the command writes the PID of the process it +// deliberately leaves behind into. That process outlives the command by +// design -- a normal exit must not reach it -- so the test stops it itself. +func leftoverPIDFile(t *testing.T) string { + t.Helper() + path := filepath.Join(t.TempDir(), "leftover.pid") + t.Cleanup(func() { + data, err := os.ReadFile(path) + if err != nil { + return + } + if pid, err := strconv.Atoi(strings.TrimSpace(string(data))); err == nil && pid > 0 { + _ = syscall.Kill(pid, syscall.SIGKILL) + } + }) + return path +} + +// processGone reports whether pid stopped existing within wait. A killed child +// of an exited shell is reparented and reaped by init, so it disappears rather +// than lingering as a zombie of this test. +func processGone(pid int, wait time.Duration) bool { + deadline := time.Now().Add(wait) + for time.Now().Before(deadline) { + if err := syscall.Kill(pid, 0); errors.Is(err, syscall.ESRCH) { + return true + } + time.Sleep(20 * time.Millisecond) + } + return false +} diff --git a/internal/process/tree_windows.go b/internal/process/tree_windows.go new file mode 100644 index 00000000..9df645bf --- /dev/null +++ b/internal/process/tree_windows.go @@ -0,0 +1,91 @@ +package process + +import ( + "os" + "os/exec" + "sync" + + "golang.org/x/sys/windows" +) + +// processTree makes cancellation reach every process a command starts, not only +// the one exec started. Windows has no process groups to signal, so the command +// is placed in a Job Object and a cancelled command terminates the job. +// +// This matters most for npm: `npm` resolves to npm.cmd, so the process exec +// starts is cmd.exe and the install itself runs in its child node.exe. Killing +// cmd.exe left node.exe running with the stdout pipe open, and Wait blocked +// until that install finished on its own. +// +// The job is not created with KILL_ON_JOB_CLOSE. A command that exits normally +// keeps whatever it deliberately left running, as it always has; only +// cancellation reaches the whole tree. +type processTree struct { + command *exec.Cmd + + // mu guards job: exec runs Cancel from its own goroutine, which can race the + // assignment in attach and the close in release. + mu sync.Mutex + job windows.Handle +} + +func newProcessTree(command *exec.Cmd) *processTree { + tree := &processTree{command: command} + command.Cancel = tree.terminate + return tree +} + +// attach places the started process in a fresh job. A process started before +// the assignment is not in the job -- Go cannot start a child suspended -- but +// cmd.exe has to read and parse the batch file before it starts node.exe, which +// is far longer than the assignment takes. Any failure here leaves the tree +// handling it had before: Cancel falls back to killing the direct child. +func (t *processTree) attach() { + if t.command.Process == nil { + return + } + job, err := windows.CreateJobObject(nil, nil) + if err != nil { + return + } + process, err := windows.OpenProcess(windows.PROCESS_SET_QUOTA|windows.PROCESS_TERMINATE, false, uint32(t.command.Process.Pid)) + if err != nil { + _ = windows.CloseHandle(job) + return + } + defer windows.CloseHandle(process) + if err := windows.AssignProcessToJobObject(job, process); err != nil { + _ = windows.CloseHandle(job) + return + } + t.mu.Lock() + t.job = job + t.mu.Unlock() +} + +func (t *processTree) release() { + t.mu.Lock() + defer t.mu.Unlock() + if t.job != 0 { + _ = windows.CloseHandle(t.job) + t.job = 0 + } +} + +func (t *processTree) terminate() error { + t.mu.Lock() + job := t.job + if job != 0 { + err := windows.TerminateJobObject(job, 1) + t.mu.Unlock() + if err == nil { + return nil + } + } else { + t.mu.Unlock() + } + if t.command.Process == nil { + return os.ErrProcessDone + } + return t.command.Process.Kill() +} From 5ecdbf9b42c5140d462d17b1d2fcc9966d6d158f Mon Sep 17 00:00:00 2001 From: yujiezhang-ops Date: Fri, 9 Oct 2026 13:55:57 +0800 Subject: [PATCH 02/18] feat: let users check a Profile's model on demand MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A model the provider lists can still be refused when an Agent calls it, and a Profile edit rewrites every Agent bound to it at once, so a single model change could break Claude Code and Codex with nothing said. Profile cards and the Agent's Profile page now have a 检测模型 button that sends one live request per protocol the Agents use and reports the verdict: available, unavailable (refused model or rejected key), or unverified (network, rate limit, server error). A failure offers the next step that fits its cause: switching back to the model the page just replaced, fixing the key, or checking again. Saving and applying a Profile never check on their own: each check is a billed request, so the user decides when it goes out. Inline model saves on the Agent's Profile page now also report Agents they could not rewrite, and 应用 waits for an in-flight model save instead of applying the stale model. The probe result carries the provider's own error message, cleaned, bounded and with the API key scrubbed. Co-Authored-By: Claude Opus 5 (1M context) --- .../BootAgent/internal/app/index.ts | 1 + .../BootAgent/internal/app/models.ts | 53 +++ .../BootAgent/internal/binding/index.ts | 1 + .../BootAgent/internal/binding/models.ts | 9 + .../internal/binding/profileservice.ts | 9 + frontend/src/backend/wails.ts | 10 + frontend/src/components/ModelCheckNotice.tsx | 166 +++++++ frontend/src/i18n.tsx | 26 ++ frontend/src/pages/AgentProfilePage.test.tsx | 167 +++++++ frontend/src/pages/AgentProfilePage.tsx | 173 ++++++- frontend/src/pages/ProfilesPage.test.tsx | 114 +++++ frontend/src/pages/ProfilesPage.tsx | 121 ++++- frontend/src/styles/app.css | 55 +++ frontend/src/types/api.ts | 13 + internal/app/profile_model_check.go | 320 +++++++++++++ internal/app/profile_model_check_test.go | 433 ++++++++++++++++++ internal/binding/services.go | 19 + internal/binding/services_test.go | 2 +- internal/provider/client.go | 56 ++- internal/provider/client_test.go | 26 ++ 20 files changed, 1734 insertions(+), 40 deletions(-) create mode 100644 frontend/src/components/ModelCheckNotice.tsx create mode 100644 internal/app/profile_model_check.go create mode 100644 internal/app/profile_model_check_test.go diff --git a/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/app/index.ts b/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/app/index.ts index aa068596..a7130b8d 100644 --- a/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/app/index.ts +++ b/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/app/index.ts @@ -34,6 +34,7 @@ export type { MarketplaceRecommendationHistory, MarketplaceRecommendationSnapshot, MarketplaceSourceStatus, + ProfileModelCheck, ProfileSummary, RuntimeStatus, SaveProfileResult, diff --git a/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/app/models.ts b/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/app/models.ts index cb720242..3c733c10 100644 --- a/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/app/models.ts +++ b/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/app/models.ts @@ -346,6 +346,59 @@ export interface MarketplaceSourceStatus { "error"?: string; } +/** + * ProfileModelCheck is the verdict of a live request to a Profile's model, + * sent only when the user asks for it. + * + * A model the provider lists can still be refused when an Agent calls it -- + * not served over that Agent's protocol, or not granted to the key -- and a + * Profile edit rewrites every Agent bound to it at once. The check lets the + * user find that out before an Agent fails. It is not run on save or apply: + * each check is a billed completion request, so it is the user's to send. + */ +export interface ProfileModelCheck { + "profileId": string; + "model": string; + + /** + * Status is "available" when every checked Agent's protocol answered, + * "unavailable" when the provider answered and refused, and "unverified" + * when no verdict could be had at all. The last is not a claim that the + * model is broken, and the UI must not word it as one. + */ + "status": string; + + /** + * Reason is the cause the remedy follows from: + * refused the provider refused this model on this protocol + * key the provider rejected the API key + * transient the provider answered but was busy or failing (408, 429, 5xx) + * network the provider could not be reached + * no_key no API key is saved for the Provider + * error the request could not be made (an endpoint misconfiguration) + * Empty when Status is "available". + */ + "reason"?: string; + "protocol"?: string; + + /** + * HTTPStatus is the status the provider answered with, 0 when it did not. + */ + "httpStatus"?: number; + + /** + * Detail is the provider's own explanation, cleaned and bounded. + */ + "detail"?: string; + + /** + * Agents are the Agents the verdict is about: on a failure, those whose + * requests to this model would fail; when available, those checked. Empty + * for a Profile nothing follows yet. + */ + "agents": string[] | null; +} + /** * ProfileSummary is intentionally a public projection with no credential field. */ diff --git a/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/binding/index.ts b/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/binding/index.ts index 5ec12e06..6f47de3e 100644 --- a/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/binding/index.ts +++ b/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/binding/index.ts @@ -34,6 +34,7 @@ export type { ActivateRequest, ActivateResponse, AgentInstallResult, + CheckProfileModelRequest, DesktopAgentProfileRequest, DesktopAgentRequest, InstallRequest, diff --git a/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/binding/models.ts b/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/binding/models.ts index 080be507..cc2c457e 100644 --- a/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/binding/models.ts +++ b/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/binding/models.ts @@ -34,6 +34,15 @@ export interface AgentInstallResult { "retryable": boolean; } +export interface CheckProfileModelRequest { + "profile_id": string; + + /** + * Agents narrows the check; empty means every Agent bound to the Profile. + */ + "agents": string[] | null; +} + export interface DesktopAgentProfileRequest { "agent_id": string; "profile_id": string; diff --git a/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/binding/profileservice.ts b/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/binding/profileservice.ts index 1e167097..b5fc2a8b 100644 --- a/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/binding/profileservice.ts +++ b/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/binding/profileservice.ts @@ -13,6 +13,15 @@ import * as app$0 from "../app/models.js"; // @ts-ignore: Unused imports import * as $models from "./models.js"; +/** + * CheckModel sends a live request to a Profile's model when the user asks for + * it. Saving and applying a Profile never check on their own: each check is a + * billed request. + */ +export function CheckModel(request: $models.CheckProfileModelRequest): $CancellablePromise { + return $Call.ByID(3431853122, request); +} + export function DeleteProfile(request: $models.ProviderIDRequest): $CancellablePromise<$models.ProviderMutationResponse> { return $Call.ByID(2548092141, request); } diff --git a/frontend/src/backend/wails.ts b/frontend/src/backend/wails.ts index aef0613c..04c14a1e 100644 --- a/frontend/src/backend/wails.ts +++ b/frontend/src/backend/wails.ts @@ -39,6 +39,7 @@ import type { ModelsResponse, OpenRegistrationResponse, ProbeResponse, + ProfileModelCheck, ProfileSummary, ProviderEntry, ProviderId, @@ -337,4 +338,13 @@ export const wailsApi = { config_mode: input.configMode, protocol: input.protocol ?? "", })) as CancellableRequest, + // Sends a live (billed) request to the Profile's model; only called from a + // button the user presses. `agents` narrows the check to the Agent a page is + // configuring; omitted means every Agent bound to the Profile. + checkProfileModel: (profileId: string, agents?: string[]): Promise => + call(() => ProfileService.CheckModel({ profile_id: profileId, agents: agents?.length ? agents : null })) + .then((check) => { + if (!check) throw new Error("Model check returned no result"); + return { ...check, agents: check.agents ?? [] }; + }) as Promise, }; diff --git a/frontend/src/components/ModelCheckNotice.tsx b/frontend/src/components/ModelCheckNotice.tsx new file mode 100644 index 00000000..c23f793a --- /dev/null +++ b/frontend/src/components/ModelCheckNotice.tsx @@ -0,0 +1,166 @@ +import { Activity, AlertTriangle, CheckCircle2 } from "lucide-react"; +import { useNavigate } from "react-router-dom"; + +import type { Translate } from "../i18n"; +import { useI18n } from "../i18n"; +import type { ProfileModelCheck } from "../types/api"; + +const protocolLabels: Record = { + anthropic: "Anthropic Messages", + openai: "OpenAI Chat Completions", + responses: "OpenAI Responses", +}; + +/** + * The headline and explanation for a model check, without the actions. + * + * "unverified" is worded as "could not verify", never as "not available": a + * network failure is no evidence against the model, and telling the user it + * is broken would send them to abandon a model that works. + */ +export function describeModelCheck(check: ProfileModelCheck, t: Translate, nameOf: (agentId: string) => string, separator: string) { + const agents = (check.agents ?? []).map(nameOf).join(separator); + const protocol = protocolLabels[check.protocol ?? ""] ?? check.protocol ?? ""; + if (check.status === "available") { + return { + title: t("模型「{model}」可用", { model: check.model }), + reason: protocol ? t("模型服务已通过 {protocol} 正常响应。", { protocol }) : t("模型服务已正常响应。"), + agents: agents ? t("已检测的 Agent：{agents}", { agents }) : "", + detail: "", + }; + } + const title = check.status === "unverified" + ? t("无法验证模型「{model}」", { model: check.model }) + : t("模型「{model}」不可用", { model: check.model }); + let reason: string; + switch (check.reason) { + case "refused": + reason = check.httpStatus + ? t("模型服务拒绝了该模型（{protocol}，HTTP {status}）。", { protocol, status: check.httpStatus }) + : t("模型服务拒绝了该模型（{protocol}）。", { protocol }); + break; + case "key": + reason = t("模型服务拒绝了 API Key，换模型不能解决这个问题。"); + break; + case "transient": + reason = check.httpStatus + ? t("模型服务暂时繁忙或出错（HTTP {status}），模型本身可能没有问题。", { status: check.httpStatus }) + : t("模型服务暂时繁忙或出错，模型本身可能没有问题。"); + break; + case "network": + reason = t("暂时连不上模型服务，模型本身可能没有问题。"); + break; + case "no_key": + reason = t("该模型服务还没有保存 API Key。"); + break; + default: + reason = t("无法向模型服务发送验证请求。"); + } + return { + title, + reason, + agents: agents ? t("受影响的 Agent：{agents}", { agents }) : "", + detail: check.detail ? t("模型服务返回：{detail}", { detail: check.detail }) : "", + }; +} + +interface ModelCheckButtonProps { + checking: boolean; + disabled?: boolean; + onCheck: () => void; + /** Names the Profile for screen readers where several buttons share a page. */ + ariaLabel?: string; +} + +/** + * The only way a model check starts. Saving and applying a Profile never send + * one on their own: each is a billed request to the provider, so the user + * decides when it goes out, and the button says so before they press it. + */ +export function ModelCheckButton({ checking, disabled, onCheck, ariaLabel }: ModelCheckButtonProps) { + const { t } = useI18n(); + return ( + + ); +} + +interface ModelCheckNoticeProps { + check: ProfileModelCheck; + /** Provider the Profile uses, for the key remedies' link. */ + providerId: string; + nameOf: (agentId: string) => string; + /** Where the Provider editor returns to once the key is fixed. */ + returnTo: string; + /** Names the Profile when the page lists several. */ + profileName?: string; + busy?: boolean; + /** Saves the Profile back to check.previousModel. Omitted where no save path exists. */ + onRevert?: () => void; + onRecheck?: () => void; + onDismiss: () => void; +} + +/** + * The result of a check the user asked for. A failure leaves a next step, not + * just a problem: going back to the model the user had before, fixing the key + * that was refused, or checking again once the network is back. Which remedy + * is offered follows from why the check failed -- going back to the old model + * cannot fix a rejected key. + */ +export function ModelCheckNotice({ check, providerId, nameOf, returnTo, profileName, busy, onRevert, onRecheck, onDismiss }: ModelCheckNoticeProps) { + const { t, locale } = useI18n(); + const navigate = useNavigate(); + const parts = describeModelCheck(check, t, nameOf, locale === "en" ? ", " : "、"); + const ok = check.status === "available"; + const keyProblem = check.reason === "key" || check.reason === "no_key"; + const canRevert = Boolean(!ok && onRevert && check.previousModel && check.previousModel !== check.model && !keyProblem); + const editKey = () => navigate(`/providers?provider=${encodeURIComponent(providerId)}&returnTo=${encodeURIComponent(returnTo)}`); + const Icon = ok ? CheckCircle2 : AlertTriangle; + // The live region holds the message only. Buttons inside it would be read out + // with it. A failure is assertive; a pass only needs to be announced politely. + return ( +
+
+ ); +} diff --git a/frontend/src/i18n.tsx b/frontend/src/i18n.tsx index 1a5c6f1e..c076cab3 100644 --- a/frontend/src/i18n.tsx +++ b/frontend/src/i18n.tsx @@ -661,6 +661,32 @@ const english = { "模型服务正在被 {agents} 使用，无法删除": "Provider is used by {agents} and cannot be deleted", "已重新应用到 {agents}": "Reapplied to {agents}", "{agents} 重新应用失败：{message}": "Could not reapply to {agents}: {message}", + "模型「{model}」可用": "Model \"{model}\" is available", + "模型「{model}」不可用": "Model \"{model}\" is not available", + "无法验证模型「{model}」": "Could not verify model \"{model}\"", + "模型服务已通过 {protocol} 正常响应。": "The provider answered over {protocol}.", + "模型服务已正常响应。": "The provider answered.", + "已检测的 Agent：{agents}": "Checked for: {agents}", + "模型服务拒绝了该模型（{protocol}，HTTP {status}）。": "The provider refused this model ({protocol}, HTTP {status}).", + "模型服务拒绝了该模型（{protocol}）。": "The provider refused this model ({protocol}).", + "模型服务拒绝了 API Key，换模型不能解决这个问题。": "The provider rejected the API key; changing the model will not fix this.", + "模型服务暂时繁忙或出错（HTTP {status}），模型本身可能没有问题。": "The provider was busy or failing (HTTP {status}); the model itself may be fine.", + "模型服务暂时繁忙或出错，模型本身可能没有问题。": "The provider was busy or failing; the model itself may be fine.", + "暂时连不上模型服务，模型本身可能没有问题。": "The provider could not be reached; the model itself may be fine.", + "该模型服务还没有保存 API Key。": "This provider has no API key saved yet.", + "无法向模型服务发送验证请求。": "A verification request could not be sent to the provider.", + "受影响的 Agent：{agents}": "Affected Agents: {agents}", + "模型服务返回：{detail}": "Provider said: {detail}", + "改回 {model}": "Switch back to {model}", + "检测模型": "Check model", + "检测 {name} 的模型": "Check the model of {name}", + "正在检测": "Checking", + "重新检测": "Check again", + "向模型服务发送一次真实请求，确认 Agent 能调用这个模型。可能产生少量费用。": "Sends one real request to the provider to confirm Agents can call this model. This may incur a small charge.", + "检查模型服务的 API Key": "Check the provider's API key", + "为模型服务添加 API Key": "Add an API key for the provider", + "忽略": "Dismiss", + "无法检测模型": "Could not check the model", "删除模型服务“{name}”？": "Delete provider \"{name}\"?", "无法删除模型服务": "Could not delete provider", "管理模型服务、端点与本机保存的 API Key": "Manage model providers, endpoints, and locally saved API keys", diff --git a/frontend/src/pages/AgentProfilePage.test.tsx b/frontend/src/pages/AgentProfilePage.test.tsx index 2d25da36..966841ac 100644 --- a/frontend/src/pages/AgentProfilePage.test.tsx +++ b/frontend/src/pages/AgentProfilePage.test.tsx @@ -104,10 +104,21 @@ beforeEach(() => { ok: true, reachable: true, status: 200, error_code: "", retryable: false, models: ["deepseek/deepseek-v3", "deepseek/deepseek-r1"], message: "", }); + // Only 检测模型 sends a check; a test that expects none asserts on this spy. + vi.spyOn(api, "checkProfileModel").mockResolvedValue(availableCheck); }); afterEach(() => vi.restoreAllMocks()); +const refusedCheck = { + profileId: "team-ppio", model: "claude-opus-5-5", + status: "unavailable" as const, reason: "refused" as const, protocol: "responses", httpStatus: 404, + detail: "model does not exist", agents: ["codex"], +}; +const availableCheck = { + profileId: "team-ppio", model: "deepseek/deepseek-v3", status: "available" as const, protocol: "responses", agents: ["codex"], +}; + describe("AgentProfilePage", () => { // Requirement: after applying, land where the result is visible instead of // staying on a screen whose job is done. @@ -162,6 +173,162 @@ describe("AgentProfilePage", () => { await waitFor(() => expect(refreshStatus).toHaveBeenCalled()); }); + // A check is a billed request, so it is the user's to send: an inline model + // edit and an Apply send none. + it("does not check the model on an inline edit or on apply", async () => { + vi.spyOn(api, "saveProfile").mockResolvedValue({ ...saved({ model: "deepseek/deepseek-r1" }), reapplied: ["codex"] }); + vi.spyOn(api, "activateAgent").mockResolvedValue({ + ok: true, agent: "codex", config: "/c", provider: "ppio", + model: "deepseek/deepseek-r1", restart: "restart", next: "next", + }); + const check = vi.spyOn(api, "checkProfileModel"); + renderPage([profile()]); + await waitFor(() => expect(screen.getByLabelText("模型")).toHaveAttribute("placeholder", "搜索模型")); + const field = screen.getByLabelText("模型"); + await userEvent.clear(field); + await userEvent.type(field, "deepseek/deepseek-r1"); + await userEvent.tab(); + fireEvent.click(screen.getByRole("button", { name: "应用" })); + + expect(await screen.findByRole("heading", { name: "overview" })).toBeTruthy(); + expect(check).not.toHaveBeenCalled(); + }); + + // The reported path: picking a model inline saved it and rewrote every Agent + // on the Profile. 检测模型 then names the refusal, with the model to go back + // to, checked for this Agent only. + it("checks on request after an inline edit and switches back", async () => { + const save = vi.spyOn(api, "saveProfile") + .mockResolvedValueOnce({ ...saved({ model: "claude-opus-5-5" }), reapplied: ["codex"] }) + .mockResolvedValueOnce({ ...saved(), reapplied: ["codex"] }); + const check = vi.spyOn(api, "checkProfileModel").mockResolvedValue(refusedCheck); + renderPage([profile()]); + await waitFor(() => expect(screen.getByLabelText("模型")).toHaveAttribute("placeholder", "搜索模型")); + const field = screen.getByLabelText("模型"); + await userEvent.clear(field); + await userEvent.type(field, "claude-opus-5-5"); + await userEvent.tab(); + await waitFor(() => expect(save).toHaveBeenCalledTimes(1)); + expect(screen.queryByRole("alert")).toBeNull(); + + await userEvent.click(screen.getByRole("button", { name: "检测模型" })); + const notice = await screen.findByRole("alert"); + expect(check).toHaveBeenCalledWith("team-ppio", ["codex"]); + expect(notice.textContent).toContain("模型「claude-opus-5-5」不可用"); + expect(notice.textContent).toContain("Codex"); + await userEvent.click(screen.getByRole("button", { name: "改回 deepseek/deepseek-v3" })); + await waitFor(() => expect(save).toHaveBeenCalledTimes(2)); + expect(save.mock.calls[1][0]).toMatchObject({ id: "team-ppio", model: "deepseek/deepseek-v3" }); + await waitFor(() => expect(screen.queryByRole("alert")).toBeNull()); + expect(check).toHaveBeenCalledTimes(1); + }); + + // Pressing 检测模型 straight after typing checks what is on screen: the typed + // model is saved first, once, and the check offers the model it replaced. + it("saves the typed model once before checking it", async () => { + const save = vi.spyOn(api, "saveProfile").mockResolvedValue({ ...saved({ model: "claude-opus-5-5" }), reapplied: ["codex"] }); + const check = vi.spyOn(api, "checkProfileModel").mockResolvedValue(refusedCheck); + renderPage([profile()]); + await waitFor(() => expect(screen.getByLabelText("模型")).toHaveAttribute("placeholder", "搜索模型")); + const field = screen.getByLabelText("模型"); + await userEvent.clear(field); + await userEvent.type(field, "claude-opus-5-5"); + await userEvent.click(screen.getByRole("button", { name: "检测模型" })); + + await screen.findByRole("alert"); + expect(save).toHaveBeenCalledTimes(1); + expect(save.mock.calls[0][0]).toMatchObject({ model: "claude-opus-5-5" }); + expect(check).toHaveBeenCalledTimes(1); + expect(screen.getByRole("button", { name: "改回 deepseek/deepseek-v3" })).toBeTruthy(); + }); + + // A check the user asked for always ends with an answer, a pass included. + it("reports a model that answered", async () => { + renderPage([profile()]); + await userEvent.click(await screen.findByRole("button", { name: "检测模型" })); + expect(await screen.findByText("模型「deepseek/deepseek-v3」可用")).toBeTruthy(); + expect(screen.queryByRole("alert")).toBeNull(); + expect(screen.queryByRole("button", { name: /改回/ })).toBeNull(); + }); + + // Both inline save paths dropped result.failures, so an Agent that could not + // be rewritten went unreported here. + it("reports an Agent an inline model edit could not rewrite", async () => { + vi.spyOn(api, "saveProfile").mockResolvedValue({ ...saved({ model: "deepseek/deepseek-r1" }), failures: { codex: "写入失败" } }); + renderPage([profile()]); + await waitFor(() => expect(screen.getByLabelText("模型")).toHaveAttribute("placeholder", "搜索模型")); + const field = screen.getByLabelText("模型"); + await userEvent.clear(field); + await userEvent.type(field, "deepseek/deepseek-r1"); + await userEvent.tab(); + expect(await screen.findByText(/Codex.*重新应用失败.*写入失败/)).toBeTruthy(); + }); + + // Apply right after typing: the flush saves the new model, and the Apply has + // to activate with that one. It used to read the model from before the click + // and write the old model back over the Agent the flush had just moved. + it("applies the model the flush saved, not the one from before the click", async () => { + vi.spyOn(api, "saveProfile").mockResolvedValue({ ...saved({ model: "deepseek/deepseek-r1" }), reapplied: ["codex"] }); + const activate = vi.spyOn(api, "activateAgent").mockResolvedValue({ + ok: true, agent: "codex", config: "/c", provider: "ppio", + model: "deepseek/deepseek-r1", restart: "restart", next: "next", + }); + renderPage([profile()]); + await waitFor(() => expect(screen.getByLabelText("模型")).toHaveAttribute("placeholder", "搜索模型")); + const field = screen.getByLabelText("模型"); + await userEvent.clear(field); + await userEvent.type(field, "deepseek/deepseek-r1"); + fireEvent.click(screen.getByRole("button", { name: "应用" })); + + await waitFor(() => expect(activate).toHaveBeenCalled()); + expect(activate.mock.calls[0][1]).toMatchObject({ model: "deepseek/deepseek-r1", profileId: "team-ppio" }); + }); + + // A flush whose save failed leaves the Profile on the old model. Applying + // anyway would apply that while the field showed the new one. + it("does not apply when flushing the typed model fails", async () => { + const { BootAgentApiError } = await import("../backend/errors"); + vi.spyOn(api, "saveProfile").mockRejectedValue(new BootAgentApiError("disk full", "CONFIG_WRITE_FAILED", false, 500)); + const activate = vi.spyOn(api, "activateAgent"); + renderPage([profile()]); + await waitFor(() => expect(screen.getByLabelText("模型")).toHaveAttribute("placeholder", "搜索模型")); + const field = screen.getByLabelText("模型"); + await userEvent.clear(field); + await userEvent.type(field, "deepseek/deepseek-r1"); + fireEvent.click(screen.getByRole("button", { name: "应用" })); + + // CONFIG_WRITE_FAILED is shown as its localized copy, not the raw message. + expect(await screen.findByText("无法写入配置文件")).toBeTruthy(); + expect(activate).not.toHaveBeenCalled(); + expect(screen.queryByRole("heading", { name: "overview" })).toBeNull(); + }); + + // The review's race: leaving the field starts a save. Clicking 应用 in that + // window found no draft left to flush and applied the stored model -- the + // old one, since the save had not finished. 应用 waits for it instead. + it("does not apply while a model the user just left is still saving", async () => { + let finishSave: (value: SaveProfileResult) => void = () => undefined; + vi.spyOn(api, "saveProfile").mockImplementation(() => new Promise((resolve) => { finishSave = resolve; }) as never); + const activate = vi.spyOn(api, "activateAgent").mockResolvedValue({ + ok: true, agent: "codex", config: "/c", provider: "ppio", + model: "deepseek/deepseek-r1", restart: "restart", next: "next", + }); + renderPage([profile()]); + await waitFor(() => expect(screen.getByLabelText("模型")).toHaveAttribute("placeholder", "搜索模型")); + const field = screen.getByLabelText("模型"); + await userEvent.clear(field); + await userEvent.type(field, "deepseek/deepseek-r1"); + await userEvent.tab(); + + const button = screen.getByRole("button", { name: "应用" }); + expect(button).toBeDisabled(); + fireEvent.click(button); + expect(activate).not.toHaveBeenCalled(); + + finishSave({ ...saved({ model: "deepseek/deepseek-r1" }), reapplied: ["codex"] }); + await waitFor(() => expect(button).not.toBeDisabled()); + }); + it("does not write when the model is unchanged or blank", async () => { const save = vi.spyOn(api, "saveProfile").mockResolvedValue(saved()); renderPage([profile()]); diff --git a/frontend/src/pages/AgentProfilePage.tsx b/frontend/src/pages/AgentProfilePage.tsx index b812226d..6afba3d1 100644 --- a/frontend/src/pages/AgentProfilePage.tsx +++ b/frontend/src/pages/AgentProfilePage.tsx @@ -3,6 +3,7 @@ import { useEffect, useMemo, useState, type FormEvent } from "react"; import { useNavigate, useParams } from "react-router-dom"; import { api, describeFailure } from "../backend/api"; +import { ModelCheckButton, ModelCheckNotice } from "../components/ModelCheckNotice"; import { PageScaffold } from "../components/PageScaffold"; import { ProviderModelPicker } from "../components/ProviderModelPicker"; import { ProviderSegment } from "../components/ProviderSegment"; @@ -13,7 +14,25 @@ import { confirmAction } from "../state/confirmDelete"; import { byProfileCreatedAt, byProviderCreatedAt, preferProviderWithKey } from "../state/ranking"; import { converterProfileName, isConverterID } from "../state/conversion"; import { useWizard } from "../state/WizardContext"; -import type { DesktopAgentProfileAssessment, ProfileSummary, ProtocolId, ProviderId } from "../types/api"; +import type { DesktopAgentProfileAssessment, ProfileModelCheck, ProfileSummary, ProtocolId, ProviderId } from "../types/api"; + +type SaveRequest = Parameters[0]; + +// The last model edit saved from this page, so a check that then fails can +// offer the model the Profile held before it. Only an edit on the same +// Provider qualifies: a model ID is not portable between Providers. +interface ModelEdit { + profileId: string; + previousModel: string; + request: SaveRequest; +} + +// What flushing an inline model edit did, which Apply and 检测模型 act on: a +// failed save must stop them, and a saved one supplies the model to use. +type CommitOutcome = + | { kind: "unchanged" } + | { kind: "failed" } + | { kind: "saved"; model: string; edit: ModelEdit | null }; interface ProfileDraft { id: string; @@ -42,7 +61,7 @@ function draftFrom(profile: ProfileSummary): ProfileDraft { export function AgentProfilePage() { const { agentId = "" } = useParams(); const navigate = useNavigate(); - const { t } = useI18n(); + const { locale, t } = useI18n(); const { state, refreshStatus } = useWizard(); const status = state.status; const app = status ? desktopApps(status).find((candidate) => candidate.id === agentId) || null : null; @@ -61,6 +80,10 @@ export function AgentProfilePage() { const [savingModel, setSavingModel] = useState(false); const [assessment, setAssessment] = useState(null); const [assessing, setAssessing] = useState(false); + // A check runs only when the user presses 检测模型: it is a billed request. + const [checked, setChecked] = useState(null); + const [checking, setChecking] = useState(false); + const [lastEdit, setLastEdit] = useState(null); const profiles = useMemo(() => { if (!status) return []; @@ -122,6 +145,57 @@ export function AgentProfilePage() { !requiresConversion || assessment?.compatibility === "convertible" )); + const separator = locale === "en" ? ", " : "、"; + const nameOf = (id: string) => + status.catalog.find((item) => item.id === id)?.name + || desktopApps(status).find((candidate) => candidate.id === id)?.name + || id; + + /** + * Saves a Profile and reports the Agents the save could not rewrite. Those + * used to be dropped here, so an inline model change that broke an Agent on + * the Profile showed nothing at all. Nothing is sent to the model: that is + * what 检测模型 is for. + */ + const persistProfile = async (request: SaveRequest) => { + const before = status.profiles.find((profile) => profile.id === request.id); + const result = await api.saveProfile(request); + const failures = Object.entries(result.failures ?? {}); + if (failures.length) { + setFailure(t("{agents} 重新应用失败：{message}", { + agents: failures.map(([id]) => nameOf(id)).join(separator), + message: failures[0][1] ?? "", + })); + } + // A result for this Profile describes the model it held before this save. + if (checked?.profileId === request.id) setChecked(null); + const edit = before && before.provider === request.provider && before.model && before.model !== request.model + ? { profileId: request.id, previousModel: before.model, request } + : null; + setLastEdit(edit); + // Returned as well as stored: 检测模型 reads it right after a flush, before + // React has rendered the state update. + return { result, edit }; + }; + + // Saves the model back without checking it again; the user can press + // 检测模型 once more if they want the old one confirmed. + const revertModel = async () => { + if (!checked?.previousModel || lastEdit?.profileId !== checked.profileId) return; + const request = { ...lastEdit.request, model: checked.previousModel }; + setBusy(true); + setFailure(""); + try { + await persistProfile(request); + setChecked(null); + await refreshStatus(); + } catch (error) { + setFailure(describeFailure(error, t("无法保存配置模版"), t).message); + } finally { + setBusy(false); + } + }; + const openCreate = () => { const usable = byProviderCreatedAt(status.providers).filter(([, meta]) => protocol === "anthropic" ? meta.anthropic_base_url : meta.base_url, @@ -157,7 +231,7 @@ export function AgentProfilePage() { setBusy(true); setFailure(""); try { - const saved = await api.saveProfile({ + const { result: saved } = await persistProfile({ id: draft.id.trim().toLowerCase(), label: draft.label.trim(), provider: draft.provider, @@ -187,14 +261,14 @@ export function AgentProfilePage() { * write a Profile per letter typed. Everything else about the Profile is passed * through unchanged, so this cannot quietly reset a field the editor owns. */ - const commitModel = async (profile: ProfileSummary, next: string) => { + const commitModel = async (profile: ProfileSummary, next: string): Promise => { const model = next.trim(); setModelDraft(null); - if (!model || model === (profile.model || "")) return; + if (!model || model === (profile.model || "")) return { kind: "unchanged" }; setSavingModel(true); setFailure(""); try { - await api.saveProfile({ + const { edit } = await persistProfile({ id: profile.id, label: profile.label, provider: profile.provider, @@ -207,20 +281,57 @@ export function AgentProfilePage() { protocol: profile.protocol || (app ? desktopProtocol(app) : catalog?.protocol || ""), }); await refreshStatus(); + return { kind: "saved", model, edit }; } catch (error) { setFailure(describeFailure(error, t("无法保存配置模版"), t).message); + return { kind: "failed" }; } finally { setSavingModel(false); } }; + /** + * Checks the selected Profile's model for this Agent, on the user's request. + * An uncommitted inline edit is saved first, so the check covers the model on + * screen rather than the one stored before it. + */ + const checkModel = async () => { + if (!selected || checking || busy || savingModel) return; + setChecking(true); + setFailure(""); + try { + const flushed: CommitOutcome = modelDraft !== null ? await commitModel(selected, modelDraft) : { kind: "unchanged" }; + if (flushed.kind === "failed") return; + const check = await api.checkProfileModel(selected.id, [app?.id || owner]); + const edit = flushed.kind === "saved" ? flushed.edit : lastEdit; + setChecked({ ...check, previousModel: edit?.profileId === selected.id && edit.request.model === check.model ? edit.previousModel : undefined }); + } catch (error) { + setFailure(describeFailure(error, t("无法检测模型"), t).message); + } finally { + setChecking(false); + } + }; + const apply = async () => { - if (!selected || !canApply) return; - // An uncommitted inline model edit is flushed first, so clicking 应用 straight - // after typing applies what is on screen rather than the older stored model. - if (modelDraft !== null) await commitModel(selected, modelDraft); + if (!selected || !canApply || busy || savingModel || checking) return; + // Busy before anything is awaited: setting it only after the flush below + // left 应用 enabled while the save was in flight, so a second click started + // a second Apply. setBusy(true); setFailure(""); + // An uncommitted inline model edit is flushed first, so clicking 应用 straight + // after typing applies what is on screen rather than the older stored model. + const flushed: CommitOutcome = modelDraft !== null ? await commitModel(selected, modelDraft) : { kind: "unchanged" }; + if (flushed.kind === "failed") { + // The Profile still holds the old model; applying now would apply that + // while the field showed the new one. The save's error is already shown. + setBusy(false); + return; + } + // The model the flush saved, not selected.model: that is the value from + // before the click, and activating with it would write the old model back + // over the Agents the flush had just moved to the new one. + const model = flushed.kind === "saved" ? flushed.model : selected.model || ""; try { if (app) { if (requiresConversion) { @@ -242,21 +353,22 @@ export function AgentProfilePage() { provider: selected.provider, apiBaseUrl: "", apiKey: "", - model: selected.model || "", + model, profileId: selected.id, }); } await refreshStatus(); - // Straight back to the overview, which is where the result is visible: the - // Agent's row now names this Profile. Staying here left the user on a - // screen whose job was done, with a success notice as the only feedback and - // a second click needed to see the effect. `replace` so Back does not - // return to a selection that has already been applied. - navigate("/overview", { replace: true }); } catch (error) { setFailure(describeFailure(error, t("应用配置模版失败"), t).message); setBusy(false); + return; } + // Straight back to the overview, which is where the result is visible: the + // Agent's row now names this Profile. Staying here left the user on a + // screen whose job was done, with a success notice as the only feedback and + // a second click needed to see the effect. `replace` so Back does not + // return to a selection that has already been applied. + navigate("/overview", { replace: true }); }; return ( @@ -266,7 +378,7 @@ export function AgentProfilePage() { onBack={() => navigate("/overview")} primaryLabel={busy ? t("应用中") : requiresConversion ? t("启用协议适配并应用") : t("应用")} onPrimary={() => void apply()} - primaryDisabled={!canApply || busy} + primaryDisabled={!canApply || busy || savingModel || checking} footerNote={selected ? displayProfileName(selected) : t("选择一个配置模版")} // Stays secondary: this footer already has a primary ("应用"), and two // blue buttons side by side would compete for the same attention. @@ -277,6 +389,19 @@ export function AgentProfilePage() { )} > {failure ?
{failure}
: null} + {checked ? ( + profile.id === checked.profileId)?.provider || ""} + nameOf={nameOf} + returnTo={`/agents/${encodeURIComponent(agentId)}`} + profileName={checked.profileId === selected?.id ? undefined : profiles.find((profile) => profile.id === checked.profileId)?.label} + busy={busy || savingModel || checking} + onRevert={() => void revertModel()} + onRecheck={checked.profileId === selected?.id ? () => void checkModel() : undefined} + onDismiss={() => setChecked(null)} + /> + ) : null} {requiresConversion ? (
{assessing @@ -394,7 +519,17 @@ export function AgentProfilePage() {
{selected && !isConverterID(selected.id) ? (
-
{t("编辑 {name}", { name: displayProfileName(selected) })}{savingModel ? {t("正在保存模型")} : null}
+
+ {t("编辑 {name}", { name: displayProfileName(selected) })} + + {savingModel ? {t("正在保存模型")} : null} + void checkModel()} + /> + +
{ await waitFor(() => expect(screen.getByText(/Codex.*重新应用失败.*写入失败/)).toBeTruthy()); }); + // A check is a billed request, so saving never sends one; the card's + // 检测模型 does, for every Agent following the Profile. + it("checks the model only when 检测模型 is pressed", async () => { + vi.spyOn(api, "saveProfile").mockResolvedValue({ profile: profile({ model: "claude-opus-5-5" }), reapplied: ["codex"], failures: null }); + const check = vi.spyOn(api, "checkProfileModel").mockResolvedValue({ + profileId: "team-ppio", model: "deepseek/deepseek-v3", status: "available", protocol: "responses", agents: ["codex"], + }); + renderPage([profile()]); + fireEvent.click(screen.getByRole("button", { name: "编辑 团队 PPIO" })); + fireEvent.change(screen.getByLabelText("模型"), { target: { value: "claude-opus-5-5" } }); + fireEvent.click(screen.getByRole("button", { name: "保存配置模版" })); + await waitFor(() => expect(screen.queryByRole("button", { name: "保存配置模版" })).toBeNull()); + expect(check).not.toHaveBeenCalled(); + + fireEvent.click(screen.getByRole("button", { name: "检测 团队 PPIO 的模型" })); + await waitFor(() => expect(check).toHaveBeenCalledWith("team-ppio")); + expect(await screen.findByText(/模型「deepseek\/deepseek-v3」可用/)).toBeTruthy(); + expect(screen.getByText("已检测的 Agent：Codex")).toBeTruthy(); + }); + + // The report: a model the provider refuses broke every Agent on the Profile. + // A check after the edit names the model, the Agents and the provider's + // words, and offers the model the edit replaced -- which saves the same + // Profile with only the model changed. + it("names a refused model and switches back to the one the edit replaced", async () => { + const save = vi.spyOn(api, "saveProfile") + .mockResolvedValueOnce({ profile: profile({ model: "claude-opus-5-5" }), reapplied: ["codex"], failures: null }) + .mockResolvedValueOnce({ profile: profile(), reapplied: ["codex"], failures: null }); + vi.spyOn(api, "checkProfileModel").mockResolvedValue({ + profileId: "team-ppio", model: "claude-opus-5-5", status: "unavailable", reason: "refused", protocol: "responses", httpStatus: 404, + detail: "model does not exist or you do not have access", agents: ["codex"], + }); + renderPage([profile()]); + fireEvent.click(screen.getByRole("button", { name: "编辑 团队 PPIO" })); + fireEvent.change(screen.getByLabelText("模型"), { target: { value: "claude-opus-5-5" } }); + fireEvent.click(screen.getByRole("button", { name: "保存配置模版" })); + await waitFor(() => expect(save).toHaveBeenCalledTimes(1)); + fireEvent.click(screen.getByRole("button", { name: "检测 团队 PPIO 的模型" })); + + const notice = await screen.findByRole("alert"); + expect(notice.textContent).toContain("模型「claude-opus-5-5」不可用"); + expect(notice.textContent).toContain("HTTP 404"); + expect(notice.textContent).toContain("Codex"); + expect(notice.textContent).toContain("model does not exist or you do not have access"); + expect(notice.querySelectorAll("button")).toHaveLength(0); + + fireEvent.click(screen.getByRole("button", { name: "改回 deepseek/deepseek-v3" })); + await waitFor(() => expect(save).toHaveBeenCalledTimes(2)); + expect(save.mock.calls[1][0]).toMatchObject({ id: "team-ppio", model: "deepseek/deepseek-v3", provider: "ppio" }); + await waitFor(() => expect(screen.queryByRole("alert")).toBeNull()); + }); + + // Without an edit on this page there is no earlier model to go back to. + it("offers no switch-back for a model this page did not change", async () => { + vi.spyOn(api, "checkProfileModel").mockResolvedValue({ + profileId: "team-ppio", model: "deepseek/deepseek-v3", status: "unavailable", reason: "refused", agents: ["codex"], + }); + renderPage([profile()]); + fireEvent.click(screen.getByRole("button", { name: "检测 团队 PPIO 的模型" })); + await screen.findByRole("alert"); + expect(screen.queryByRole("button", { name: /改回/ })).toBeNull(); + }); + + // A network failure is no evidence against the model. It is worded as + // "could not verify", and a recheck that now passes replaces it. + it("words an unreachable provider as unverified and rechecks on request", async () => { + const check = vi.spyOn(api, "checkProfileModel") + .mockResolvedValueOnce({ profileId: "team-ppio", model: "deepseek/deepseek-v3", status: "unverified", reason: "network", agents: ["codex"] }) + .mockResolvedValueOnce({ profileId: "team-ppio", model: "deepseek/deepseek-v3", status: "available", agents: ["codex"] }); + renderPage([profile()]); + fireEvent.click(screen.getByRole("button", { name: "检测 团队 PPIO 的模型" })); + + const notice = await screen.findByRole("alert"); + expect(notice.textContent).toContain("无法验证模型「deepseek/deepseek-v3」"); + expect(notice.textContent).not.toContain("不可用"); + + fireEvent.click(screen.getByRole("button", { name: "重新检测" })); + await waitFor(() => expect(check).toHaveBeenCalledTimes(2)); + await waitFor(() => expect(screen.queryByRole("alert")).toBeNull()); + expect(screen.getByText(/可用/)).toBeTruthy(); + }); + + // A busy provider is no evidence against the model. + it("words a busy provider as unverified, not as the model failing", async () => { + vi.spyOn(api, "checkProfileModel").mockResolvedValue({ + profileId: "team-ppio", model: "new-model", status: "unverified", reason: "transient", httpStatus: 429, agents: ["codex"], + }); + renderPage([profile({ model: "new-model" })]); + fireEvent.click(screen.getByRole("button", { name: "检测 团队 PPIO 的模型" })); + const notice = await screen.findByRole("alert"); + expect(notice.textContent).toContain("无法验证模型「new-model」"); + expect(notice.textContent).toContain("HTTP 429"); + expect(notice.textContent).not.toContain("拒绝"); + }); + + // Switching models cannot fix a rejected key, so the notice points at the + // key instead of offering to switch back. + it("sends a rejected key to the Provider instead of offering to switch back", async () => { + vi.spyOn(api, "saveProfile").mockResolvedValue({ profile: profile({ model: "new-model" }), reapplied: ["codex"], failures: null }); + vi.spyOn(api, "checkProfileModel").mockResolvedValue({ + profileId: "team-ppio", model: "new-model", status: "unavailable", reason: "key", httpStatus: 401, agents: ["codex"], + }); + renderPage([profile()]); + fireEvent.click(screen.getByRole("button", { name: "编辑 团队 PPIO" })); + fireEvent.change(screen.getByLabelText("模型"), { target: { value: "new-model" } }); + fireEvent.click(screen.getByRole("button", { name: "保存配置模版" })); + await waitFor(() => expect(screen.queryByRole("button", { name: "保存配置模版" })).toBeNull()); + fireEvent.click(screen.getByRole("button", { name: "检测 团队 PPIO 的模型" })); + + await screen.findByRole("alert"); + expect(screen.queryByRole("button", { name: /改回/ })).toBeNull(); + expect(screen.getByRole("button", { name: "检查模型服务的 API Key" })).toBeTruthy(); + }); + it("saves a Profile whose name was left blank", async () => { // The backend fills an empty label in from the existing value or the ID // (internal/profile/write.go:71-74), so requiring one here was stricter than diff --git a/frontend/src/pages/ProfilesPage.tsx b/frontend/src/pages/ProfilesPage.tsx index 48f5e2d5..e9fbe658 100644 --- a/frontend/src/pages/ProfilesPage.tsx +++ b/frontend/src/pages/ProfilesPage.tsx @@ -4,6 +4,7 @@ import { useNavigate } from "react-router-dom"; import { api, describeFailure } from "../backend/api"; import { CardUsers } from "../components/CardUsers"; +import { ModelCheckButton, ModelCheckNotice } from "../components/ModelCheckNotice"; import { PageScaffold } from "../components/PageScaffold"; import { ProviderModelPicker } from "../components/ProviderModelPicker"; import { ProviderSegment } from "../components/ProviderSegment"; @@ -13,7 +14,18 @@ import { confirmDelete } from "../state/confirmDelete"; import { byProviderCreatedAt, preferProviderWithKey } from "../state/ranking"; import { isConverterID } from "../state/conversion"; import { useWizard } from "../state/WizardContext"; -import { PROTOCOL_LABELS, type ProfileSummary, type ProtocolId, type ProviderId } from "../types/api"; +import { PROTOCOL_LABELS, type ProfileModelCheck, type ProfileSummary, type ProtocolId, type ProviderId } from "../types/api"; + +type SaveRequest = Parameters[0]; + +// The last model edit saved from this page, so a check that then fails can +// offer the model the Profile held before it. Only an edit on the same +// Provider qualifies: a model ID is not portable between Providers. +interface ModelEdit { + profileId: string; + previousModel: string; + request: SaveRequest; +} // A Profile no longer carries its own key: the Provider it points at owns one, // and asking twice for the same secret was the bug worth deleting. @@ -50,6 +62,10 @@ export function ProfilesPage() { const [busy, setBusy] = useState(false); const [failure, setFailure] = useState(""); const [applied, setApplied] = useState(""); + // A check runs only when the user presses 检测模型: it is a billed request. + const [checked, setChecked] = useState(null); + const [checkingId, setCheckingId] = useState(""); + const [lastEdit, setLastEdit] = useState(null); if (!status) { return ( @@ -61,8 +77,11 @@ export function ProfilesPage() { const profiles = status.profiles.filter((profile) => !isConverterID(profile.id)); const configurableAgents = status.catalog.filter((agent) => agent.configMode === "auto"); + // Desktop Agents follow Profiles too, and the model check names them. const nameOf = (agentId: string) => - status.catalog.find((item) => item.id === agentId)?.name || agentId; + status.catalog.find((item) => item.id === agentId)?.name + || status.desktopAgents?.find((app) => app.id === agentId)?.name + || agentId; const protocolOf = (profile: ProfileSummary) => profile.protocol || status.catalog.find((agent) => status.agents[agent.id]?.profileId === profile.id)?.protocol || ""; const providerHasKey = Boolean(editor && status.providers[editor.provider]?.has_key); @@ -139,6 +158,33 @@ export function ProfilesPage() { return { ...draft, provider, model, id, label }; }; + // Switching the Provider or model rewrites every Agent already following this + // Profile, so the outcome has to be reported rather than applied silently -- + // the Agent's own config file moves with it. Shared by the editor's save and + // the check result's "switch back", which is the same save with the old model. + const persist = async (request: SaveRequest) => { + const before = status.profiles.find((profile) => profile.id === request.id); + const result = await api.saveProfile(request); + // A result for this Profile describes the model it held before this save. + if (checked?.profileId === request.id) setChecked(null); + setLastEdit(before && before.provider === request.provider && before.model && before.model !== request.model + ? { profileId: request.id, previousModel: before.model, request } + : null); + const reapplied = result.reapplied ?? []; + const failures = Object.entries(result.failures ?? {}); + if (failures.length) { + setFailure(t("{agents} 重新应用失败：{message}", { + agents: failures.map(([agentId]) => nameOf(agentId)).join(locale === "en" ? ", " : "、"), + message: failures[0][1] ?? "", + })); + } else if (reapplied.length) { + setApplied(t("已重新应用到 {agents}", { + agents: reapplied.map(nameOf).join(locale === "en" ? ", " : "、"), + })); + } + await refreshStatus(); + }; + const save = async (event: FormEvent) => { event.preventDefault(); if (!editor || !canSave) return; @@ -146,10 +192,7 @@ export function ProfilesPage() { setFailure(""); setApplied(""); try { - // Switching the Provider or model rewrites every Agent already following - // this Profile, so the outcome has to be reported rather than applied - // silently -- the Agent's own config file moves with it. - const result = await api.saveProfile({ + await persist({ id: editor.id.trim().toLowerCase(), label: editor.label.trim(), provider: editor.provider, @@ -161,19 +204,6 @@ export function ProfilesPage() { configMode: "provider", protocol: editor.protocol, }); - const reapplied = result.reapplied ?? []; - const failures = Object.entries(result.failures ?? {}); - if (failures.length) { - setFailure(t("{agents} 重新应用失败：{message}", { - agents: failures.map(([agentId]) => nameOf(agentId)).join(locale === "en" ? ", " : "、"), - message: failures[0][1] ?? "", - })); - } else if (reapplied.length) { - setApplied(t("已重新应用到 {agents}", { - agents: reapplied.map(nameOf).join(locale === "en" ? ", " : "、"), - })); - } - await refreshStatus(); setEditor(null); } catch (error) { setFailure(describeFailure(error, t("无法保存配置模版"), t).message); @@ -182,6 +212,38 @@ export function ProfilesPage() { } }; + const checkModel = async (profileId: string) => { + setCheckingId(profileId); + setFailure(""); + try { + const check = await api.checkProfileModel(profileId); + const edit = lastEdit?.profileId === profileId && lastEdit.request.model === check.model ? lastEdit : null; + setChecked({ ...check, previousModel: edit?.previousModel }); + } catch (error) { + setFailure(describeFailure(error, t("无法检测模型"), t).message); + } finally { + setCheckingId(""); + } + }; + + // Saves the model back without checking it again; the user can press + // 检测模型 once more if they want the old one confirmed. + const revertModel = async () => { + if (!checked?.previousModel || lastEdit?.profileId !== checked.profileId) return; + const request = { ...lastEdit.request, model: checked.previousModel }; + setBusy(true); + setFailure(""); + setApplied(""); + try { + await persist(request); + setChecked(null); + } catch (error) { + setFailure(describeFailure(error, t("无法保存配置模版"), t).message); + } finally { + setBusy(false); + } + }; + const remove = async (profile: ProfileSummary, users: typeof configurableAgents) => { if (users.length) { setFailure(t("配置模版正在被 {agents} 使用，无法删除", { @@ -351,6 +413,19 @@ export function ProfilesPage() { {failure ?

{failure}

: null} {applied ?
{t("应用完成")}{applied}
: null} + {checked ? ( + profile.id === checked.profileId)?.provider || ""} + nameOf={nameOf} + returnTo="/profiles" + profileName={profiles.find((profile) => profile.id === checked.profileId)?.label} + busy={busy || Boolean(checkingId)} + onRevert={() => void revertModel()} + onRecheck={() => void checkModel(checked.profileId)} + onDismiss={() => setChecked(null)} + /> + ) : null} {!profiles.length && !editor ? (
@@ -401,6 +476,14 @@ export function ProfilesPage() { card no longer writes anything. */}
+ {profile.model ? ( + void checkModel(profile.id)} + ariaLabel={t("检测 {name} 的模型", { name: profile.label })} + /> + ) : null}
); diff --git a/frontend/src/styles/app.css b/frontend/src/styles/app.css index 8f067f2a..0f327a8a 100644 --- a/frontend/src/styles/app.css +++ b/frontend/src/styles/app.css @@ -1917,6 +1917,61 @@ .agent-manage-applied strong { font-size: 13px; } +/* Orange, not red: the save succeeded. What is reported is the model failing a + live request afterwards, which the user may knowingly accept. */ +.model-check-notice { + margin: 12px 0; + padding: 10px 12px; + display: flex; + gap: 9px; + align-items: flex-start; + border-radius: var(--radius-control); + background: var(--orange-soft); + color: var(--orange); + font-size: 13px; +} + +.model-check-notice > svg { flex-shrink: 0; margin-top: 1px; } + +.model-check-body { + min-width: 0; + display: grid; + gap: 4px; +} + +.model-check-body strong { font-size: 13px; } + +.model-check-body span { color: var(--text-primary); } + +/* The provider's own words: unbounded vocabulary, so it must wrap rather than + widen the page. */ +.model-check-detail { + color: var(--text-secondary); + overflow-wrap: anywhere; +} + +.model-check-actions { + margin-top: 4px; + display: flex; + flex-wrap: wrap; + gap: 6px; +} + +/* A check the user asked for that passed. */ +.model-check-notice.is-ok { + background: var(--green-soft); + color: var(--green); +} + +/* Sits in a card footer beside the Agent avatars, so it stays small. */ +.model-check-button { + flex-shrink: 0; + min-height: 26px; + padding: 0 9px; + gap: 5px; + font-size: 12px; +} + .agent-manage-applied pre { margin: 0; padding: 7px 9px; diff --git a/frontend/src/types/api.ts b/frontend/src/types/api.ts index db418617..8fb12f45 100644 --- a/frontend/src/types/api.ts +++ b/frontend/src/types/api.ts @@ -90,6 +90,19 @@ export type ProfileSummary = Omit & { + status: ProfileModelCheckStatus; + reason?: ProfileModelCheckReason; + agents: string[]; + // Filled in by the page, not the backend: the model this Profile held before + // an edit made on that page, offered as the way back from a failed check. + previousModel?: string; +}; // Mirrors SaveProviderResult: saving a Profile can rewrite the Agents bound to // it, so the outcome per Agent travels with the saved record. export type SaveProfileResult = Omit & { profile: ProfileSummary }; diff --git a/internal/app/profile_model_check.go b/internal/app/profile_model_check.go new file mode 100644 index 00000000..a9b9fe39 --- /dev/null +++ b/internal/app/profile_model_check.go @@ -0,0 +1,320 @@ +package app + +import ( + "context" + "slices" + "sort" + "strings" + + "github.com/MaimoryLab/BootAgent/internal/catalog" + "github.com/MaimoryLab/BootAgent/internal/desktopapp" + oneerrors "github.com/MaimoryLab/BootAgent/internal/errors" + profileStore "github.com/MaimoryLab/BootAgent/internal/profile" + "github.com/MaimoryLab/BootAgent/internal/provider" +) + +// ProfileModelCheck is the verdict of a live request to a Profile's model, +// sent only when the user asks for it. +// +// A model the provider lists can still be refused when an Agent calls it -- +// not served over that Agent's protocol, or not granted to the key -- and a +// Profile edit rewrites every Agent bound to it at once. The check lets the +// user find that out before an Agent fails. It is not run on save or apply: +// each check is a billed completion request, so it is the user's to send. +type ProfileModelCheck struct { + ProfileID string `json:"profileId"` + Model string `json:"model"` + // Status is "available" when every checked Agent's protocol answered, + // "unavailable" when the provider answered and refused, and "unverified" + // when no verdict could be had at all. The last is not a claim that the + // model is broken, and the UI must not word it as one. + Status string `json:"status"` + // Reason is the cause the remedy follows from: + // refused the provider refused this model on this protocol + // key the provider rejected the API key + // transient the provider answered but was busy or failing (408, 429, 5xx) + // network the provider could not be reached + // no_key no API key is saved for the Provider + // error the request could not be made (an endpoint misconfiguration) + // Empty when Status is "available". + Reason string `json:"reason,omitempty"` + Protocol string `json:"protocol,omitempty"` + // HTTPStatus is the status the provider answered with, 0 when it did not. + HTTPStatus int `json:"httpStatus,omitempty"` + // Detail is the provider's own explanation, cleaned and bounded. + Detail string `json:"detail,omitempty"` + // Agents are the Agents the verdict is about: on a failure, those whose + // requests to this model would fail; when available, those checked. Empty + // for a Profile nothing follows yet. + Agents []string `json:"agents"` +} + +const ( + ModelCheckAvailable = "available" + ModelCheckUnavailable = "unavailable" + ModelCheckUnverified = "unverified" +) + +// CheckProfileModel sends one live request per protocol the Profile's Agents +// use and reports the verdict. agentIDs narrows the check to those Agents -- +// the one a page is configuring, even before it follows the Profile; empty +// means every Agent bound to the Profile. +func (u *UseCases) CheckProfileModel(ctx context.Context, profileID string, agentIDs []string) (*ProfileModelCheck, error) { + if u == nil || u.provider == nil { + return nil, oneerrors.New(oneerrors.InternalError, "Profile service is not configured", oneerrors.WithStatus(501)) + } + if err := contextError(ctx, "Profile request was cancelled"); err != nil { + return nil, err + } + profile := u.profileByID(profileID) + if profile.ID == "" { + return nil, oneerrors.New(oneerrors.InvalidRequest, "Profile not found: "+strings.TrimSpace(profileID)) + } + // The local protocol adapter's own Profiles point at the adapter on + // 127.0.0.1, which forwards the probe upstream as a billed request whose + // verdict the adapter already established when it was enabled. + if isConverterID(profile.ID) { + return nil, oneerrors.New(oneerrors.InvalidRequest, "The protocol adapter's Profile is checked when the adapter is enabled") + } + if profileModel(profile) == "" { + return nil, oneerrors.New(oneerrors.InvalidRequest, "Profile has no model to check") + } + agents := slices.Clone(agentIDs) + if len(agents) == 0 { + bindings, err := u.profiles.ListAgentBindings() + if err != nil { + return nil, err + } + for agentID, binding := range bindings { + if binding.ProfileRef == profile.ID { + agents = append(agents, agentID) + } + } + } + check := u.checkProfileModel(ctx, profile, agents) + if check == nil { + // Cancelled, not answered: saying "unverified" would blame the network + // for the user closing the window. + return nil, contextError(ctx, "Profile request was cancelled") + } + return check, nil +} + +// checkProfileModel returns nil only when ctx was cancelled mid-check. +func (u *UseCases) checkProfileModel(ctx context.Context, profile profileStore.Profile, agentIDs []string) *ProfileModelCheck { + model := profileModel(profile) + sort.Strings(agentIDs) + agentIDs = slices.Compact(agentIDs) + check := &ProfileModelCheck{ProfileID: profile.ID, Model: model, Agents: []string{}} + + target, err := u.providers.Get(profile.Provider) + if err != nil { + check.Status, check.Reason, check.Agents = ModelCheckUnverified, "error", agentIDs + check.Detail = oneerrors.As(err).Message + return check + } + if strings.TrimSpace(target.APIKey) == "" { + check.Status, check.Reason, check.Agents = ModelCheckUnverified, "no_key", agentIDs + return check + } + + options := map[string][]string{} + needed := map[string]bool{} + for _, agentID := range agentIDs { + accepted := u.agentProtocolOptions(agentID, profile) + if len(accepted) == 0 { + // An Agent BootAgent does not configure has no request to check. + continue + } + options[agentID] = accepted + for _, protocolID := range accepted { + needed[protocolID] = true + } + } + // A Profile nothing follows yet is still about to be applied; check the + // protocol it is set up for, so the first Agent to use it is not the first + // to find out. + unbound := len(options) == 0 + if unbound { + protocolID := profile.Protocol + if protocolID == "" { + protocolID = provider.ProtocolOpenAI + } + needed[protocolID] = true + options[""] = []string{protocolID} + } + protocols := make([]string, 0, len(needed)) + for protocolID := range needed { + protocols = append(protocols, protocolID) + } + sort.Strings(protocols) + + results, err := u.probeProtocols(ctx, protocols, target.APIKey, model, target.BaseFor) + if ctx.Err() != nil { + // Cancelled, not answered: saying "unverified" would blame the network + // for the user closing the window. + return nil + } + if err != nil { + check.Status, check.Reason = ModelCheckUnverified, "error" + check.Detail = oneerrors.As(err).Message + check.Agents = agentIDs + return check + } + + var worst *provider.ProbeResult + worstProtocol := "" + for agentID, accepted := range options { + agentOK := false + var agentWorst *provider.ProbeResult + agentProtocol := "" + for _, protocolID := range accepted { + result, ok := results[protocolID] + if !ok { + continue + } + if result.OK { + agentOK = true + break + } + if agentWorst == nil || modelCheckRank(result) > modelCheckRank(*agentWorst) { + candidate := result + agentWorst, agentProtocol = &candidate, protocolID + } + } + if agentOK { + continue + } + if agentWorst == nil { + // Every probe for this Agent errored before a verdict. + unknown := provider.ProbeResult{} + agentWorst = &unknown + agentProtocol = accepted[0] + } + if agentID != "" { + check.Agents = append(check.Agents, agentID) + } + if worst == nil || modelCheckRank(*agentWorst) > modelCheckRank(*worst) { + worst, worstProtocol = agentWorst, agentProtocol + } + } + if worst == nil { + check.Status = ModelCheckAvailable + for agentID := range options { + if agentID != "" { + check.Agents = append(check.Agents, agentID) + } + } + sort.Strings(check.Agents) + if len(protocols) == 1 { + check.Protocol = protocols[0] + } + return check + } + sort.Strings(check.Agents) + check.Protocol = worstProtocol + check.HTTPStatus = worst.Status + check.Detail = worst.Detail + switch { + case !worst.Reachable && worst.Status == 0 && worst.ErrorCode == nil && worst.Message == "": + check.Status, check.Reason = ModelCheckUnverified, "error" + case !worst.Reachable: + check.Status, check.Reason = ModelCheckUnverified, "network" + check.HTTPStatus = 0 + case worst.ErrorCode != nil && *worst.ErrorCode == oneerrors.APIKeyRejected: + check.Status, check.Reason = ModelCheckUnavailable, "key" + case transientProbe(*worst): + check.Status, check.Reason = ModelCheckUnverified, "transient" + default: + check.Status, check.Reason = ModelCheckUnavailable, "refused" + } + if check.Detail == "" && worst.Message != "" && check.Reason != "refused" { + check.Detail = worst.Message + } + return check +} + +// modelCheckRank orders failures by how much the user can do about them: a +// refusal names the model, a rejected key names the credential, and a busy or +// unreachable endpoint says nothing about either. +func modelCheckRank(result provider.ProbeResult) int { + switch { + case !result.Reachable: + return 1 + case transientProbe(result): + return 2 + case result.ErrorCode != nil && *result.ErrorCode == oneerrors.APIKeyRejected: + return 3 + default: + return 4 + } +} + +// transientProbe reports an answer that is about the provider's state, not +// the model: a timeout, rate limiting, or a server error. Calling those +// "unavailable" would send the user to abandon a model that works -- and +// importing several Profiles at once is exactly what draws a 429. A 5xx the +// probe already read as "this endpoint does not serve the protocol" is a real +// refusal and stays one. +func transientProbe(result provider.ProbeResult) bool { + if !result.Reachable { + return false + } + if result.ErrorCode != nil && *result.ErrorCode == oneerrors.ProtocolUnsupported { + return false + } + return result.Status == 408 || result.Status == 429 || result.Status >= 500 +} + +// agentProtocolOptions lists the protocols an Agent's request to this Profile's +// model can travel over; the Agent works if any of them does. Empty means +// BootAgent writes no model into that Agent, so there is nothing to check. +func (u *UseCases) agentProtocolOptions(agentID string, profile profileStore.Profile) []string { + if definition, ok := desktopapp.DefinitionFor(agentID); ok { + if definition.SharedConfigAgentID != "" { + manifest, err := catalog.LoadEmbedded() + if err != nil { + return nil + } + shared, exists := manifest.Agents[definition.SharedConfigAgentID] + if !exists { + return nil + } + return adapterProtocolOptions(shared.ConfigAdapter) + } + if agentID == desktopapp.ClaudeDesktopID && profile.Protocol == provider.ProtocolOpenAI { + // A Chat Completions Profile reaches Claude Desktop through local + // protocol adaptation, which calls the upstream over Chat Completions. + return []string{provider.ProtocolOpenAI} + } + if definition.Protocol != "" { + return []string{definition.Protocol} + } + if definition.ConfigAdapter != "" { + return adapterProtocolOptions(definition.ConfigAdapter) + } + return nil + } + manifest, err := catalog.LoadEmbedded() + if err != nil { + return nil + } + agent, ok := manifest.Agents[agentID] + if !ok || agent.ConfigAdapter == "" { + return nil + } + return adapterProtocolOptions(agent.ConfigAdapter) +} + +// adapterProtocolOptions is the protocol a Profile edit or Apply writes for an +// adapter. It has to match the write exactly: checking a protocol the write +// does not use passes a model the Agent then cannot call. +// +// dsh is the case that makes this matter. Its pi-ai route can speak either +// OpenAI wire protocol, and the install probe accepts either, but +// writeManagedAgentConfig is only told which one by the install path; a +// Profile save and an Apply always write Chat Completions. Accepting Responses +// here passed a Responses-only model while dsh was rewritten to completions. +func adapterProtocolOptions(adapter string) []string { + return []string{provider.ProtocolForAdapter(adapter)} +} diff --git a/internal/app/profile_model_check_test.go b/internal/app/profile_model_check_test.go new file mode 100644 index 00000000..4b69af64 --- /dev/null +++ b/internal/app/profile_model_check_test.go @@ -0,0 +1,433 @@ +package app + +import ( + "context" + "encoding/json" + "errors" + "io" + "net/http" + "slices" + "strings" + "sync" + "sync/atomic" + "testing" + + oneerrors "github.com/MaimoryLab/BootAgent/internal/errors" + profileStore "github.com/MaimoryLab/BootAgent/internal/profile" + "github.com/MaimoryLab/BootAgent/internal/provider" +) + +// modelGateway answers each probe by the model in its body, so a test can make +// one model work and another fail on the same endpoint. +type modelGateway struct { + mu sync.Mutex + verdicts map[string]func(path string) (*http.Response, error) + requests []string + // onRequest runs inside each probe, for tests that look at what is held while + // the request is in flight. + onRequest func() +} + +func (g *modelGateway) Do(request *http.Request) (*http.Response, error) { + var body struct { + Model string `json:"model"` + } + if request.Body != nil { + data, _ := io.ReadAll(request.Body) + _ = json.Unmarshal(data, &body) + } + g.mu.Lock() + g.requests = append(g.requests, request.URL.Path+" "+body.Model) + verdict := g.verdicts[body.Model] + onRequest := g.onRequest + g.mu.Unlock() + if onRequest != nil { + onRequest() + } + if verdict == nil { + return appProviderResponse(http.StatusOK, `{}`), nil + } + return verdict(request.URL.Path) +} + +func (g *modelGateway) count() int { + g.mu.Lock() + defer g.mu.Unlock() + return len(g.requests) +} + +func (g *modelGateway) reset() { + g.mu.Lock() + g.requests = nil + g.mu.Unlock() +} + +const checkKey = "sk-check-secret" + +func refusing(status int, body string) func(string) (*http.Response, error) { + return func(string) (*http.Response, error) { return appProviderResponse(status, body), nil } +} + +// gatewayCore is a Profile on a custom gateway, followed by Claude Code +// (Anthropic Messages) and Codex (Responses) -- the pair the report describes. +func gatewayCore(t *testing.T, gateway *modelGateway) *UseCases { + t.Helper() + core := activationCore(t, t.TempDir(), provider.NewClient(gateway), "linux") + if _, err := core.SaveProvider(context.Background(), provider.Entry{ + ID: "gw", Name: "Gateway", BaseURL: "https://gw.example/v1", AnthropicBaseURL: "https://gw.example/v1", APIKey: checkKey, + }, true, false); err != nil { + t.Fatal(err) + } + if _, err := core.SaveProfile(context.Background(), SaveProfileOptions{ID: "team", Label: "Team", Provider: "gw", Model: "good-model", ConfigMode: "provider"}); err != nil { + t.Fatal(err) + } + for _, agentID := range []string{"claude-code", "codex"} { + if _, err := core.ActivateAgent(context.Background(), ActivateAgentOptions{AgentID: agentID, Provider: "gw", ProfileID: "team"}); err != nil { + t.Fatalf("activate %s: %v", agentID, err) + } + } + gateway.reset() + return core +} + +func saveModel(t *testing.T, core *UseCases, model string) SaveProfileResult { + t.Helper() + result, err := core.SaveProfile(context.Background(), SaveProfileOptions{ID: "team", Label: "Team", Provider: "gw", Model: model, ConfigMode: "provider"}) + if err != nil { + t.Fatalf("SaveProfile(%s) error = %v", model, err) + } + return result +} + +// checkModel saves model on the team Profile, then runs the check the user asks +// for from the UI. +func checkModel(t *testing.T, core *UseCases, model string) *ProfileModelCheck { + t.Helper() + saveModel(t, core, model) + check, err := core.CheckProfileModel(context.Background(), "team", nil) + if err != nil { + t.Fatalf("CheckProfileModel(%s) error = %v", model, err) + } + if check == nil { + t.Fatalf("CheckProfileModel(%s) = nil, want a verdict", model) + } + return check +} + +// Each check is a billed completion request, so it is the user's to send. +// Saving a Profile -- even one that changes the model every Agent follows -- +// and applying one send nothing to the model. +func TestSaveAndApplyDoNotCheckTheModel(t *testing.T) { + gateway := &modelGateway{verdicts: map[string]func(string) (*http.Response, error){ + "bad-model": refusing(http.StatusNotFound, `{"error":{"message":"no such model"}}`), + }} + core := gatewayCore(t, gateway) + result := saveModel(t, core, "bad-model") + if !slices.Equal(result.Reapplied, []string{"claude-code", "codex"}) { + t.Fatalf("reapplied = %v, want both Agents", result.Reapplied) + } + if _, err := core.ActivateAgent(context.Background(), ActivateAgentOptions{AgentID: "claude-code", Provider: "gw", ProfileID: "team"}); err != nil { + t.Fatal(err) + } + for _, request := range gateway.requests { + if strings.HasSuffix(request, " bad-model") { + t.Fatalf("requests = %v; save and apply must not probe the model", gateway.requests) + } + } +} + +// The report: a model the gateway lists but refuses broke Claude Code and +// Codex together. Asked for, the check names the model, both Agents, and the +// provider's own words. +func TestModelCheckNamesARefusedModelAndTheAgentsItBreaks(t *testing.T) { + gateway := &modelGateway{verdicts: map[string]func(string) (*http.Response, error){ + "claude-opus-5-5": refusing(http.StatusNotFound, `{"type":"error","error":{"type":"not_found_error","message":"model: claude-opus-5-5 does not exist or you do not have access to it (key `+checkKey+`)"}}`), + }} + core := gatewayCore(t, gateway) + + check := checkModel(t, core, "claude-opus-5-5") + if check.Status != ModelCheckUnavailable || check.Reason != "refused" { + t.Errorf("status/reason = %s/%s, want unavailable/refused", check.Status, check.Reason) + } + if check.Model != "claude-opus-5-5" { + t.Errorf("model = %q, want claude-opus-5-5", check.Model) + } + if !slices.Equal(check.Agents, []string{"claude-code", "codex"}) { + t.Errorf("agents = %v, want both Agents", check.Agents) + } + if check.HTTPStatus != http.StatusNotFound { + t.Errorf("http status = %d, want 404", check.HTTPStatus) + } + if !strings.Contains(check.Detail, "does not exist or you do not have access") { + t.Errorf("detail = %q, want the provider's own explanation", check.Detail) + } + if strings.Contains(check.Detail, checkKey) { + t.Errorf("detail = %q leaks the API key the gateway echoed back", check.Detail) + } +} + +// A model that answers is reported as such, naming the Agents it was checked +// for, so a check the user asked for always ends with an answer. +func TestModelCheckReportsAWorkingModelAsAvailable(t *testing.T) { + gateway := &modelGateway{} + core := gatewayCore(t, gateway) + check := checkModel(t, core, "good-model") + if check.Status != ModelCheckAvailable || check.Reason != "" { + t.Fatalf("check = %#v, want available with no reason", check) + } + if !slices.Equal(check.Agents, []string{"claude-code", "codex"}) { + t.Errorf("agents = %v, want the Agents checked", check.Agents) + } + if gateway.count() == 0 { + t.Error("no request was sent") + } +} + +// Each Agent is checked over the protocol it actually sends. Here only +// Anthropic Messages refuses, so only Claude Code is named. +func TestModelCheckNamesOnlyTheAgentsWhoseProtocolRefuses(t *testing.T) { + gateway := &modelGateway{verdicts: map[string]func(string) (*http.Response, error){ + "chat-only": func(path string) (*http.Response, error) { + if strings.HasSuffix(path, "/messages") { + return appProviderResponse(http.StatusBadRequest, `{"error":{"message":"model not available on messages"}}`), nil + } + return appProviderResponse(http.StatusOK, `{}`), nil + }, + }} + core := gatewayCore(t, gateway) + check := checkModel(t, core, "chat-only") + if check.Status != ModelCheckUnavailable || !slices.Equal(check.Agents, []string{"claude-code"}) { + t.Fatalf("check = %#v, want only claude-code", check) + } + if check.Protocol != provider.ProtocolAnthropic { + t.Errorf("protocol = %q, want anthropic", check.Protocol) + } +} + +// An unreachable provider is not evidence against the model. The result says +// "could not verify", not "unavailable", so the UI does not tell the user to +// abandon a model that may be fine. +func TestModelCheckReportsAnUnreachableProviderAsUnverified(t *testing.T) { + gateway := &modelGateway{verdicts: map[string]func(string) (*http.Response, error){ + "new-model": func(string) (*http.Response, error) { return nil, errors.New("dial tcp: connection refused") }, + }} + core := gatewayCore(t, gateway) + check := checkModel(t, core, "new-model") + if check.Status != ModelCheckUnverified || check.Reason != "network" { + t.Fatalf("check = %#v, want unverified/network", check) + } + if check.HTTPStatus != 0 { + t.Errorf("http status = %d, want 0 for a request that got no answer", check.HTTPStatus) + } +} + +// A rate limit or a server error is about the provider's state, not the model. +// It is reported as unverified so the user is not told to abandon a model that +// works. A 5xx that says the endpoint does not serve the protocol is a real +// refusal and stays one. +func TestModelCheckDoesNotBlameTheModelForABusyProvider(t *testing.T) { + for _, status := range []int{http.StatusRequestTimeout, http.StatusTooManyRequests, http.StatusInternalServerError, http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout} { + gateway := &modelGateway{verdicts: map[string]func(string) (*http.Response, error){ + "new-model": refusing(status, `{"error":{"message":"try again later"}}`), + }} + core := gatewayCore(t, gateway) + if check := checkModel(t, core, "new-model"); check.Status != ModelCheckUnverified || check.Reason != "transient" { + t.Errorf("HTTP %d: check = %#v, want unverified/transient", status, check) + } + } + gateway := &modelGateway{verdicts: map[string]func(string) (*http.Response, error){ + "new-model": refusing(http.StatusInternalServerError, `{"error":{"message":"this model does not support endpoint /v1/messages"}}`), + }} + core := gatewayCore(t, gateway) + if check := checkModel(t, core, "new-model"); check.Reason != "refused" { + t.Errorf("unsupported-endpoint 500: check = %#v, want refused", check) + } +} + +// A rejected key is named as a key problem, because changing the model would +// not fix it. +func TestModelCheckNamesARejectedKey(t *testing.T) { + gateway := &modelGateway{verdicts: map[string]func(string) (*http.Response, error){ + "new-model": refusing(http.StatusUnauthorized, `{"error":{"message":"invalid api key"}}`), + }} + core := gatewayCore(t, gateway) + if check := checkModel(t, core, "new-model"); check.Status != ModelCheckUnavailable || check.Reason != "key" { + t.Fatalf("check = %#v, want unavailable/key", check) + } +} + +// With no key saved there is nothing to send. That is reported, not probed. +func TestModelCheckReportsAMissingKeyWithoutARequest(t *testing.T) { + gateway := &modelGateway{} + core := activationCore(t, t.TempDir(), provider.NewClient(gateway), "linux") + if _, err := core.SaveProvider(context.Background(), provider.Entry{ID: "nokey", Name: "No key", BaseURL: "https://gw.example/v1"}, true, false); err != nil { + t.Fatal(err) + } + if _, err := core.SaveProfile(context.Background(), SaveProfileOptions{ID: "p", Label: "P", Provider: "nokey", Model: "m", ConfigMode: "provider"}); err != nil { + t.Fatal(err) + } + check, err := core.CheckProfileModel(context.Background(), "p", nil) + if err != nil { + t.Fatal(err) + } + if check == nil || check.Reason != "no_key" || check.Status != ModelCheckUnverified { + t.Fatalf("check = %#v, want unverified/no_key", check) + } + if gateway.count() != 0 { + t.Errorf("sent %d requests without a key", gateway.count()) + } +} + +// A Profile nothing follows yet is checked over the protocol it is set up +// for, and names no Agents. +func TestModelCheckCoversAProfileNoAgentFollowsYet(t *testing.T) { + gateway := &modelGateway{verdicts: map[string]func(string) (*http.Response, error){ + "bad": refusing(http.StatusNotFound, `{"error":{"message":"no such model"}}`), + }} + core := activationCore(t, t.TempDir(), provider.NewClient(gateway), "linux") + if _, err := core.SaveProvider(context.Background(), provider.Entry{ID: "gw", Name: "Gateway", BaseURL: "https://gw.example/v1", AnthropicBaseURL: "https://gw.example/v1", APIKey: checkKey}, true, false); err != nil { + t.Fatal(err) + } + if _, err := core.SaveProfile(context.Background(), SaveProfileOptions{ID: "fresh", Label: "Fresh", Provider: "gw", Model: "bad", ConfigMode: "provider", Protocol: provider.ProtocolAnthropic}); err != nil { + t.Fatal(err) + } + check, err := core.CheckProfileModel(context.Background(), "fresh", nil) + if err != nil { + t.Fatal(err) + } + if check == nil || check.Status != ModelCheckUnavailable || len(check.Agents) != 0 || check.Protocol != provider.ProtocolAnthropic { + t.Fatalf("check = %#v, want unavailable on anthropic with no Agents", check) + } +} + +// The check is a network request, so it must run with the configuration write +// lock released: a slow provider would otherwise stall every other write. +func TestModelCheckRunsWithTheWriteLockReleased(t *testing.T) { + gateway := &modelGateway{} + core := gatewayCore(t, gateway) + var heldDuringProbe atomic.Bool + var probed atomic.Bool + gateway.mu.Lock() + gateway.onRequest = func() { + probed.Store(true) + if !core.writeMu.TryLock() { + heldDuringProbe.Store(true) + return + } + core.writeMu.Unlock() + } + gateway.mu.Unlock() + checkModel(t, core, "another-model") + if !probed.Load() { + t.Fatal("the model was not checked") + } + if heldDuringProbe.Load() { + t.Error("writeMu was held while the provider was being asked") + } +} + +// Closing the window mid-check is not a network failure, and must not be +// reported as one: the check ends with a cancellation, not a verdict. +func TestModelCheckIsCancelledNotUnverified(t *testing.T) { + gateway := &modelGateway{} + core := gatewayCore(t, gateway) + saveModel(t, core, "slow") + ctx, cancel := context.WithCancel(context.Background()) + gateway.mu.Lock() + gateway.verdicts = map[string]func(string) (*http.Response, error){ + "slow": func(string) (*http.Response, error) { + cancel() + return nil, context.Canceled + }, + } + gateway.mu.Unlock() + check, err := core.CheckProfileModel(ctx, "team", nil) + if check != nil || oneerrors.As(err).Code != oneerrors.Timeout { + t.Fatalf("check = %#v, err = %v; want a cancellation error and no verdict", check, err) + } +} + +// The review's reproduction: a model that answers only over Responses, on a +// Profile dsh follows. dsh is written with the Chat Completions route, so dsh +// cannot call that model, and the check has to say so. +func TestModelCheckWarnsWhenDSHIsWrittenWithAProtocolTheModelRefuses(t *testing.T) { + gateway := &modelGateway{verdicts: map[string]func(string) (*http.Response, error){ + "responses-only": func(path string) (*http.Response, error) { + if strings.HasSuffix(path, "/chat/completions") { + return appProviderResponse(http.StatusBadRequest, `{"error":{"message":"use the responses API for this model"}}`), nil + } + return appProviderResponse(http.StatusOK, `{}`), nil + }, + }} + core := activationCore(t, t.TempDir(), provider.NewClient(gateway), "linux") + if _, err := core.SaveProvider(context.Background(), provider.Entry{ID: "gw", Name: "Gateway", BaseURL: "https://gw.example/v1", APIKey: checkKey}, true, false); err != nil { + t.Fatal(err) + } + if _, err := core.SaveProfile(context.Background(), SaveProfileOptions{ID: "team", Label: "Team", Provider: "gw", Model: "good-model", ConfigMode: "provider"}); err != nil { + t.Fatal(err) + } + if _, err := core.ActivateAgent(context.Background(), ActivateAgentOptions{AgentID: "dsh", Provider: "gw", ProfileID: "team"}); err != nil { + t.Fatal(err) + } + check := checkModel(t, core, "responses-only") + if check.Status != ModelCheckUnavailable || !slices.Equal(check.Agents, []string{"dsh"}) || check.Protocol != provider.ProtocolOpenAI { + t.Fatalf("check = %#v, want dsh named on openai", check) + } +} + +// A page configuring one Agent checks for that Agent alone, and an unknown +// Profile, the adapter's own Profiles, and a Profile without a model are +// refused rather than answered with an empty verdict. +func TestCheckProfileModelNarrowsToTheNamedAgentAndRefusesWhatItCannotCheck(t *testing.T) { + gateway := &modelGateway{verdicts: map[string]func(string) (*http.Response, error){ + "good-model": func(path string) (*http.Response, error) { + if strings.HasSuffix(path, "/responses") { + return appProviderResponse(http.StatusBadRequest, `{"error":{"message":"responses not served for this model"}}`), nil + } + return appProviderResponse(http.StatusOK, `{}`), nil + }, + }} + core := gatewayCore(t, gateway) + check, err := core.CheckProfileModel(context.Background(), "team", []string{"claude-code"}) + if err != nil || check == nil || check.Status != ModelCheckAvailable || !slices.Equal(check.Agents, []string{"claude-code"}) { + t.Fatalf("claude-code check = %#v, %v; want available for claude-code", check, err) + } + check, err = core.CheckProfileModel(context.Background(), "team", nil) + if err != nil || check == nil || check.Status != ModelCheckUnavailable || !slices.Equal(check.Agents, []string{"codex"}) { + t.Fatalf("bound-Agents check = %#v, %v; want codex named", check, err) + } + if _, err := core.CheckProfileModel(context.Background(), "missing", nil); oneerrors.As(err).Code != oneerrors.InvalidRequest { + t.Fatalf("unknown Profile error = %v, want InvalidRequest", err) + } + if _, err := core.SaveProfile(context.Background(), SaveProfileOptions{ID: "empty", Label: "Empty", Provider: "gw", ConfigMode: "provider"}); err == nil { + if _, err := core.CheckProfileModel(context.Background(), "empty", nil); oneerrors.As(err).Code != oneerrors.InvalidRequest { + t.Fatalf("model-less Profile error = %v, want InvalidRequest", err) + } + } +} + +// Which protocol stands for an Agent: the one a Profile save or Apply writes. +// dsh can speak either OpenAI wire protocol, but those writes give it Chat +// Completions, so that is what is checked. A Chat Completions Profile reaches +// Claude Desktop through local adaptation, so its upstream is Chat Completions, +// not Anthropic Messages -- checking the latter would warn about a working setup. +func TestAgentProtocolOptionsFollowWhatEachAgentSends(t *testing.T) { + core := &UseCases{} + for _, test := range []struct { + agentID, profileProtocol string + want []string + }{ + {"dsh", "", []string{provider.ProtocolOpenAI}}, + {"dsh-desktop", "", []string{provider.ProtocolOpenAI}}, + {"claude-code", "", []string{provider.ProtocolAnthropic}}, + {"codex", "", []string{provider.ProtocolResponses}}, + {"claude-desktop", provider.ProtocolOpenAI, []string{provider.ProtocolOpenAI}}, + {"claude-desktop", provider.ProtocolAnthropic, []string{provider.ProtocolAnthropic}}, + {"not-an-agent", "", nil}, + } { + got := core.agentProtocolOptions(test.agentID, profileStore.Profile{Protocol: test.profileProtocol}) + if !slices.Equal(got, test.want) { + t.Errorf("agentProtocolOptions(%s, %q) = %v, want %v", test.agentID, test.profileProtocol, got, test.want) + } + } +} diff --git a/internal/binding/services.go b/internal/binding/services.go index 18610b2d..567fe1ce 100644 --- a/internal/binding/services.go +++ b/internal/binding/services.go @@ -619,6 +619,19 @@ func (s *ProfileService) SaveProfile(ctx context.Context, request SaveProfileReq }) } +// CheckModel sends a live request to a Profile's model when the user asks for +// it. Saving and applying a Profile never check on their own: each check is a +// billed request. +func (s *ProfileService) CheckModel(ctx context.Context, request CheckProfileModelRequest) (*app.ProfileModelCheck, error) { + if err := contextError(ctx); err != nil { + return nil, err + } + if s == nil || s.core == nil { + return nil, notReady("Profile service is not configured") + } + return s.core.CheckProfileModel(ctx, request.ProfileID, request.Agents) +} + func (s *ProfileService) DeleteProfile(ctx context.Context, request ProviderIDRequest) (ProviderMutationResponse, error) { if err := contextError(ctx); err != nil { return ProviderMutationResponse{}, err @@ -816,6 +829,12 @@ type SaveProfileRequest struct { Protocol string `json:"protocol"` } +type CheckProfileModelRequest struct { + ProfileID string `json:"profile_id"` + // Agents narrows the check; empty means every Agent bound to the Profile. + Agents []string `json:"agents"` +} + func contextError(ctx context.Context) error { if err := ctx.Err(); err != nil { return oneerrors.New(oneerrors.Timeout, "Request was cancelled", oneerrors.WithRetryable(true), oneerrors.WithCause(err)) diff --git a/internal/binding/services_test.go b/internal/binding/services_test.go index 03f0afcc..416906df 100644 --- a/internal/binding/services_test.go +++ b/internal/binding/services_test.go @@ -51,7 +51,7 @@ func TestServiceMethodAllowlist(t *testing.T) { {&StatusService{}, []string{"GetStatus"}}, {&ProviderService{}, []string{"DeleteProvider", "GetProvider", "ListModels", "OpenGitHub", "OpenHelp", "OpenRegistration", "Probe", "SaveProvider"}}, {&AgentService{}, []string{"Activate", "Install", "Launch", "MigrateConversations", "PreviewUninstall", "Uninstall", "Update"}}, - {&ProfileService{}, []string{"DeleteProfile", "ListProfiles", "SaveProfile"}}, + {&ProfileService{}, []string{"CheckModel", "DeleteProfile", "ListProfiles", "SaveProfile"}}, {&RuntimeService{}, []string{"GetSettings", "InstallRuntime", "ListRuntimes", "SaveSettings"}}, {&DesktopAgentService{}, []string{"AssessProfile", "Configure", "ConfigureWithConversion", "GetStatus", "Install", "Open"}}, {&TransferService{}, []string{"ExportV2", "PreviewV2", "ApplyV2", "ApplyV2WithOptions", "Read", "ReadBytes", "Write", "WriteBytes"}}, diff --git a/internal/provider/client.go b/internal/provider/client.go index 5ca41985..436987f1 100644 --- a/internal/provider/client.go +++ b/internal/provider/client.go @@ -48,6 +48,9 @@ type ProbeResult struct { ErrorCode *string `json:"error_code"` Retryable bool `json:"retryable"` Protocol *string `json:"protocol"` + // Detail is the provider's own explanation of a refused request, cleaned + // and bounded. Empty on success and on transport failures. + Detail string `json:"detail,omitempty"` } type ModelsResult struct { @@ -110,7 +113,58 @@ func (c *Client) Probe(ctx context.Context, protocol, providerID, apiKey, model, }, nil } body, _, _ := c.readBody(response.Body) - return classifyHTTPProbe(response.StatusCode, string(body), protocol, requestModel), nil + result := classifyHTTPProbe(response.StatusCode, string(body), protocol, requestModel) + result.Detail = upstreamDetail(body, apiKey) + return result, nil +} + +// maxDetailRunes bounds the provider's own explanation carried back to the UI: +// enough for "model X does not exist or you do not have access", not a page. +const maxDetailRunes = 300 + +// upstreamDetail is the provider's explanation of a refused request, which is +// the part of a failure the user can act on. "HTTP 404" does not say whether +// the model, the key, or the endpoint was refused; the provider's message +// usually does. +// +// The key is scrubbed even though the body comes from the provider, because +// some gateways echo the request back into the error. +func upstreamDetail(body []byte, apiKey string) string { + text := "" + var parsed struct { + Error json.RawMessage `json:"error"` + Message string `json:"message"` + Detail string `json:"detail"` + } + if json.Unmarshal(body, &parsed) == nil { + var nested struct { + Message string `json:"message"` + } + var flat string + switch { + case json.Unmarshal(parsed.Error, &nested) == nil && nested.Message != "": + text = nested.Message + case json.Unmarshal(parsed.Error, &flat) == nil && flat != "": + text = flat + case parsed.Message != "": + text = parsed.Message + case parsed.Detail != "": + text = parsed.Detail + } + } + if text == "" && !bytes.HasPrefix(bytes.TrimSpace(body), []byte("<")) { + // Plain text, but not an HTML error page, which says nothing useful. + text = string(body) + } + if strings.TrimSpace(apiKey) != "" { + text = strings.ReplaceAll(text, apiKey, "[redacted]") + } + text = strings.Join(strings.FieldsFunc(text, func(r rune) bool { return r < 0x20 || r == 0x7f }), " ") + text = strings.Join(strings.Fields(text), " ") + if runes := []rune(text); len(runes) > maxDetailRunes { + text = string(runes[:maxDetailRunes]) + "…" + } + return text } func (c *Client) ListModels(ctx context.Context, providerID, apiKey, customBase string) (ModelsResult, error) { diff --git a/internal/provider/client_test.go b/internal/provider/client_test.go index 39b3c41f..eddf1508 100644 --- a/internal/provider/client_test.go +++ b/internal/provider/client_test.go @@ -274,3 +274,29 @@ func TestListModelsMapsBodyReadCancellationToTimeout(t *testing.T) { t.Fatalf("body cancellation result = %#v, err=%v", result, err) } } + +// The provider's explanation is what makes a refused probe actionable, but it is +// untrusted text bound for the UI: it is extracted from every error shape +// gateways use, an HTML error page is dropped as saying nothing, control +// characters cannot reach the screen, the key is scrubbed, and length is bounded. +func TestUpstreamDetailExtractsCleansAndBounds(t *testing.T) { + long := strings.Repeat("x", maxDetailRunes+50) + for _, test := range []struct { + name, body, want string + }{ + {"anthropic nested", `{"type":"error","error":{"type":"not_found_error","message":"model: m does not exist"}}`, "model: m does not exist"}, + {"flat string error", `{"error":"model not found"}`, "model not found"}, + {"top-level message", `{"message":"quota exceeded"}`, "quota exceeded"}, + {"fastapi detail", `{"detail":"Not Found"}`, "Not Found"}, + {"plain text", "upstream says no\n", "upstream says no"}, + {"html page", "502 Bad Gateway", ""}, + {"control characters", "{\"error\":{\"message\":\"bad\\u001b[31m model\\u0007\"}}", "bad [31m model"}, + {"key echoed back", `{"error":{"message":"key sk-live-123 is not allowed"}}`, "key [redacted] is not allowed"}, + {"bounded", `{"error":{"message":"` + long + `"}}`, strings.Repeat("x", maxDetailRunes) + "…"}, + {"empty", ``, ""}, + } { + if got := upstreamDetail([]byte(test.body), "sk-live-123"); got != test.want { + t.Errorf("%s: upstreamDetail = %q, want %q", test.name, got, test.want) + } + } +} From e743baf959796391d6f69883488bccb5aafacba4 Mon Sep 17 00:00:00 2001 From: yujiezhang-ops Date: Fri, 9 Oct 2026 13:56:10 +0800 Subject: [PATCH 03/18] fix: show only the editor while editing a Provider or Profile Opening the editor on the Providers, Profiles and Agent Profile pages left the full list and the last save's notices rendered below the form, which read as the previous page appended to the editor. The editor now replaces the list until it is saved or cancelled, and opens scrolled to the top so a card's editor is not left above the fold. Co-Authored-By: Claude Opus 5 (1M context) --- frontend/src/pages/AgentProfilePage.test.tsx | 11 +++++++++ frontend/src/pages/AgentProfilePage.tsx | 6 +++-- frontend/src/pages/ProfilesPage.test.tsx | 11 +++++++++ frontend/src/pages/ProfilesPage.tsx | 24 +++++++++++++++----- frontend/src/pages/ProvidersPage.test.tsx | 19 ++++++++++++++++ frontend/src/pages/ProvidersPage.tsx | 19 +++++++++++++--- 6 files changed, 79 insertions(+), 11 deletions(-) diff --git a/frontend/src/pages/AgentProfilePage.test.tsx b/frontend/src/pages/AgentProfilePage.test.tsx index 966841ac..0d6a23fc 100644 --- a/frontend/src/pages/AgentProfilePage.test.tsx +++ b/frontend/src/pages/AgentProfilePage.test.tsx @@ -173,6 +173,17 @@ describe("AgentProfilePage", () => { await waitFor(() => expect(refreshStatus).toHaveBeenCalled()); }); + // The editor is a page of its own; the Profile list returns when it closes. + it("shows only the editor while a Profile is being edited", async () => { + renderPage([profile()]); + expect(screen.getByTestId("agent-profile-team-ppio")).toBeTruthy(); + fireEvent.click(screen.getByRole("button", { name: "编辑 团队 PPIO" })); + expect(await screen.findByText("编辑配置模版")).toBeTruthy(); + expect(screen.queryByTestId("agent-profile-team-ppio")).toBeNull(); + fireEvent.click(screen.getByRole("button", { name: "取消" })); + expect(screen.getByTestId("agent-profile-team-ppio")).toBeTruthy(); + }); + // A check is a billed request, so it is the user's to send: an inline model // edit and an Apply send none. it("does not check the model on an inline edit or on apply", async () => { diff --git a/frontend/src/pages/AgentProfilePage.tsx b/frontend/src/pages/AgentProfilePage.tsx index 6afba3d1..e8fd979d 100644 --- a/frontend/src/pages/AgentProfilePage.tsx +++ b/frontend/src/pages/AgentProfilePage.tsx @@ -389,7 +389,7 @@ export function AgentProfilePage() { )} > {failure ?
{failure}
: null} - {checked ? ( + {checked && !draft ? ( profile.id === checked.profileId)?.provider || ""} @@ -492,7 +492,9 @@ export function AgentProfilePage() { ) : null} - {profiles.length ? ( + {/* The editor is a page of its own; the list returns when it closes. + Rendering it below the form read as the previous page appended. */} + {draft ? null : profiles.length ? ( <>
{profiles.map((profile) => { diff --git a/frontend/src/pages/ProfilesPage.test.tsx b/frontend/src/pages/ProfilesPage.test.tsx index 6fa2c7e5..7ab6e98c 100644 --- a/frontend/src/pages/ProfilesPage.test.tsx +++ b/frontend/src/pages/ProfilesPage.test.tsx @@ -278,6 +278,17 @@ describe("ProfilesPage", () => { await waitFor(() => expect(screen.getByText(/Codex.*重新应用失败.*写入失败/)).toBeTruthy()); }); + // The editor is a page of its own. The list used to stay rendered below the + // form, which read as the previous page appended to this one. + it("shows only the editor while a Profile is being edited", () => { + renderPage([profile()]); + expect(screen.getByTestId("profile-team-ppio")).toBeTruthy(); + fireEvent.click(screen.getByRole("button", { name: "编辑 团队 PPIO" })); + expect(screen.queryByTestId("profile-team-ppio")).toBeNull(); + fireEvent.click(screen.getByRole("button", { name: "取消" })); + expect(screen.getByTestId("profile-team-ppio")).toBeTruthy(); + }); + // A check is a billed request, so saving never sends one; the card's // 检测模型 does, for every Agent following the Profile. it("checks the model only when 检测模型 is pressed", async () => { diff --git a/frontend/src/pages/ProfilesPage.tsx b/frontend/src/pages/ProfilesPage.tsx index e9fbe658..53a45955 100644 --- a/frontend/src/pages/ProfilesPage.tsx +++ b/frontend/src/pages/ProfilesPage.tsx @@ -1,5 +1,5 @@ import { KeyRound, Layers, Pencil, Plus, Save, Trash2, X } from "lucide-react"; -import { useState, type FormEvent } from "react"; +import { useEffect, useRef, useState, type FormEvent } from "react"; import { useNavigate } from "react-router-dom"; import { api, describeFailure } from "../backend/api"; @@ -66,6 +66,14 @@ export function ProfilesPage() { const [checked, setChecked] = useState(null); const [checkingId, setCheckingId] = useState(""); const [lastEdit, setLastEdit] = useState(null); + const editorRef = useRef(null); + const editing = editor ? editor.originalId || "new" : ""; + + // The editor takes the page over, so it opens at the top. Clicking 编辑 on a + // card further down the list otherwise left the form above the fold. + useEffect(() => { + if (editing) editorRef.current?.scrollIntoView?.({ block: "start" }); + }, [editing]); if (!status) { return ( @@ -110,6 +118,7 @@ export function ProfilesPage() { const openCreate = () => { const [provider = "ppio", providerMeta] = preferProviderWithKey(byProviderCreatedAt(Object.fromEntries(Object.entries(status.providers).filter(([providerId]) => !isConverterID(providerId))))) || []; setFailure(""); + setApplied(""); setEditor({ id: suggestProfileID(provider), label: suggestProfileLabel(provider), @@ -285,7 +294,7 @@ export function ProfilesPage() { > {editor ? ( -
void save(event)}> + void save(event)}>
{editor.originalId ? t("编辑 {name}", { name: editor.label || editor.id }) : t("创建配置模版")} {/* disabled while busy: without it a double-click sent two diff --git a/frontend/src/pages/ProvidersPage.test.tsx b/frontend/src/pages/ProvidersPage.test.tsx index b48c4f6b..3c7022c2 100644 --- a/frontend/src/pages/ProvidersPage.test.tsx +++ b/frontend/src/pages/ProvidersPage.test.tsx @@ -145,6 +145,25 @@ describe("ProvidersPage", () => { await waitFor(() => expect(save).toHaveBeenCalledWith(expect.objectContaining({ id: "ppio", name: "PPIO Cloud", api_key: "sk-saved" }))); }); + // The editor is a page of its own. The list used to stay rendered below the + // form, which read as the previous page appended to this one. + it("shows only the editor while a Provider is being edited", async () => { + const entry = { + id: "ppio", name: "PPIO", home: "", base_url: "https://api.ppio.com/openai", + anthropic_base_url: "", api_key: "sk-saved", built_in: true, + }; + vi.spyOn(api, "getProvider").mockResolvedValue(entry); + renderPage({ codex: "ppio" }); + expect(screen.getByTestId("provider-ppio")).toBeTruthy(); + + fireEvent.click(screen.getByRole("button", { name: "编辑 PPIO" })); + await waitFor(() => expect(screen.getByLabelText("API Key")).toHaveValue("sk-saved")); + expect(screen.queryByTestId("provider-ppio")).toBeNull(); + + fireEvent.click(screen.getByRole("button", { name: "取消" })); + expect(screen.getByTestId("provider-ppio")).toBeTruthy(); + }); + it("opens the requested Provider editor", async () => { const entry = { id: "ppio", name: "PPIO", home: "https://ppio.com/", base_url: "https://api.ppio.com/openai", diff --git a/frontend/src/pages/ProvidersPage.tsx b/frontend/src/pages/ProvidersPage.tsx index f5c3077e..f22c7cb0 100644 --- a/frontend/src/pages/ProvidersPage.tsx +++ b/frontend/src/pages/ProvidersPage.tsx @@ -72,6 +72,16 @@ export function ProvidersPage({ create = false }: { create?: boolean }) { const returnTo = requestedReturn?.startsWith("/") && !requestedReturn.startsWith("//") ? requestedReturn : "/providers"; const openedProvider = useRef(""); const prefilled = useRef(false); + const editorRef = useRef(null); + const editorOpen = Boolean(editor); + + // The editor takes the page over, so it opens at the top. Clicking 编辑 on a + // card further down the list otherwise left the form above the fold. Keyed on + // the editor opening, not on its contents: the ID field changes with every + // keystroke and must not scroll the page each time. + useEffect(() => { + if (editorOpen) editorRef.current?.scrollIntoView?.({ block: "start" }); + }, [editorOpen]); const nameOf = (agentId: string) => status?.catalog.find((item) => item.id === agentId)?.name || agentId; @@ -273,7 +283,7 @@ export function ProvidersPage({ create = false }: { create?: boolean }) { > {editor ? ( - void save(event)}> + void save(event)}>
{creating ? t("新增模型服务") : t("编辑 {name}", { name: editor.name || editor.id })} +
+ {error ?

{error}

: null} + {detection && !providers.length ? ( +

+ {t("还没有保存 API Key 的模型服务。")} {t("去添加模型服务")} +

+ ) : null} + {providers.map((entry) => { + const groups = groupsByProvider.get(entry.id) ?? []; + return ( +
+
+ {entry.name || entry.id} + {!entry.listed ? {t("无法读取模型列表：{message}", { message: entry.message || entry.errorCode || "" })} : null} +
+ {!groups.length ? ( +

{entry.listed ? t("没有找到多模态模型。") : t("可以稍后重新检测。")}

+ ) : ( +
    + {groups.map((group) => { + const key = probeKey(entry.id, group.id); + const chosen = group.options.find((option) => option.model === selected[key]) ?? group.options[0]; + const install = installLabel(chosen.capability, t); + const state = probes[key]; + const verdict = state?.result && state.result.model === chosen.model ? describeProbe(state.result, t) : null; + const selectID = `multimodal-${entry.id}-${group.id}`; + return ( +
  • + + + {sourceLabel(chosen.capability, t)} + {install.text} + {chosen.capability.probe === "minimal" ? ( + + ) : null} + {verdict ? ( +
    +
    + {verdict.title} + {verdict.reason} + {verdict.detail ? {verdict.detail} : null} +
    +
    + ) : null} + {state?.error ?

    {state.error}

    : null} +
    • + ); + })} +
+ )} +
+ ); + })} +
+ ); +} diff --git a/frontend/src/i18n.tsx b/frontend/src/i18n.tsx index c076cab3..7fc88010 100644 --- a/frontend/src/i18n.tsx +++ b/frontend/src/i18n.tsx @@ -912,6 +912,39 @@ const english = { "粘贴到任意命令行 Agent 对话框执行即可": "paste it into any CLI agent to run", "收藏数": "Stars", "下载量": "Downloads", + // multimodal capabilities + "多模态能力": "Multimodal capabilities", + "用已保存 API Key 的模型服务查找能处理图片、音频和视频的模型。列出模型不收费；点击实测会发出真实请求。": "Finds models that handle images, audio and video on the providers you have saved an API key for. Listing models is free; Probe sends a real request.", + "图片理解": "Image understanding", + "图片生成": "Image generation", + "语音识别": "Speech-to-text", + "语音合成": "Text-to-speech", + "视频生成": "Video generation", + "视频理解": "Video understanding", + "已实测通过": "Probe passed", + "实测未通过": "Probe failed", + "模型服务声明": "Declared by provider", + "内置目录": "Built-in catalog", + "按模型名推断": "Inferred from model name", + "可安装": "Installable", + "不可安装": "Not installable", + "实测通过后可安装": "Installable after a passing probe", + "暂不可安装": "Not installable yet", + "「{model}」支持{capability}": "\"{model}\" supports {capability}", + "模型服务接受了这次实测请求。": "The provider accepted the probe request.", + "无法验证「{model}」的{capability}": "Could not verify {capability} for \"{model}\"", + "「{model}」不支持{capability}": "\"{model}\" does not support {capability}", + "模型服务拒绝了这次实测请求（HTTP {status}）。": "The provider refused the probe request (HTTP {status}).", + "模型服务拒绝了这次实测请求。": "The provider refused the probe request.", + "无法检测多模态能力": "Could not detect multimodal capabilities", + "还没有保存 API Key 的模型服务。": "No provider has a saved API key yet.", + "去添加模型服务": "Add a provider", + "无法读取模型列表：{message}": "Could not read the model list: {message}", + "没有找到多模态模型。": "No multimodal models found.", + "可以稍后重新检测。": "You can detect again later.", + "向模型服务发送一次带小图片的真实请求，确认这个模型能做{capability}。可能产生少量费用。": "Sends one real request with a small image to confirm this model can do {capability}. This may incur a small charge.", + "正在实测": "Probing", + "实测": "Probe", } as const; export type Locale = "zh-CN" | "en"; diff --git a/frontend/src/pages/SkillsPage.test.tsx b/frontend/src/pages/SkillsPage.test.tsx index 7940d5b8..65a82eb0 100644 --- a/frontend/src/pages/SkillsPage.test.tsx +++ b/frontend/src/pages/SkillsPage.test.tsx @@ -8,6 +8,8 @@ vi.mock("../backend/api", () => ({ listSkills: vi.fn(() => Promise.resolve([])), scanSkills: vi.fn(() => Promise.resolve({ skills: [], candidates: [], eligible_agents: [] })), setSkillDraftState: vi.fn(() => Promise.resolve()), + detectMultimodal: vi.fn(() => Promise.resolve({ providers: [] })), + probeMultimodal: vi.fn(), }, })); vi.mock("../state/WizardContext", () => ({ diff --git a/frontend/src/pages/SkillsPage.tsx b/frontend/src/pages/SkillsPage.tsx index 5998ae73..5d97acab 100644 --- a/frontend/src/pages/SkillsPage.tsx +++ b/frontend/src/pages/SkillsPage.tsx @@ -9,6 +9,7 @@ import { AgentTargetGroup } from "../components/AgentTargetGroup"; import { EmptyState } from "../components/EmptyState"; import { ManagementSearch } from "../components/ManagementSearch"; import { ModalDialog } from "../components/ModalDialog"; +import { MultimodalSection } from "../components/MultimodalSection"; import { PageScaffold } from "../components/PageScaffold"; import { useI18n } from "../i18n"; import { useWizard } from "../state/WizardContext"; @@ -150,6 +151,7 @@ export function SkillsPage() { {rows.length ?
: null} {!rows.length ? {t("或者去工具市场发现")}} /> : !visibleRows.length ? :
{visibleRows.map((row) => { const candidate = candidateMap.get(row.id); const hash = candidate?.hash ?? changes[row.id]?.variant_hash ?? row.variant_hashes?.[0] ?? ""; const detail = row.description || t("暂无详情"); const isDeleted = Boolean(pendingDeletes[row.id]); return ; })}
{t("名称")}{t("详情")}{t("状态")}{t("选择目标 Agent")}
{row.name || row.id}{compactDescription(detail)}{row.id}{isDeleted ? {t("待删除")} : row.conflict ? {t("冲突")} : {row.variants} variant} selectTarget(row.id, hash, agent, checked, row.agents ?? [])} />
} + {previewOpen ? setPreviewOpen(false)}>

{t("导入预览")}

{candidates.map((candidate) => { const detail = candidate.description || candidate.hash.slice(0, 12); return
; })}
 : null} ; } diff --git a/frontend/src/styles/app.css b/frontend/src/styles/app.css index 0f327a8a..2f626cf1 100644 --- a/frontend/src/styles/app.css +++ b/frontend/src/styles/app.css @@ -3943,3 +3943,18 @@ @media (min-width: 641px) and (max-width: 899px) { .marketplace-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); grid-auto-rows: 108px; } } + +/* Multimodal capabilities on the Skills page: one block per keyed Provider, + one row per capability. */ +.multimodal-section { margin-top: 28px; } +.multimodal-section .section-heading h2 { display: flex; align-items: center; gap: 6px; } +.multimodal-provider { display: grid; gap: 8px; padding: 12px 14px; border: 1px solid var(--border); border-radius: var(--radius-panel); } +.multimodal-provider > header { display: flex; flex-wrap: wrap; align-items: baseline; gap: 10px; font-size: 13px; } +.multimodal-provider-note { color: var(--text-secondary); font-size: 12px; overflow-wrap: anywhere; } +.multimodal-capabilities { margin: 0; padding: 0; list-style: none; display: grid; gap: 8px; } +.multimodal-capabilities > li { display: flex; flex-wrap: wrap; align-items: center; gap: 8px; font-size: 13px; } +.multimodal-capability-name { min-width: 88px; font-weight: 600; } +.multimodal-capabilities select { min-width: 0; max-width: 320px; min-height: 28px; padding: 0 8px; border: 1px solid var(--border); border-radius: var(--radius-control); background: var(--window-bg); color: var(--text-primary); font: inherit; font-size: 12px; } +.multimodal-verdict { flex-basis: 100%; margin: 0; } +.multimodal-empty { margin: 0; color: var(--text-secondary); font-size: 13px; } +.multimodal-error { margin: 0; color: var(--red); font-size: 13px; } diff --git a/frontend/src/types/api.ts b/frontend/src/types/api.ts index 8fb12f45..1e98ec86 100644 --- a/frontend/src/types/api.ts +++ b/frontend/src/types/api.ts @@ -2,6 +2,7 @@ import type * as AppModels from "../../bindings/github.com/MaimoryLab/BootAgent/ import type * as BindingModels from "../../bindings/github.com/MaimoryLab/BootAgent/internal/binding/models.js"; import type * as CatalogModels from "../../bindings/github.com/MaimoryLab/BootAgent/internal/catalog/models.js"; import type * as MCPModels from "../../bindings/github.com/MaimoryLab/BootAgent/internal/mcp/models.js"; +import type * as ModalityModels from "../../bindings/github.com/MaimoryLab/BootAgent/internal/modality/models.js"; import type * as PlatformModels from "../../bindings/github.com/MaimoryLab/BootAgent/internal/platform/models.js"; import type * as ProviderModels from "../../bindings/github.com/MaimoryLab/BootAgent/internal/provider/models.js"; import type * as TransferModels from "../../bindings/github.com/MaimoryLab/BootAgent/internal/transfer/models.js"; @@ -103,6 +104,25 @@ export type ProfileModelCheck = Omit & { + id: MultimodalCapabilityId; + source: MultimodalSource; + probe: MultimodalProbeKind; +}; +export type MultimodalModel = Omit & { capabilities: MultimodalCapability[] }; +export type MultimodalProvider = Omit & { models: MultimodalModel[] }; +export type MultimodalDetection = { providers: MultimodalProvider[] }; +export type MultimodalProbe = Omit & { + capability: MultimodalCapabilityId; + status: ProfileModelCheckStatus; + reason?: ProfileModelCheckReason; +}; // Mirrors SaveProviderResult: saving a Profile can rewrite the Agents bound to // it, so the outcome per Agent travels with the saved record. export type SaveProfileResult = Omit & { profile: ProfileSummary }; diff --git a/internal/app/multimodal.go b/internal/app/multimodal.go new file mode 100644 index 00000000..bb5e70d6 --- /dev/null +++ b/internal/app/multimodal.go @@ -0,0 +1,186 @@ +package app + +import ( + "context" + "sort" + "strings" + "sync" + + "github.com/MaimoryLab/BootAgent/internal/catalog" + oneerrors "github.com/MaimoryLab/BootAgent/internal/errors" + "github.com/MaimoryLab/BootAgent/internal/modality" + "github.com/MaimoryLab/BootAgent/internal/provider" +) + +// MultimodalDetection lists, for every Provider with a saved key, the models +// that offer a multimodal capability and where each finding came from. +type MultimodalDetection struct { + Providers []MultimodalProvider `json:"providers"` +} + +type MultimodalProvider struct { + ID string `json:"id"` + Name string `json:"name"` + // Listed is false when /models could not be read. Message and ErrorCode + // then say why; catalog generation models are still listed, since they do + // not come from /models. + Listed bool `json:"listed"` + Message string `json:"message,omitempty"` + ErrorCode string `json:"errorCode,omitempty"` + Models []modality.Model `json:"models"` +} + +// MultimodalProbe is the verdict of one capability probe. Status and Reason +// use the Profile model check's vocabulary; see ProfileModelCheck. +type MultimodalProbe struct { + ProviderID string `json:"providerId"` + Model string `json:"model"` + Capability string `json:"capability"` + Status string `json:"status"` + Reason string `json:"reason,omitempty"` + HTTPStatus int `json:"httpStatus,omitempty"` + Detail string `json:"detail,omitempty"` +} + +// DetectMultimodal lists each keyed Provider's models and resolves their +// capabilities. Listing models is an unbilled GET, so opening the Skills page +// may call this; it never sends a probe. +func (u *UseCases) DetectMultimodal(ctx context.Context) (MultimodalDetection, error) { + if u == nil || u.provider == nil { + return MultimodalDetection{}, oneerrors.New(oneerrors.InternalError, "Multimodal detection is not configured", oneerrors.WithStatus(501)) + } + if err := contextError(ctx, "Multimodal detection was cancelled"); err != nil { + return MultimodalDetection{}, err + } + rules, err := modality.Load() + if err != nil { + return MultimodalDetection{}, err + } + public, err := u.providers.Public() + if err != nil { + return MultimodalDetection{}, err + } + var targets []provider.Entry + for id, meta := range public { + if !meta.HasKey { + continue + } + entry, err := u.providers.Get(id) + if err != nil || strings.TrimSpace(entry.APIKey) == "" { + continue + } + targets = append(targets, entry) + } + sort.Slice(targets, func(i, j int) bool { + left, right := catalog.ProviderOrder(targets[i].ID), catalog.ProviderOrder(targets[j].ID) + // Built-in Providers keep the catalog order; custom ones (order 0) + // follow, by ID. + if (left == 0) != (right == 0) { + return left != 0 + } + if left != right { + return left < right + } + return targets[i].ID < targets[j].ID + }) + + results := make([]MultimodalProvider, len(targets)) + var group sync.WaitGroup + for index, target := range targets { + group.Add(1) + go func(index int, target provider.Entry) { + defer group.Done() + results[index] = u.detectProvider(ctx, rules, target) + }(index, target) + } + group.Wait() + if err := contextError(ctx, "Multimodal detection was cancelled"); err != nil { + return MultimodalDetection{}, err + } + return MultimodalDetection{Providers: results}, nil +} + +func (u *UseCases) detectProvider(ctx context.Context, rules modality.Rules, target provider.Entry) MultimodalProvider { + result := MultimodalProvider{ID: target.ID, Name: target.Name, Models: []modality.Model{}} + listing, err := u.provider.ListModels(ctx, "custom", target.APIKey, target.BaseURL) + var entries []provider.ModelEntry + switch { + case err != nil: + result.Message = oneerrors.As(err).Message + result.ErrorCode = oneerrors.As(err).Code + case !listing.OK: + result.Message = listing.Message + if listing.ErrorCode != nil { + result.ErrorCode = *listing.ErrorCode + } + default: + result.Listed = true + entries = listing.Entries + } + models := rules.Resolve(target.ID, entries, u.capabilities.Modalities(target)) + if models != nil { + result.Models = models + } + return result +} + +// ProbeMultimodal sends one live request testing a model's capability. It is a +// billed request, so only a button the user presses may call it; detection, +// saving and installing never do. A conclusive answer -- accepted, or refused +// for this model -- is cached and outranks every other source on the next +// detection. A rejected key or an unreachable endpoint says nothing about the +// model and is not cached. +func (u *UseCases) ProbeMultimodal(ctx context.Context, providerID, model, capability string) (*MultimodalProbe, error) { + if u == nil || u.provider == nil { + return nil, oneerrors.New(oneerrors.InternalError, "Multimodal probing is not configured", oneerrors.WithStatus(501)) + } + if err := contextError(ctx, "Multimodal probe was cancelled"); err != nil { + return nil, err + } + model = strings.TrimSpace(model) + if model == "" { + return nil, oneerrors.New(oneerrors.InvalidRequest, "A model is required") + } + adapter := modality.ChatAdapter(capability) + // Only probes that cost a token or so can be sent from here. A probe that + // produces billed output needs its own confirmation path, and the rest do + // not exist yet. + if modality.ProbeKind(adapter) != modality.ProbeMinimal { + return nil, oneerrors.New(oneerrors.InvalidRequest, "No live check is available for this capability: "+strings.TrimSpace(capability)) + } + check := &MultimodalProbe{ProviderID: providerID, Model: model, Capability: capability} + target, err := u.providers.Get(providerID) + if err != nil { + return nil, err + } + if strings.TrimSpace(target.APIKey) == "" { + check.Status, check.Reason = ModelCheckUnverified, "no_key" + return check, nil + } + result, err := u.provider.ProbeVision(ctx, target.APIKey, model, target.BaseURL) + if ctx.Err() != nil { + return nil, contextError(ctx, "Multimodal probe was cancelled") + } + if err != nil { + check.Status, check.Reason, check.Detail = ModelCheckUnverified, "error", oneerrors.As(err).Message + return check, nil + } + if result.OK { + check.Status = ModelCheckAvailable + } else { + check.Status, check.Reason, check.HTTPStatus, check.Detail = probeVerdict(result) + } + if check.Status == ModelCheckAvailable || check.Reason == "refused" { + u.capabilityMu.Lock() + // Derived data: a failed cache write must not turn a verdict the user + // paid for into an error. + _ = u.capabilities.SaveModality(ctx, target, model, capability, provider.ModalityVerdict{ + Supported: check.Status == ModelCheckAvailable, + Source: modality.SourceProbe, + Adapter: adapter, + Status: result.Status, + }) + u.capabilityMu.Unlock() + } + return check, nil +} diff --git a/internal/app/multimodal_test.go b/internal/app/multimodal_test.go new file mode 100644 index 00000000..1aa4977c --- /dev/null +++ b/internal/app/multimodal_test.go @@ -0,0 +1,236 @@ +package app + +import ( + "context" + "encoding/json" + "io" + "net/http" + "os" + "strings" + "sync" + "testing" + + "github.com/MaimoryLab/BootAgent/internal/modality" + "github.com/MaimoryLab/BootAgent/internal/provider" +) + +const multimodalKey = "sk-multimodal-secret" + +// multimodalGateway serves /models from a fixed listing and answers chat +// requests by model, recording every request so a test can prove detection +// sends nothing billed. +type multimodalGateway struct { + mu sync.Mutex + listing string + listCode int + chat map[string]int + requests []string +} + +func (g *multimodalGateway) Do(request *http.Request) (*http.Response, error) { + var body struct { + Model string `json:"model"` + } + if request.Body != nil { + data, _ := io.ReadAll(request.Body) + _ = json.Unmarshal(data, &body) + } + g.mu.Lock() + defer g.mu.Unlock() + g.requests = append(g.requests, request.Method+" "+request.URL.Path+" "+body.Model) + if request.Method == http.MethodGet && strings.HasSuffix(request.URL.Path, "/models") { + code := g.listCode + if code == 0 { + code = http.StatusOK + } + return appProviderResponse(code, g.listing), nil + } + status, ok := g.chat[body.Model] + if !ok { + status = http.StatusOK + } + if status == http.StatusOK { + return appProviderResponse(status, `{}`), nil + } + return appProviderResponse(status, `{"error":{"message":"model `+body.Model+` does not support image input, key `+multimodalKey+`"}}`), nil +} + +func (g *multimodalGateway) billed() []string { + g.mu.Lock() + defer g.mu.Unlock() + var posts []string + for _, request := range g.requests { + if strings.HasPrefix(request, http.MethodPost) { + posts = append(posts, request) + } + } + return posts +} + +func multimodalCore(t *testing.T, gateway *multimodalGateway) *UseCases { + t.Helper() + core := activationCore(t, t.TempDir(), provider.NewClient(gateway), "linux") + if _, err := core.SaveProvider(context.Background(), provider.Entry{ + ID: "gw", Name: "Gateway", BaseURL: "https://gw.example/v1", APIKey: multimodalKey, + }, true, false); err != nil { + t.Fatal(err) + } + return core +} + +const gatewayListing = `{"data":[ + {"id":"qwen/qwen3-vl-8b","input_modalities":["text","image"],"output_modalities":["text"]}, + {"id":"moonshotai/kimi-k3"}, + {"id":"deepseek/deepseek-v4-pro","input_modalities":["text"],"output_modalities":["text"]} +]}` + +func findModel(t *testing.T, detection MultimodalDetection, providerID, model string) modality.Model { + t.Helper() + for _, entry := range detection.Providers { + if entry.ID != providerID { + continue + } + for _, candidate := range entry.Models { + if candidate.ID == model { + return candidate + } + } + } + t.Fatalf("%s/%s not detected in %#v", providerID, model, detection) + return modality.Model{} +} + +// Opening the Skills page lists models, which is free. It must never probe. +func TestDetectMultimodalListsWithoutBilledRequests(t *testing.T) { + gateway := &multimodalGateway{listing: gatewayListing} + core := multimodalCore(t, gateway) + detection, err := core.DetectMultimodal(context.Background()) + if err != nil { + t.Fatal(err) + } + if posts := gateway.billed(); len(posts) != 0 { + t.Fatalf("detection sent billed requests: %v", posts) + } + // Only the Provider with a key is listed; built-ins without one are not. + if len(detection.Providers) != 1 || detection.Providers[0].ID != "gw" || !detection.Providers[0].Listed { + t.Fatalf("providers = %#v", detection.Providers) + } + declared := findModel(t, detection, "gw", "qwen/qwen3-vl-8b").Capabilities[0] + if declared.Source != modality.SourceDeclared || !declared.Installable || declared.Probe != modality.ProbeMinimal { + t.Fatalf("declared vision = %#v", declared) + } + guessed := findModel(t, detection, "gw", "moonshotai/kimi-k3").Capabilities[0] + if guessed.Source != modality.SourceIDRule || guessed.Installable { + t.Fatalf("ID-rule vision = %#v", guessed) + } + data, _ := json.Marshal(detection) + if strings.Contains(string(data), multimodalKey) { + t.Fatal("detection result carries the API key") + } +} + +func TestDetectMultimodalReportsAFailedListing(t *testing.T) { + gateway := &multimodalGateway{listCode: http.StatusUnauthorized, listing: `{}`} + core := multimodalCore(t, gateway) + detection, err := core.DetectMultimodal(context.Background()) + if err != nil { + t.Fatal(err) + } + entry := detection.Providers[0] + if entry.Listed || entry.ErrorCode == "" || entry.Message == "" || entry.Models == nil { + t.Fatalf("failed listing = %#v", entry) + } +} + +func TestProbeMultimodalCachesAPassAndOutranksTheIDRule(t *testing.T) { + gateway := &multimodalGateway{listing: gatewayListing} + core := multimodalCore(t, gateway) + check, err := core.ProbeMultimodal(context.Background(), "gw", "moonshotai/kimi-k3", modality.Vision) + if err != nil { + t.Fatal(err) + } + if check.Status != ModelCheckAvailable || len(gateway.billed()) != 1 { + t.Fatalf("check = %#v, requests = %v", check, gateway.billed()) + } + detection, err := core.DetectMultimodal(context.Background()) + if err != nil { + t.Fatal(err) + } + vision := findModel(t, detection, "gw", "moonshotai/kimi-k3").Capabilities[0] + if vision.Source != modality.SourceProbe || !vision.Installable || !vision.Supported { + t.Fatalf("probed vision = %#v", vision) + } +} + +func TestProbeMultimodalCachesARefusalButNotAKeyProblem(t *testing.T) { + gateway := &multimodalGateway{listing: gatewayListing, chat: map[string]int{ + "qwen/qwen3-vl-8b": http.StatusBadRequest, + "moonshotai/kimi-k3": http.StatusUnauthorized, + }} + core := multimodalCore(t, gateway) + refused, err := core.ProbeMultimodal(context.Background(), "gw", "qwen/qwen3-vl-8b", modality.Vision) + if err != nil { + t.Fatal(err) + } + if refused.Status != ModelCheckUnavailable || refused.Reason != "refused" || refused.HTTPStatus != http.StatusBadRequest { + t.Fatalf("refused = %#v", refused) + } + if strings.Contains(refused.Detail, multimodalKey) || !strings.Contains(refused.Detail, "does not support image input") { + t.Fatalf("detail = %q", refused.Detail) + } + key, err := core.ProbeMultimodal(context.Background(), "gw", "moonshotai/kimi-k3", modality.Vision) + if err != nil { + t.Fatal(err) + } + if key.Status != ModelCheckUnavailable || key.Reason != "key" { + t.Fatalf("key = %#v", key) + } + detection, err := core.DetectMultimodal(context.Background()) + if err != nil { + t.Fatal(err) + } + qwen := findModel(t, detection, "gw", "qwen/qwen3-vl-8b").Capabilities[0] + if qwen.Source != modality.SourceProbe || qwen.Supported || qwen.Installable { + t.Fatalf("refusal not cached: %#v", qwen) + } + // A rejected key says nothing about the model: the ID rule still answers. + kimi := findModel(t, detection, "gw", "moonshotai/kimi-k3").Capabilities[0] + if kimi.Source != modality.SourceIDRule { + t.Fatalf("key failure was cached: %#v", kimi) + } + data, err := os.ReadFile(core.capabilities.Path()) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(data), multimodalKey) { + t.Fatal("capability cache carries the API key") + } +} + +func TestProbeMultimodalRefusesWhatItCannotCheck(t *testing.T) { + gateway := &multimodalGateway{listing: gatewayListing} + core := multimodalCore(t, gateway) + for _, capability := range []string{modality.VideoGeneration, modality.ImageGeneration, modality.TextToSpeech, "smell"} { + if _, err := core.ProbeMultimodal(context.Background(), "gw", "m", capability); err == nil { + t.Errorf("%s probe accepted", capability) + } + } + if _, err := core.ProbeMultimodal(context.Background(), "gw", " ", modality.Vision); err == nil { + t.Error("probe without a model accepted") + } + if posts := gateway.billed(); len(posts) != 0 { + t.Fatalf("refused probes still sent requests: %v", posts) + } +} + +func TestProbeMultimodalWithoutAKeySendsNothing(t *testing.T) { + gateway := &multimodalGateway{listing: gatewayListing} + core := activationCore(t, t.TempDir(), provider.NewClient(gateway), "linux") + check, err := core.ProbeMultimodal(context.Background(), "novita", "qwen/qwen3-vl-8b", modality.Vision) + if err != nil { + t.Fatal(err) + } + if check.Status != ModelCheckUnverified || check.Reason != "no_key" || len(gateway.billed()) != 0 { + t.Fatalf("check = %#v, requests = %v", check, gateway.billed()) + } +} diff --git a/internal/app/profile_model_check.go b/internal/app/profile_model_check.go index a9b9fe39..1720a644 100644 --- a/internal/app/profile_model_check.go +++ b/internal/app/profile_model_check.go @@ -213,25 +213,33 @@ func (u *UseCases) checkProfileModel(ctx context.Context, profile profileStore.P } sort.Strings(check.Agents) check.Protocol = worstProtocol - check.HTTPStatus = worst.Status - check.Detail = worst.Detail + check.Status, check.Reason, check.HTTPStatus, check.Detail = probeVerdict(*worst) + return check +} + +// probeVerdict classifies a failed probe into the status and reason the UI +// words its remedy by. Every billed check BootAgent sends -- the Profile model +// check and the multimodal capability probes -- goes through it, so a refusal, +// a rejected key and a busy endpoint read the same everywhere. +func probeVerdict(result provider.ProbeResult) (status, reason string, httpStatus int, detail string) { + httpStatus, detail = result.Status, result.Detail switch { - case !worst.Reachable && worst.Status == 0 && worst.ErrorCode == nil && worst.Message == "": - check.Status, check.Reason = ModelCheckUnverified, "error" - case !worst.Reachable: - check.Status, check.Reason = ModelCheckUnverified, "network" - check.HTTPStatus = 0 - case worst.ErrorCode != nil && *worst.ErrorCode == oneerrors.APIKeyRejected: - check.Status, check.Reason = ModelCheckUnavailable, "key" - case transientProbe(*worst): - check.Status, check.Reason = ModelCheckUnverified, "transient" + case !result.Reachable && result.Status == 0 && result.ErrorCode == nil && result.Message == "": + status, reason = ModelCheckUnverified, "error" + case !result.Reachable: + status, reason = ModelCheckUnverified, "network" + httpStatus = 0 + case result.ErrorCode != nil && *result.ErrorCode == oneerrors.APIKeyRejected: + status, reason = ModelCheckUnavailable, "key" + case transientProbe(result): + status, reason = ModelCheckUnverified, "transient" default: - check.Status, check.Reason = ModelCheckUnavailable, "refused" + status, reason = ModelCheckUnavailable, "refused" } - if check.Detail == "" && worst.Message != "" && check.Reason != "refused" { - check.Detail = worst.Message + if detail == "" && result.Message != "" && reason != "refused" { + detail = result.Message } - return check + return status, reason, httpStatus, detail } // modelCheckRank orders failures by how much the user can do about them: a diff --git a/internal/app/status.go b/internal/app/status.go index 0af5f269..24ef0e34 100644 --- a/internal/app/status.go +++ b/internal/app/status.go @@ -85,6 +85,9 @@ type UseCases struct { conversion *convertproxy.Server conversionMu sync.Mutex capabilities provider.CapabilityStore + // capabilityMu serializes capability-cache writes from multimodal probes, + // which run outside writeMu so a slow probe never blocks an install. + capabilityMu sync.Mutex } // DraftState combines the independent MCP and Skills drafts for the native diff --git a/internal/binding/multimodal.go b/internal/binding/multimodal.go new file mode 100644 index 00000000..cd87ee0e --- /dev/null +++ b/internal/binding/multimodal.go @@ -0,0 +1,44 @@ +package binding + +import ( + "context" + + "github.com/MaimoryLab/BootAgent/internal/app" +) + +// MultimodalService exposes multimodal capability detection. Detect is free -- +// it lists models -- so pages may call it on open. Probe is a billed request +// and may only be called from a button the user presses. +type MultimodalService struct { + core *app.UseCases +} + +func NewMultimodalService(core *app.UseCases) *MultimodalService { + return &MultimodalService{core: core} +} + +type MultimodalProbeRequest struct { + Provider string `json:"provider"` + Model string `json:"model"` + Capability string `json:"capability"` +} + +func (s *MultimodalService) Detect(ctx context.Context) (app.MultimodalDetection, error) { + if err := contextError(ctx); err != nil { + return app.MultimodalDetection{}, err + } + if s == nil || s.core == nil { + return app.MultimodalDetection{}, notReady("Multimodal detection is not configured") + } + return s.core.DetectMultimodal(ctx) +} + +func (s *MultimodalService) Probe(ctx context.Context, request MultimodalProbeRequest) (*app.MultimodalProbe, error) { + if err := contextError(ctx); err != nil { + return nil, err + } + if s == nil || s.core == nil { + return nil, notReady("Multimodal probing is not configured") + } + return s.core.ProbeMultimodal(ctx, request.Provider, request.Model, request.Capability) +} diff --git a/internal/binding/services.go b/internal/binding/services.go index 567fe1ce..ecb3b0fc 100644 --- a/internal/binding/services.go +++ b/internal/binding/services.go @@ -34,6 +34,7 @@ type Services struct { Conversion *ConversionService Marketplace *MarketplaceService Task *TaskService + Multimodal *MultimodalService } type ServicesOptions struct { @@ -61,6 +62,7 @@ func NewServicesWithOptions(core *app.UseCases, opener BrowserOpener, options Se Conversion: NewConversionService(core), Marketplace: NewMarketplaceService(core, opener), Task: &TaskService{core: core}, + Multimodal: NewMultimodalService(core), } } diff --git a/internal/binding/services_test.go b/internal/binding/services_test.go index 416906df..327aa5ca 100644 --- a/internal/binding/services_test.go +++ b/internal/binding/services_test.go @@ -59,6 +59,7 @@ func TestServiceMethodAllowlist(t *testing.T) { {&ConversionService{}, []string{"Get", "RegenerateKey", "Save"}}, {&MarketplaceService{}, []string{"Catalog", "ClearRecommendationHistory", "DeleteRecommendationHistory", "DiscoverSources", "FetchMCPServerDetail", "FetchMCPServerReadme", "FetchMCPServersDirectoryDetail", "FetchMCPServersDirectoryReadme", "FetchShowcase", "FetchSkillDetail", "FetchSkillFile", "ListRecommendationHistory", "OpenExternal", "Recommend", "RecommendationAgents", "SaveRecommendationHistory"}}, {&UpdateService{}, []string{"Check", "DownloadAndInstall", "Restart", "Version"}}, + {&MultimodalService{}, []string{"Detect", "Probe"}}, } for _, test := range tests { typeOf := reflect.TypeOf(test.service) diff --git a/internal/catalog/modalities.go b/internal/catalog/modalities.go new file mode 100644 index 00000000..e31c1023 --- /dev/null +++ b/internal/catalog/modalities.go @@ -0,0 +1,138 @@ +package catalog + +import ( + "encoding/json" + "fmt" + "regexp" + "strings" + "sync" + + bootagent "github.com/MaimoryLab/BootAgent" + oneerrors "github.com/MaimoryLab/BootAgent/internal/errors" +) + +const ModalitySchemaVersion = 1 + +var ( + modalityOnce sync.Once + modalityManifest ModalityManifest + modalityErr error +) + +// ModalityManifest is modalities.lock.json: ID rules that map a model ID to +// the capabilities it most likely has, plus per Provider the native route and +// the generation models /models does not list. The capability and adapter +// vocabulary is internal/modality's, which validates it on load; this parser +// checks structure only. +type ModalityManifest struct { + SchemaVersion int `json:"schema_version"` + IDRules []ModalityIDRule `json:"id_rules"` + Providers map[string]ModalityProvider `json:"providers"` +} + +// ModalityIDRule matches the lower-cased model ID with any vendor/ prefix +// removed. Exclude, when set, vetoes a match. +type ModalityIDRule struct { + Pattern string `json:"pattern"` + Exclude string `json:"exclude,omitempty"` + Capabilities []string `json:"capabilities"` +} + +type ModalityProvider struct { + // NativeBaseURL is the host root native async and audio routes hang off. It + // is listed separately because the catalog's base_url is the OpenAI route + // (https://api.novita.ai/openai), which is not where those routes live. + NativeBaseURL string `json:"native_base_url"` + Generation []ModalityGenerationModel `json:"generation"` +} + +// ModalityGenerationModel is a model only the catalog knows about: the async +// generation and audio models are not in any /models listing. +type ModalityGenerationModel struct { + ID string `json:"id"` + Capabilities []string `json:"capabilities"` + Adapter string `json:"adapter"` +} + +func LoadEmbeddedModalities() (ModalityManifest, error) { + modalityOnce.Do(func() { + data, err := bootagent.EmbeddedModalityLock() + if err != nil { + modalityErr = oneerrors.New(oneerrors.InvalidRequest, "Cannot load embedded modality lock manifest", oneerrors.WithCause(err)) + return + } + modalityManifest, modalityErr = ParseModalities(data) + }) + if modalityErr != nil { + return ModalityManifest{}, modalityErr + } + return cloneModalityManifest(modalityManifest), nil +} + +func ParseModalities(data []byte) (ModalityManifest, error) { + var manifest ModalityManifest + if err := json.Unmarshal(data, &manifest); err != nil { + return ModalityManifest{}, invalidModalities(fmt.Sprintf("Cannot load modality lock manifest: %v", err)) + } + if manifest.SchemaVersion != ModalitySchemaVersion { + return ModalityManifest{}, invalidModalities("Unsupported modality lock manifest schema") + } + if len(manifest.IDRules) == 0 { + return ModalityManifest{}, invalidModalities("Modality lock manifest has no ID rules") + } + for index, rule := range manifest.IDRules { + if _, err := regexp.Compile(rule.Pattern); err != nil || strings.TrimSpace(rule.Pattern) == "" { + return ModalityManifest{}, invalidModalities(fmt.Sprintf("Modality ID rule %d has an invalid pattern", index)) + } + if rule.Exclude != "" { + if _, err := regexp.Compile(rule.Exclude); err != nil { + return ModalityManifest{}, invalidModalities(fmt.Sprintf("Modality ID rule %d has an invalid exclude pattern", index)) + } + } + if len(rule.Capabilities) == 0 { + return ModalityManifest{}, invalidModalities(fmt.Sprintf("Modality ID rule %d names no capability", index)) + } + } + for providerID, entry := range manifest.Providers { + if _, ok := ProviderByID(providerID); !ok { + return ModalityManifest{}, invalidModalities(fmt.Sprintf("Modality provider %q is not a built-in Provider", providerID)) + } + if !httpsURL(entry.NativeBaseURL) || strings.HasSuffix(entry.NativeBaseURL, "/") { + return ModalityManifest{}, invalidModalities(fmt.Sprintf("Modality provider %q has an invalid native_base_url", providerID)) + } + seen := map[string]bool{} + for _, model := range entry.Generation { + if strings.TrimSpace(model.ID) == "" || len(model.Capabilities) == 0 || strings.TrimSpace(model.Adapter) == "" { + return ModalityManifest{}, invalidModalities(fmt.Sprintf("Modality provider %q has an incomplete generation model", providerID)) + } + if seen[model.ID] { + return ModalityManifest{}, invalidModalities(fmt.Sprintf("Modality provider %q lists %q twice", providerID, model.ID)) + } + seen[model.ID] = true + } + } + return cloneModalityManifest(manifest), nil +} + +func invalidModalities(message string) error { + return oneerrors.New(oneerrors.InvalidRequest, message) +} + +func cloneModalityManifest(source ModalityManifest) ModalityManifest { + result := ModalityManifest{SchemaVersion: source.SchemaVersion} + result.IDRules = make([]ModalityIDRule, len(source.IDRules)) + for index, rule := range source.IDRules { + rule.Capabilities = append([]string(nil), rule.Capabilities...) + result.IDRules[index] = rule + } + result.Providers = make(map[string]ModalityProvider, len(source.Providers)) + for id, entry := range source.Providers { + models := make([]ModalityGenerationModel, len(entry.Generation)) + for index, model := range entry.Generation { + model.Capabilities = append([]string(nil), model.Capabilities...) + models[index] = model + } + result.Providers[id] = ModalityProvider{NativeBaseURL: entry.NativeBaseURL, Generation: models} + } + return result +} diff --git a/internal/catalog/modalities_test.go b/internal/catalog/modalities_test.go new file mode 100644 index 00000000..48a94fde --- /dev/null +++ b/internal/catalog/modalities_test.go @@ -0,0 +1,47 @@ +package catalog + +import ( + "strings" + "testing" +) + +func TestEmbeddedModalitiesParse(t *testing.T) { + manifest, err := LoadEmbeddedModalities() + if err != nil { + t.Fatal(err) + } + if len(manifest.IDRules) == 0 || manifest.Providers["novita"].NativeBaseURL != "https://api.novita.ai" { + t.Fatalf("embedded modalities = %#v", manifest) + } + // The returned value is a copy; mutating it must not leak into the next load. + manifest.IDRules[0].Capabilities[0] = "mutated" + again, _ := LoadEmbeddedModalities() + if again.IDRules[0].Capabilities[0] == "mutated" { + t.Fatal("LoadEmbeddedModalities returned shared state") + } +} + +func TestParseModalitiesRejectsBrokenEntries(t *testing.T) { + valid := `{"schema_version":1,"id_rules":[{"pattern":"^x","capabilities":["vision"]}],"providers":{"novita":{"native_base_url":"https://api.novita.ai","generation":[]}}}` + if _, err := ParseModalities([]byte(valid)); err != nil { + t.Fatal(err) + } + cases := map[string]string{ + "schema": strings.Replace(valid, `"schema_version":1`, `"schema_version":2`, 1), + "no rules": `{"schema_version":1,"id_rules":[],"providers":{}}`, + "bad pattern": strings.Replace(valid, `"^x"`, `"(unclosed"`, 1), + "bad exclude": strings.Replace(valid, `"pattern":"^x"`, `"pattern":"^x","exclude":"[z"`, 1), + "no capability": strings.Replace(valid, `["vision"]`, `[]`, 1), + "unknown vendor": strings.Replace(valid, `"novita"`, `"nobody"`, 1), + "http base": strings.Replace(valid, `https://api.novita.ai`, `http://api.novita.ai`, 1), + "trailing slash": strings.Replace(valid, `https://api.novita.ai`, `https://api.novita.ai/`, 1), + "incomplete gen": strings.Replace(valid, `"generation":[]`, `"generation":[{"id":"m","capabilities":["video-generation"]}]`, 1), + "duplicate gen": strings.Replace(valid, `"generation":[]`, `"generation":[{"id":"m","capabilities":["video-generation"],"adapter":"novita-async"},{"id":"m","capabilities":["video-generation"],"adapter":"novita-async"}]`, 1), + "not json": `{`, + } + for name, data := range cases { + if _, err := ParseModalities([]byte(data)); err == nil { + t.Errorf("%s accepted", name) + } + } +} diff --git a/internal/modality/capability.go b/internal/modality/capability.go new file mode 100644 index 00000000..33d5bcbe --- /dev/null +++ b/internal/modality/capability.go @@ -0,0 +1,104 @@ +// Package modality decides which multimodal capabilities a Provider's models +// offer, and names the adapter that reaches each one. It sends no request +// itself: listing models and probing are the caller's, and a probe is billed, +// so only a user's click may send one. +package modality + +// Capability IDs. They double as the suffix of the generated Skill ID +// (bootagent-) and as keys in the capability cache, so they must +// never be renamed. +const ( + Vision = "vision" + ImageGeneration = "image-generation" + SpeechToText = "speech-to-text" + TextToSpeech = "text-to-speech" + VideoGeneration = "video-generation" + VideoUnderstanding = "video-understanding" +) + +// Capabilities lists every capability in the order the UI shows them. +var Capabilities = []string{Vision, ImageGeneration, SpeechToText, TextToSpeech, VideoGeneration, VideoUnderstanding} + +// Adapter IDs: the calling convention the generated script uses, shared with +// the probes so a probe tests exactly what the Skill will send. +const ( + // AdapterChatVision sends images as Chat Completions image_url parts. + AdapterChatVision = "openai-chat-vision" + // AdapterChatVideo sends video as Chat Completions content parts, for models + // that take video natively. + AdapterChatVideo = "openai-chat-video" + // AdapterChatImageOutput asks a chat model for an image in its reply + // (gemini-*-image, gpt-image-* on JieKou.AI). How gateways return the image + // is not yet verified, so models on this route install only after a probe. + AdapterChatImageOutput = "openai-chat-image-output" + // AdapterChatAudioInput sends audio as Chat Completions input_audio parts + // (*-omni-*, gemini-*, mimo-v2.6-*). Unverified across gateways, like the + // image-output route, so it installs only after a probe. + AdapterChatAudioInput = "openai-chat-audio-input" + // The OpenAI-compatible generation and audio routes. + AdapterImages = "openai-images" + AdapterAudioTranscriptions = "openai-audio-transcriptions" + AdapterAudioSpeech = "openai-audio-speech" + // Native routes at the Provider's host root, named by the catalog. + AdapterNovitaAsync = "novita-async" + AdapterNovitaAudio = "novita-audio" + AdapterPPIOVideo = "ppio-video" +) + +// Adapters is the closed vocabulary the catalog may name. +var Adapters = []string{ + AdapterChatVision, AdapterChatVideo, AdapterChatImageOutput, AdapterChatAudioInput, + AdapterImages, AdapterAudioTranscriptions, AdapterAudioSpeech, + AdapterNovitaAsync, AdapterNovitaAudio, AdapterPPIOVideo, +} + +// Sources say how a capability was established, in resolution order. +const ( + SourceProbe = "probe" + SourceDeclared = "declared" + SourceCatalog = "catalog" + SourceIDRule = "id-rule" +) + +// chatAdapter is the adapter a capability found on a chat model uses. Text to +// speech and video generation have no chat route: no rule claims them, and a +// chat model that declares audio or video output is not offered for either, +// because how a gateway returns that output is unknown. +var chatAdapter = map[string]string{ + Vision: AdapterChatVision, + VideoUnderstanding: AdapterChatVideo, + SpeechToText: AdapterChatAudioInput, + ImageGeneration: AdapterChatImageOutput, +} + +// probeOnly are the adapters whose request and response format across gateways +// is not verified: neither a declaration nor a rule is enough to install them, +// only a probe that passed. +var probeOnly = map[string]bool{ + AdapterChatImageOutput: true, + AdapterChatAudioInput: true, +} + +// Probe kinds tell the UI whether a capability can be checked live and whether +// the check needs a confirmation first. +const ( + // ProbeNone: no live check exists for this adapter yet, or ever (video + // generation is a full billed generation that cannot be cancelled). + ProbeNone = "" + // ProbeMinimal: a request costing a token or so; the button warns of cost. + ProbeMinimal = "minimal" + // ProbeConfirm: a request that produces billed output, such as one image; + // the UI asks before sending. + ProbeConfirm = "confirm" +) + +// probeKind lists the adapters that can be probed today. Others are added as +// their probes are written; until then the UI offers no button for them. +var probeKind = map[string]string{ + AdapterChatVision: ProbeMinimal, +} + +// ProbeKind reports how an adapter can be probed. +func ProbeKind(adapter string) string { + return probeKind[adapter] +} diff --git a/internal/modality/resolve.go b/internal/modality/resolve.go new file mode 100644 index 00000000..23b4e0f0 --- /dev/null +++ b/internal/modality/resolve.go @@ -0,0 +1,265 @@ +package modality + +import ( + "fmt" + "regexp" + "slices" + "sort" + "strings" + + "github.com/MaimoryLab/BootAgent/internal/catalog" + oneerrors "github.com/MaimoryLab/BootAgent/internal/errors" + "github.com/MaimoryLab/BootAgent/internal/provider" +) + +// Capability is one capability found on one model. +type Capability struct { + ID string `json:"id"` + Source string `json:"source"` + Adapter string `json:"adapter"` + // Supported is false only for a probe that the Provider refused. It is kept + // in the result so the UI can say the check failed, rather than silently + // dropping a model the user just checked. + Supported bool `json:"supported"` + // Installable says the evidence is strong enough to write a Skill for it. + // An ID rule alone never is, and neither is a declaration for an adapter + // whose wire format across gateways is unverified. + Installable bool `json:"installable"` + // Probe is the live check available for this capability; see ProbeKind. + Probe string `json:"probe"` +} + +// Model is one model with every capability found for it, in Capabilities order. +type Model struct { + ID string `json:"id"` + Capabilities []Capability `json:"capabilities"` +} + +// Rules is the loaded catalog, compiled and checked against the vocabulary. +type Rules struct { + rules []compiledRule + providers map[string]catalog.ModalityProvider +} + +type compiledRule struct { + pattern *regexp.Regexp + exclude *regexp.Regexp + capabilities []string +} + +// Load reads the embedded modalities.lock.json. +func Load() (Rules, error) { + manifest, err := catalog.LoadEmbeddedModalities() + if err != nil { + return Rules{}, err + } + return NewRules(manifest) +} + +// NewRules compiles a parsed catalog. It rejects a capability or adapter +// outside the vocabulary, so a typo in the lock file fails tests instead of +// silently matching nothing. +func NewRules(manifest catalog.ModalityManifest) (Rules, error) { + result := Rules{providers: manifest.Providers} + for index, rule := range manifest.IDRules { + for _, capability := range rule.Capabilities { + if !slices.Contains(Capabilities, capability) { + return Rules{}, invalid(fmt.Sprintf("Modality ID rule %d names unknown capability %q", index, capability)) + } + if _, ok := chatAdapter[capability]; !ok { + return Rules{}, invalid(fmt.Sprintf("Modality ID rule %d claims %q, which no chat model route serves", index, capability)) + } + } + pattern, err := regexp.Compile(rule.Pattern) + if err != nil { + return Rules{}, invalid(fmt.Sprintf("Modality ID rule %d has an invalid pattern", index)) + } + compiled := compiledRule{pattern: pattern, capabilities: rule.Capabilities} + if rule.Exclude != "" { + if compiled.exclude, err = regexp.Compile(rule.Exclude); err != nil { + return Rules{}, invalid(fmt.Sprintf("Modality ID rule %d has an invalid exclude pattern", index)) + } + } + result.rules = append(result.rules, compiled) + } + for providerID, entry := range manifest.Providers { + for _, model := range entry.Generation { + if !slices.Contains(Adapters, model.Adapter) { + return Rules{}, invalid(fmt.Sprintf("Modality provider %q model %q names unknown adapter %q", providerID, model.ID, model.Adapter)) + } + for _, capability := range model.Capabilities { + if !slices.Contains(Capabilities, capability) { + return Rules{}, invalid(fmt.Sprintf("Modality provider %q model %q names unknown capability %q", providerID, model.ID, capability)) + } + } + } + } + return result, nil +} + +// NativeBaseURL is the host root for a Provider's native routes, empty when +// the catalog names none. +func (r Rules) NativeBaseURL(providerID string) string { + return r.providers[providerID].NativeBaseURL +} + +// RuleKey is the form ID rules match against: lower-cased, with any vendor/ +// prefix removed. Gateway suffixes are left alone because rules are anchored +// at the start. +func RuleKey(modelID string) string { + key := strings.ToLower(strings.TrimSpace(modelID)) + if index := strings.LastIndex(key, "/"); index >= 0 { + key = key[index+1:] + } + return key +} + +// MatchIDRules returns the capabilities the ID rules claim for a model, in +// Capabilities order. +func (r Rules) MatchIDRules(modelID string) []string { + key := RuleKey(modelID) + claimed := map[string]bool{} + for _, rule := range r.rules { + if rule.pattern.MatchString(key) && (rule.exclude == nil || !rule.exclude.MatchString(key)) { + for _, capability := range rule.capabilities { + claimed[capability] = true + } + } + } + return ordered(claimed) +} + +// DeclaredCapabilities maps what a /models entry declares onto capabilities. +// Audio and video output on a chat model map to nothing: there is no chat +// route for either, and how a gateway would return them is unknown. +func DeclaredCapabilities(entry provider.ModelEntry) []string { + claimed := map[string]bool{} + for _, value := range entry.InputModalities { + switch value { + case "image": + claimed[Vision] = true + case "video": + claimed[VideoUnderstanding] = true + case "audio": + claimed[SpeechToText] = true + } + } + if slices.Contains(entry.OutputModalities, "image") { + claimed[ImageGeneration] = true + } + return ordered(claimed) +} + +// Resolve decides each model's capabilities for one Provider. For every model +// and capability the first source that answers wins: +// +// 1. a probe verdict in the capability cache (verdicts, keyed by model ID); +// 2. the model's declared modalities, when the Provider sent any; +// 3. an ID rule; +// 4. otherwise the capability is not offered. +// +// The declared metadata answers for the whole model: a model that declares +// modalities is not given more by an ID rule, because the Provider's own word +// about a model it serves outranks a guess from its name. Generation models +// the catalog lists for the Provider are appended with source catalog. +func (r Rules) Resolve(providerID string, entries []provider.ModelEntry, verdicts map[string]map[string]provider.ModalityVerdict) []Model { + models := make([]Model, 0, len(entries)) + seen := map[string]bool{} + for _, entry := range entries { + id := strings.TrimSpace(entry.ID) + if id == "" || seen[id] { + continue + } + seen[id] = true + source, claimed := SourceIDRule, r.MatchIDRules(id) + if entry.Declared() { + source, claimed = SourceDeclared, DeclaredCapabilities(entry) + } + probed := verdicts[id] + var capabilities []Capability + for _, capabilityID := range Capabilities { + if verdict, ok := probed[capabilityID]; ok { + capabilities = append(capabilities, probedCapability(capabilityID, verdict)) + continue + } + if !slices.Contains(claimed, capabilityID) { + continue + } + adapter := chatAdapter[capabilityID] + capabilities = append(capabilities, Capability{ + ID: capabilityID, + Source: source, + Adapter: adapter, + Supported: true, + Installable: source == SourceDeclared && !probeOnly[adapter], + Probe: ProbeKind(adapter), + }) + } + if len(capabilities) > 0 { + models = append(models, Model{ID: id, Capabilities: capabilities}) + } + } + for _, generation := range r.providers[providerID].Generation { + if seen[generation.ID] { + continue + } + seen[generation.ID] = true + probed := verdicts[generation.ID] + var capabilities []Capability + for _, capabilityID := range Capabilities { + if !slices.Contains(generation.Capabilities, capabilityID) { + continue + } + if verdict, ok := probed[capabilityID]; ok { + capabilities = append(capabilities, probedCapability(capabilityID, verdict)) + continue + } + capabilities = append(capabilities, Capability{ + ID: capabilityID, + Source: SourceCatalog, + Adapter: generation.Adapter, + Supported: true, + Installable: true, + Probe: ProbeKind(generation.Adapter), + }) + } + models = append(models, Model{ID: generation.ID, Capabilities: capabilities}) + } + sort.SliceStable(models, func(i, j int) bool { return models[i].ID < models[j].ID }) + return models +} + +func probedCapability(capabilityID string, verdict provider.ModalityVerdict) Capability { + adapter := verdict.Adapter + if adapter == "" { + adapter = chatAdapter[capabilityID] + } + return Capability{ + ID: capabilityID, + Source: SourceProbe, + Adapter: adapter, + Supported: verdict.Supported, + Installable: verdict.Supported, + Probe: ProbeKind(adapter), + } +} + +// ChatAdapter is the adapter a capability on a chat model uses, empty when no +// chat route serves it. +func ChatAdapter(capability string) string { + return chatAdapter[capability] +} + +func ordered(claimed map[string]bool) []string { + result := make([]string, 0, len(claimed)) + for _, capability := range Capabilities { + if claimed[capability] { + result = append(result, capability) + } + } + return result +} + +func invalid(message string) error { + return oneerrors.New(oneerrors.InvalidRequest, message) +} diff --git a/internal/modality/resolve_test.go b/internal/modality/resolve_test.go new file mode 100644 index 00000000..6a463f3c --- /dev/null +++ b/internal/modality/resolve_test.go @@ -0,0 +1,245 @@ +package modality + +import ( + "encoding/json" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/MaimoryLab/BootAgent/internal/catalog" + "github.com/MaimoryLab/BootAgent/internal/provider" +) + +// snapshots are the public /models listings fetched on 2026-10-09, trimmed to +// the fields detection reads. +var snapshots = []string{"novita", "ppio", "jiekou"} + +func loadSnapshot(t *testing.T, name string) []provider.ModelEntry { + t.Helper() + data, err := os.ReadFile(filepath.Join("testdata", name+"-models.json")) + if err != nil { + t.Fatal(err) + } + var file struct { + Data []provider.ModelEntry `json:"data"` + } + if err := json.Unmarshal(data, &file); err != nil { + t.Fatal(err) + } + if len(file.Data) == 0 { + t.Fatalf("snapshot %s is empty", name) + } + return file.Data +} + +func embeddedRules(t *testing.T) Rules { + t.Helper() + rules, err := Load() + if err != nil { + t.Fatal(err) + } + return rules +} + +func nonText(values []string) bool { + for _, value := range values { + if value != "text" { + return true + } + } + return false +} + +// The ID rules exist for Providers that declare nothing. Wherever a gateway +// does declare, a rule claiming more than the declaration is a rule that would +// mislead on an undeclaring Provider too. +func TestIDRulesNeverContradictDeclaredMetadata(t *testing.T) { + rules := embeddedRules(t) + for _, name := range snapshots { + for _, entry := range loadSnapshot(t, name) { + if !entry.Declared() { + continue + } + declared := DeclaredCapabilities(entry) + for _, claimed := range rules.MatchIDRules(entry.ID) { + if !slices.Contains(declared, claimed) { + t.Errorf("%s: rule claims %s for %s, which declares %v", name, claimed, entry.ID, declared) + } + } + } + } +} + +func TestEveryIDRuleMatchesTheSnapshots(t *testing.T) { + rules := embeddedRules(t) + hits := make([]int, len(rules.rules)) + multimodal, covered := 0, 0 + for _, name := range snapshots { + for _, entry := range loadSnapshot(t, name) { + key := RuleKey(entry.ID) + for index, rule := range rules.rules { + if rule.pattern.MatchString(key) && (rule.exclude == nil || !rule.exclude.MatchString(key)) { + hits[index]++ + } + } + if nonText(entry.InputModalities) || nonText(entry.OutputModalities) { + multimodal++ + if len(rules.MatchIDRules(entry.ID)) > 0 { + covered++ + } + } + } + } + for index, count := range hits { + if count == 0 { + t.Errorf("ID rule %d (%s) matches no snapshot model", index, rules.rules[index].pattern) + } + } + // The design records 194 of 207; a drop means a rule was narrowed too far. + if multimodal != 207 || covered != 194 { + t.Errorf("rules cover %d of %d declared multimodal entries, want 194 of 207", covered, multimodal) + } +} + +func TestResolveReplaysSnapshotsSafely(t *testing.T) { + rules := embeddedRules(t) + for _, name := range snapshots { + entries := loadSnapshot(t, name) + models := rules.Resolve(name, entries, nil) + if len(models) == 0 { + t.Fatalf("%s: nothing detected", name) + } + for _, model := range models { + for _, capability := range model.Capabilities { + if capability.Source == SourceIDRule && capability.Installable { + t.Errorf("%s: %s %s installable on an ID rule alone", name, model.ID, capability.ID) + } + if probeOnly[capability.Adapter] && capability.Installable { + t.Errorf("%s: %s %s installable on unverified adapter %s without a probe", name, model.ID, capability.ID, capability.Adapter) + } + if capability.ID == TextToSpeech || capability.ID == VideoGeneration { + t.Errorf("%s: %s offered %s from a chat listing", name, model.ID, capability.ID) + } + } + } + } +} + +func TestResolveOrder(t *testing.T) { + rules := embeddedRules(t) + entries := []provider.ModelEntry{ + {ID: "moonshotai/kimi-k3"}, + {ID: "qwen/qwen3-vl-8b", InputModalities: []string{"text", "image"}, OutputModalities: []string{"text"}}, + // Declared text-only: the Provider's word beats the ID rule. + {ID: "claude-sonnet-x", InputModalities: []string{"text"}, OutputModalities: []string{"text"}}, + {ID: "plain-text-model"}, + {ID: "gemini-3-pro-image", InputModalities: []string{"text", "image"}, OutputModalities: []string{"text", "image"}}, + } + verdicts := map[string]map[string]provider.ModalityVerdict{ + "moonshotai/kimi-k3": {Vision: {Supported: true, Source: SourceProbe, Adapter: AdapterChatVision}}, + "qwen/qwen3-vl-8b": {Vision: {Supported: false, Source: SourceProbe, Adapter: AdapterChatVision}}, + } + models := rules.Resolve("custom", entries, verdicts) + byID := map[string]Model{} + for _, model := range models { + byID[model.ID] = model + } + if _, ok := byID["claude-sonnet-x"]; ok { + t.Error("an ID rule overrode a text-only declaration") + } + if _, ok := byID["plain-text-model"]; ok { + t.Error("a model nothing claims was offered") + } + kimi := byID["moonshotai/kimi-k3"].Capabilities + if len(kimi) != 2 || kimi[0] != (Capability{ID: Vision, Source: SourceProbe, Adapter: AdapterChatVision, Supported: true, Installable: true, Probe: ProbeMinimal}) { + t.Fatalf("kimi vision = %#v", kimi) + } + if kimi[1].ID != VideoUnderstanding || kimi[1].Source != SourceIDRule || kimi[1].Installable { + t.Fatalf("kimi video = %#v", kimi[1]) + } + qwen := byID["qwen/qwen3-vl-8b"].Capabilities + if len(qwen) != 1 || qwen[0].Supported || qwen[0].Installable || qwen[0].Source != SourceProbe { + t.Fatalf("refused probe = %#v", qwen) + } + gemini := byID["gemini-3-pro-image"].Capabilities + if len(gemini) != 2 || !gemini[0].Installable || gemini[1].ID != ImageGeneration || gemini[1].Installable || gemini[1].Adapter != AdapterChatImageOutput { + t.Fatalf("declared image output = %#v", gemini) + } +} + +func TestResolveAddsCatalogGenerationModels(t *testing.T) { + manifest := catalog.ModalityManifest{ + SchemaVersion: 1, + IDRules: []catalog.ModalityIDRule{{Pattern: "^claude-", Capabilities: []string{Vision}}}, + Providers: map[string]catalog.ModalityProvider{ + "novita": {NativeBaseURL: "https://api.novita.ai", Generation: []catalog.ModalityGenerationModel{ + {ID: "kling-v3.0-4k-t2v", Capabilities: []string{VideoGeneration}, Adapter: AdapterNovitaAsync}, + }}, + }, + } + rules, err := NewRules(manifest) + if err != nil { + t.Fatal(err) + } + models := rules.Resolve("novita", nil, nil) + if len(models) != 1 || models[0].Capabilities[0] != (Capability{ID: VideoGeneration, Source: SourceCatalog, Adapter: AdapterNovitaAsync, Supported: true, Installable: true}) { + t.Fatalf("catalog models = %#v", models) + } + if got := rules.Resolve("ppio", nil, nil); len(got) != 0 { + t.Fatalf("another Provider got novita's catalog: %#v", got) + } + if rules.NativeBaseURL("novita") != "https://api.novita.ai" || rules.NativeBaseURL("deepseek") != "" { + t.Fatal("native base lookup") + } +} + +func TestNewRulesRejectsWordsOutsideTheVocabulary(t *testing.T) { + base := func() catalog.ModalityManifest { + return catalog.ModalityManifest{SchemaVersion: 1, IDRules: []catalog.ModalityIDRule{{Pattern: "^x", Capabilities: []string{Vision}}}} + } + cases := map[string]func(*catalog.ModalityManifest){ + "unknown capability": func(m *catalog.ModalityManifest) { m.IDRules[0].Capabilities = []string{"smell"} }, + // No chat route speaks text-to-speech, so a rule claiming it could + // never be honoured. + "rule without chat route": func(m *catalog.ModalityManifest) { m.IDRules[0].Capabilities = []string{TextToSpeech} }, + "unknown adapter": func(m *catalog.ModalityManifest) { + m.Providers = map[string]catalog.ModalityProvider{"novita": {NativeBaseURL: "https://api.novita.ai", Generation: []catalog.ModalityGenerationModel{{ID: "m", Capabilities: []string{VideoGeneration}, Adapter: "fax"}}}} + }, + "unknown generation capability": func(m *catalog.ModalityManifest) { + m.Providers = map[string]catalog.ModalityProvider{"novita": {NativeBaseURL: "https://api.novita.ai", Generation: []catalog.ModalityGenerationModel{{ID: "m", Capabilities: []string{"smell"}, Adapter: AdapterNovitaAsync}}}} + }, + } + for name, mutate := range cases { + manifest := base() + mutate(&manifest) + if _, err := NewRules(manifest); err == nil { + t.Errorf("%s accepted", name) + } + } +} + +func TestRuleKeyStripsVendorAndCase(t *testing.T) { + for input, want := range map[string]string{ + "moonshotai/kimi-k3": "kimi-k3", + "Claude-Opus-5-5": "claude-opus-5-5", + " a/b/qwen3-vl-8b ": "qwen3-vl-8b", + } { + if got := RuleKey(input); got != want { + t.Errorf("RuleKey(%q) = %q, want %q", input, got, want) + } + } +} + +func TestDeclaredCapabilitiesIgnoreChatAudioAndVideoOutput(t *testing.T) { + got := DeclaredCapabilities(provider.ModelEntry{ + ID: "gpt-image-2.5-flare", + InputModalities: []string{"text", "image", "video", "audio"}, + OutputModalities: []string{"text", "image", "video", "audio"}, + }) + want := []string{Vision, ImageGeneration, SpeechToText, VideoUnderstanding} + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("declared = %v, want %v", got, want) + } +} diff --git a/internal/provider/capability.go b/internal/provider/capability.go index c0326263..1021e14a 100644 --- a/internal/provider/capability.go +++ b/internal/provider/capability.go @@ -15,7 +15,12 @@ import ( "github.com/MaimoryLab/BootAgent/internal/securefs" ) -const capabilitySchemaVersion = 1 +// capabilitySchemaVersion 2 added per-model modality verdicts. Version 1 files +// are read as version 2 with no modalities: discarding them would throw away +// protocol verdicts and make the next activation spend a billed probe. +const capabilitySchemaVersion = 2 + +const legacyCapabilitySchemaVersion = 1 // ProtocolCapability is one probe verdict. ErrorCode is kept alongside Supported // because routing has to tell "this endpoint refuses this protocol" apart from "the @@ -35,6 +40,22 @@ type ProtocolCapability struct { type ModelCapabilities struct { ProbedAt time.Time `json:"probed_at"` Protocols map[string]ProtocolCapability `json:"protocols"` + // Modalities holds probe verdicts for multimodal capabilities, keyed by + // capability ID. Only probe verdicts are cached: catalog matches and + // declared metadata are recomputed on every read. + Modalities map[string]ModalityVerdict `json:"modalities,omitempty"` +} + +// ModalityVerdict is the outcome of one capability probe the user asked for. +// Only conclusive answers are stored -- the Provider accepted the request, or +// refused this model -- because a busy endpoint or a rejected key says nothing +// about what the model can do. +type ModalityVerdict struct { + Supported bool `json:"supported"` + Source string `json:"source"` + Adapter string `json:"adapter"` + ProbedAt time.Time `json:"probed_at"` + Status int `json:"status,omitempty"` } // ProviderCapabilities groups every probed model for one Provider. Fingerprint binds @@ -114,7 +135,7 @@ func (s CapabilityStore) Save(ctx context.Context, entry Entry, model string, ve } existing, ok := current.Models[model] if !ok || existing.Protocols == nil { - existing = ModelCapabilities{Protocols: map[string]ProtocolCapability{}} + existing.Protocols = map[string]ProtocolCapability{} } for protocolID, verdict := range verdicts { existing.Protocols[protocolID] = verdict @@ -125,6 +146,64 @@ func (s CapabilityStore) Save(ctx context.Context, entry Entry, model string, ve return s.write(ctx, file) } +// Modalities returns every cached modality verdict for one Provider, keyed by +// model and then capability. A stale fingerprint is a miss, like Get. +func (s CapabilityStore) Modalities(entry Entry) map[string]map[string]ModalityVerdict { + result := map[string]map[string]ModalityVerdict{} + file, err := s.load() + if err != nil { + return result + } + saved, ok := file.Providers[entry.ID] + if !ok || saved.Fingerprint != Fingerprint(entry) { + return result + } + for model, record := range saved.Models { + if len(record.Modalities) == 0 { + continue + } + verdicts := make(map[string]ModalityVerdict, len(record.Modalities)) + for capability, verdict := range record.Modalities { + verdicts[capability] = verdict + } + result[model] = verdicts + } + return result +} + +// SaveModality records one capability probe verdict for a model. Protocol +// verdicts and other capabilities of the same model are kept. +func (s CapabilityStore) SaveModality(ctx context.Context, entry Entry, model, capability string, verdict ModalityVerdict) error { + model = strings.TrimSpace(model) + capability = strings.TrimSpace(capability) + if model == "" || capability == "" { + return oneerrors.New(oneerrors.InvalidRequest, "A capability verdict must name the model and capability it was measured with") + } + file, err := s.load() + if err != nil { + file = newCapabilityFile() + } + fingerprint := Fingerprint(entry) + current, ok := file.Providers[entry.ID] + if !ok || current.Fingerprint != fingerprint || current.Models == nil { + current = ProviderCapabilities{Fingerprint: fingerprint, Models: map[string]ModelCapabilities{}} + } + existing := current.Models[model] + if existing.Protocols == nil { + existing.Protocols = map[string]ProtocolCapability{} + } + if existing.Modalities == nil { + existing.Modalities = map[string]ModalityVerdict{} + } + if verdict.ProbedAt.IsZero() { + verdict.ProbedAt = time.Now().UTC() + } + existing.Modalities[capability] = verdict + current.Models[model] = existing + file.Providers[entry.ID] = current + return s.write(ctx, file) +} + // Invalidate drops a Provider's records after an edit that changes what a probe would // answer. A stale fingerprint is already caught on read, so this reclaims the entry // rather than upholding correctness on its own. @@ -163,7 +242,16 @@ func (s CapabilityStore) load() (capabilityFile, error) { return newCapabilityFile(), err } file := newCapabilityFile() - if err := json.Unmarshal(data, &file); err != nil || file.SchemaVersion != capabilitySchemaVersion || file.Providers == nil { + if err := json.Unmarshal(data, &file); err != nil || file.Providers == nil { + return newCapabilityFile(), errors.New("cached Provider capabilities are unusable") + } + switch file.SchemaVersion { + case capabilitySchemaVersion: + case legacyCapabilitySchemaVersion: + // Same shape minus modalities, which unmarshal as nil. The next write + // stamps the current version. + file.SchemaVersion = capabilitySchemaVersion + default: return newCapabilityFile(), errors.New("cached Provider capabilities are unusable") } return file, nil diff --git a/internal/provider/capability_test.go b/internal/provider/capability_test.go index 4b29a88a..01c96b59 100644 --- a/internal/provider/capability_test.go +++ b/internal/provider/capability_test.go @@ -3,6 +3,7 @@ package provider import ( "context" "os" + "path/filepath" "strings" "testing" @@ -202,3 +203,98 @@ func TestCapabilitiesTreatCorruptFileAsAMiss(t *testing.T) { t.Fatal("the cache did not recover after a save") } } + +// A version 1 file must survive the upgrade: dropping it would throw away +// protocol verdicts and make the next activation spend a billed probe. +func TestCapabilitiesReadVersionOneAsVersionTwo(t *testing.T) { + home := t.TempDir() + store := capabilityStore(t, home) + entry := probedEntry() + legacy := `{"schema_version":1,"providers":{"relay":{"fingerprint":"` + Fingerprint(entry) + `","models":{"gpt-test":{"probed_at":"2026-09-01T00:00:00Z","protocols":{"openai":{"supported":true,"reachable":true,"status":200}}}}}}}` + if err := os.MkdirAll(filepath.Dir(store.Path()), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(store.Path(), []byte(legacy), 0o600); err != nil { + t.Fatal(err) + } + saved, ok := store.Get(entry, "gpt-test") + if !ok || !saved.Protocols[ProtocolOpenAI].Supported || saved.Modalities != nil { + t.Fatalf("version 1 record = %#v, %v", saved, ok) + } + if err := store.SaveModality(context.Background(), entry, "gpt-test", "vision", ModalityVerdict{Supported: true, Source: "probe", Adapter: "openai-chat-vision"}); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(store.Path()) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(data), `"schema_version": 2`) { + t.Fatalf("rewritten file is not version 2: %s", data) + } + saved, ok = store.Get(entry, "gpt-test") + if !ok || !saved.Protocols[ProtocolOpenAI].Supported || !saved.Modalities["vision"].Supported { + t.Fatalf("upgrade lost a verdict: %#v", saved) + } +} + +func TestCapabilitiesRejectUnknownSchemaVersion(t *testing.T) { + store := capabilityStore(t, t.TempDir()) + entry := probedEntry() + if err := os.MkdirAll(filepath.Dir(store.Path()), 0o700); err != nil { + t.Fatal(err) + } + future := `{"schema_version":3,"providers":{"relay":{"fingerprint":"` + Fingerprint(entry) + `","models":{"m":{"protocols":{"openai":{"supported":true}}}}}}}` + if err := os.WriteFile(store.Path(), []byte(future), 0o600); err != nil { + t.Fatal(err) + } + if _, ok := store.Get(entry, "m"); ok { + t.Fatal("a file from a newer schema was trusted") + } +} + +func TestModalityVerdictsKeepProtocolsAndFollowTheFingerprint(t *testing.T) { + store := capabilityStore(t, t.TempDir()) + entry := probedEntry() + ctx := context.Background() + if err := store.Save(ctx, entry, "m", map[string]ProtocolCapability{ProtocolOpenAI: {Supported: true, Reachable: true}}); err != nil { + t.Fatal(err) + } + if err := store.SaveModality(ctx, entry, "m", "vision", ModalityVerdict{Supported: false, Source: "probe", Adapter: "openai-chat-vision", Status: 400}); err != nil { + t.Fatal(err) + } + // Saving protocols afterwards must not drop the modality, and vice versa. + if err := store.Save(ctx, entry, "m", map[string]ProtocolCapability{ProtocolResponses: {Reachable: true, Status: 404}}); err != nil { + t.Fatal(err) + } + saved, ok := store.Get(entry, "m") + if !ok || !saved.Protocols[ProtocolOpenAI].Supported || len(saved.Protocols) != 2 { + t.Fatalf("protocols = %#v", saved.Protocols) + } + all := store.Modalities(entry) + verdict, ok := all["m"]["vision"] + if !ok || verdict.Supported || verdict.Status != 400 || verdict.ProbedAt.IsZero() { + t.Fatalf("modality verdicts = %#v", all) + } + // A model with only a modality verdict is still a protocol miss. + if err := store.SaveModality(ctx, entry, "vision-only", "vision", ModalityVerdict{Supported: true, Source: "probe"}); err != nil { + t.Fatal(err) + } + if _, ok := store.Get(entry, "vision-only"); ok { + t.Fatal("a modality verdict answered a protocol lookup") + } + changed := entry + changed.APIKey = "sk-another-value" + if got := store.Modalities(changed); len(got) != 0 { + t.Fatalf("verdicts survived a key change: %#v", got) + } + if err := store.SaveModality(ctx, entry, "", "vision", ModalityVerdict{}); err == nil { + t.Fatal("verdict without a model accepted") + } + data, err := os.ReadFile(store.Path()) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(data), entry.APIKey) { + t.Fatal("capability file contains the API key") + } +} diff --git a/internal/provider/client.go b/internal/provider/client.go index 436987f1..18227a1d 100644 --- a/internal/provider/client.go +++ b/internal/provider/client.go @@ -62,6 +62,25 @@ type ModelsResult struct { Retryable bool `json:"retryable"` Protocol *string `json:"protocol"` Models []string `json:"models"` + // Entries carries what each listed model declares about itself, in the same + // order as Models. Most callers only need the IDs; multimodal detection reads + // the modality lists. + Entries []ModelEntry `json:"entries,omitempty"` +} + +// ModelEntry is one /models item. The modality lists are nil when the +// Provider did not send the field at all, and empty when it sent an empty +// list: only the second is a declaration. +type ModelEntry struct { + ID string `json:"id"` + InputModalities []string `json:"input_modalities"` + OutputModalities []string `json:"output_modalities"` +} + +// Declared reports whether the Provider said anything about this model's +// modalities. +func (e ModelEntry) Declared() bool { + return e.InputModalities != nil || e.OutputModalities != nil } func (c *Client) Probe(ctx context.Context, protocol, providerID, apiKey, model, customBase string) (ProbeResult, error) { @@ -79,6 +98,56 @@ func (c *Client) Probe(ctx context.Context, protocol, providerID, apiKey, model, if err != nil { return ProbeResult{}, err } + return c.sendProbe(request, protocol, apiKey, requestModel), nil +} + +// probePNG is a 32x32 opaque grey PNG. Some vision endpoints refuse images +// smaller than 10 pixels a side, so the design's 8x8 would fail on them for a +// reason that says nothing about the model; 32x32 is still a few tokens. +const probePNG = "iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAAAAABWESUoAAAAFklEQVR42mNoIAAYRhWMKhhVMFIVAABCAAAfN5c2AAAAAABJRU5ErkJggg==" + +// ProbeVision sends one Chat Completions request carrying a small image, with +// max_tokens 1. It is a billed request, so only a user's click may send it. +// The answer is classified exactly like a protocol probe: a model that cannot +// take images is refused with a 4xx, which reads as "refused", not as a key or +// network problem. +func (c *Client) ProbeVision(ctx context.Context, apiKey, model, customBase string) (ProbeResult, error) { + if apiKey == "" { + return ProbeResult{}, oneerrors.New(oneerrors.InvalidRequest, "API key is required") + } + if strings.TrimSpace(model) == "" { + return ProbeResult{}, oneerrors.New(oneerrors.InvalidRequest, "A model is required") + } + base, err := ProviderBase("custom", customBase) + if err != nil { + return ProbeResult{}, err + } + payload, err := json.Marshal(map[string]any{ + "model": model, + "messages": []map[string]any{{ + "role": "user", + "content": []map[string]any{ + {"type": "text", "text": "ok"}, + {"type": "image_url", "image_url": map[string]string{"url": "data:image/png;base64," + probePNG}}, + }, + }}, + "max_tokens": 1, + }) + if err != nil { + return ProbeResult{}, oneerrors.New(oneerrors.InternalError, "Unable to build Provider request", oneerrors.WithCause(err)) + } + request, err := http.NewRequestWithContext(ctx, http.MethodPost, OpenAIBaseURL(base)+"/chat/completions", bytes.NewReader(payload)) + if err != nil { + return ProbeResult{}, oneerrors.New(oneerrors.InvalidRequest, "Provider endpoint is invalid", oneerrors.WithCause(err)) + } + request.Header.Set("Authorization", "Bearer "+apiKey) + request.Header.Set("Content-Type", "application/json") + return c.sendProbe(request, ProtocolOpenAI, apiKey, model), nil +} + +// sendProbe runs one probe request and classifies the answer. It never +// returns an error: every outcome, transport failures included, is a verdict. +func (c *Client) sendProbe(request *http.Request, protocol, apiKey, requestModel string) ProbeResult { response, cancelRequest, err := c.do(request) defer cancelRequest() if err != nil { @@ -87,12 +156,12 @@ func (c *Client) Probe(ctx context.Context, protocol, providerID, apiKey, model, } result := transportResult(err) result.Protocol = new(protocol) - return result, nil + return result } if response == nil { result := transportResult(errors.New("Provider transport returned no response")) result.Protocol = new(protocol) - return result, nil + return result } defer response.Body.Close() if response.StatusCode >= 200 && response.StatusCode < 300 { @@ -110,12 +179,12 @@ func (c *Client) Probe(ctx context.Context, protocol, providerID, apiKey, model, Message: fmt.Sprintf("%s connection test passed.", ProtocolLabel(protocol)), Retryable: false, Protocol: new(protocol), - }, nil + } } body, _, _ := c.readBody(response.Body) result := classifyHTTPProbe(response.StatusCode, string(body), protocol, requestModel) result.Detail = upstreamDetail(body, apiKey) - return result, nil + return result } // maxDetailRunes bounds the provider's own explanation carried back to the UI: @@ -213,8 +282,12 @@ func (c *Client) ListModels(ctx context.Context, providerID, apiKey, customBase Models: []string{}, }, nil } - models := modelIDs(raw) - result := ModelsResult{Reachable: true, Status: response.StatusCode, Models: models} + entries := modelEntries(raw) + models := make([]string, len(entries)) + for index, entry := range entries { + models[index] = entry.ID + } + result := ModelsResult{Reachable: true, Status: response.StatusCode, Models: models, Entries: entries} if len(models) > 0 { result.OK = true result.Message = fmt.Sprintf("Found %d models.", len(models)) @@ -429,27 +502,51 @@ func modelsFailure(message string) ModelsResult { } } -func modelIDs(raw any) []string { +func modelEntries(raw any) []ModelEntry { data := raw if object, ok := raw.(map[string]any); ok { data = object["data"] } items, ok := data.([]any) if !ok { - return []string{} + return []ModelEntry{} } - models := make([]string, 0, len(items)) + entries := make([]ModelEntry, 0, len(items)) for _, item := range items { switch value := item.(type) { case string: - models = append(models, value) + entries = append(entries, ModelEntry{ID: value}) case map[string]any: if id, ok := value["id"]; ok && id != nil { - models = append(models, fmt.Sprint(id)) + entries = append(entries, ModelEntry{ + ID: fmt.Sprint(id), + InputModalities: modalityList(value, "input_modalities"), + OutputModalities: modalityList(value, "output_modalities"), + }) } } } - return models + return entries +} + +// modalityList keeps the absent/empty distinction ModelEntry documents, and +// drops anything that is not a string: the listing is third-party data. +func modalityList(item map[string]any, key string) []string { + raw, ok := item[key] + if !ok || raw == nil { + return nil + } + values, ok := raw.([]any) + if !ok { + return nil + } + list := make([]string, 0, len(values)) + for _, value := range values { + if text, ok := value.(string); ok { + list = append(list, strings.ToLower(strings.TrimSpace(text))) + } + } + return list } func unsupportedProtocol(status int, body string) bool { diff --git a/internal/provider/client_test.go b/internal/provider/client_test.go index eddf1508..f5a86335 100644 --- a/internal/provider/client_test.go +++ b/internal/provider/client_test.go @@ -1,9 +1,12 @@ package provider import ( + "bytes" "context" + "encoding/base64" "encoding/json" "errors" + "image/png" "io" "net/http" "net/url" @@ -300,3 +303,100 @@ func TestUpstreamDetailExtractsCleansAndBounds(t *testing.T) { } } } + +func TestListModelsKeepsDeclaredModalities(t *testing.T) { + client := NewClient(fakeDoer(func(*http.Request) (*http.Response, error) { + return fakeResponse(http.StatusOK, `{"data":[ + {"id":"vision-a","input_modalities":["text","Image"],"output_modalities":["text"]}, + {"id":"silent","input_modalities":[],"output_modalities":null}, + {"id":"bare"}, + {"id":"odd","input_modalities":"image"}, + "plain" + ]}`), nil + })) + result, err := client.ListModels(context.Background(), "custom", "key", "https://proxy.test") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(result.Models, []string{"vision-a", "silent", "bare", "odd", "plain"}) { + t.Fatalf("models = %v", result.Models) + } + if len(result.Entries) != len(result.Models) { + t.Fatalf("entries = %#v", result.Entries) + } + if got := result.Entries[0]; !got.Declared() || !reflect.DeepEqual(got.InputModalities, []string{"text", "image"}) { + t.Fatalf("declared entry = %#v", got) + } + // An explicitly empty list is a declaration of "nothing"; an absent field + // says nothing at all, and neither does a value of the wrong shape. + if !result.Entries[1].Declared() { + t.Fatalf("empty list should count as declared: %#v", result.Entries[1]) + } + for _, index := range []int{2, 3, 4} { + if result.Entries[index].Declared() { + t.Fatalf("entry %d should not be declared: %#v", index, result.Entries[index]) + } + } +} + +func TestProbeVisionSendsOneImageWithOneToken(t *testing.T) { + var seen *http.Request + var body map[string]any + client := NewClient(fakeDoer(func(request *http.Request) (*http.Response, error) { + seen = request + if err := json.NewDecoder(request.Body).Decode(&body); err != nil { + t.Fatal(err) + } + return fakeResponse(http.StatusOK, `{}`), nil + })) + result, err := client.ProbeVision(context.Background(), "key", "vision-a", "https://proxy.test/openai") + if err != nil { + t.Fatal(err) + } + if !result.OK || seen.URL.String() != "https://proxy.test/openai/v1/chat/completions" { + t.Fatalf("result = %#v, url = %v", result, seen.URL) + } + if body["max_tokens"] != float64(1) || body["model"] != "vision-a" { + t.Fatalf("body = %#v", body) + } + encoded, _ := json.Marshal(body["messages"]) + if !strings.Contains(string(encoded), `"type":"image_url"`) || !strings.Contains(string(encoded), "data:image/png;base64,") { + t.Fatalf("messages = %s", encoded) + } +} + +func TestProbeVisionClassifiesRefusalAndScrubsKey(t *testing.T) { + client := NewClient(fakeDoer(func(*http.Request) (*http.Response, error) { + return fakeResponse(http.StatusBadRequest, `{"error":{"message":"model text-only does not support image input (key sk-secret)"}}`), nil + })) + result, err := client.ProbeVision(context.Background(), "sk-secret", "text-only", "https://proxy.test") + if err != nil { + t.Fatal(err) + } + if result.OK || !result.Reachable || result.Status != http.StatusBadRequest { + t.Fatalf("result = %#v", result) + } + if strings.Contains(result.Detail, "sk-secret") || !strings.Contains(result.Detail, "does not support image input") { + t.Fatalf("detail = %q", result.Detail) + } + if _, err := client.ProbeVision(context.Background(), "", "m", "https://proxy.test"); err == nil { + t.Fatal("missing key accepted") + } + if _, err := client.ProbeVision(context.Background(), "k", " ", "https://proxy.test"); err == nil { + t.Fatal("missing model accepted") + } +} + +func TestProbePNGDecodes(t *testing.T) { + data, err := base64.StdEncoding.DecodeString(probePNG) + if err != nil { + t.Fatal(err) + } + config, err := png.DecodeConfig(bytes.NewReader(data)) + if err != nil { + t.Fatal(err) + } + if config.Width < 10 || config.Height < 10 { + t.Fatalf("probe image is %dx%d; some endpoints refuse images under 10px a side", config.Width, config.Height) + } +} diff --git a/manifest_embed.go b/manifest_embed.go index 01e40a99..ae5ac852 100644 --- a/manifest_embed.go +++ b/manifest_embed.go @@ -36,6 +36,12 @@ func EmbeddedRuntimeLock() ([]byte, error) { return LockManifest.ReadFile("manifests/runtimes.lock.json") } +// EmbeddedModalityLock returns a fresh copy of modalities.lock.json, the ID +// rules and per-Provider native routes used by multimodal Skill detection. +func EmbeddedModalityLock() ([]byte, error) { + return LockManifest.ReadFile("manifests/modalities.lock.json") +} + // EmbeddedMarketplaceLock returns a fresh copy of marketplace.lock.json. func EmbeddedMarketplaceLock() ([]byte, error) { return LockManifest.ReadFile("manifests/marketplace.lock.json") diff --git a/manifests/modalities.lock.json b/manifests/modalities.lock.json new file mode 100644 index 00000000..a5a9fd19 --- /dev/null +++ b/manifests/modalities.lock.json @@ -0,0 +1,33 @@ +{ + "schema_version": 1, + "id_rules": [ + { "pattern": "^gpt-image-", "capabilities": ["image-generation"] }, + { "pattern": "^gemini-.*-image", "capabilities": ["vision", "image-generation"] }, + { "pattern": "^ming-image-", "capabilities": ["image-generation"] }, + { "pattern": "(^|-)omni(-|$)", "exclude": "captioner", "capabilities": ["vision", "speech-to-text"] }, + { "pattern": "^gemini-", "exclude": "-image", "capabilities": ["vision", "video-understanding", "speech-to-text"] }, + { "pattern": "^claude-", "capabilities": ["vision"] }, + { "pattern": "^(gpt-4o|gpt-4\\.1|gpt-5|gpt-6|o3)", "exclude": "^gpt-oss", "capabilities": ["vision"] }, + { "pattern": "^grok-(4|code)", "capabilities": ["vision"] }, + { "pattern": "^kimi-k(2\\.[5-9]|3)", "capabilities": ["vision", "video-understanding"] }, + { "pattern": "^deepseek-(v4\\.[1-9]-flash|v4-flash-vision)", "capabilities": ["vision"] }, + { "pattern": "(^|-)ocr(-|$)", "capabilities": ["vision"] }, + { "pattern": "(^|-)vl(-|$)", "capabilities": ["vision"] }, + { "pattern": "^glm-[\\d.]+v", "capabilities": ["vision"] }, + { "pattern": "^gemma-[34]-", "capabilities": ["vision"] }, + { "pattern": "^llama-4-", "capabilities": ["vision"] }, + { "pattern": "^minimax-m3", "capabilities": ["vision", "video-understanding"] }, + { "pattern": "^qwen3\\.(5|6|8)-(\\d+b|flash|max|plus)", "capabilities": ["vision", "video-understanding"] }, + { "pattern": "^mimo-v2\\.(5$|6-)", "capabilities": ["vision", "video-understanding"] } + ], + "providers": { + "novita": { + "native_base_url": "https://api.novita.ai", + "generation": [] + }, + "ppio": { + "native_base_url": "https://api.ppio.com", + "generation": [] + } + } +} From a9b2e5f05b81cabf175dca6ab042c14f10d13812 Mon Sep 17 00:00:00 2001 From: yujiezhang-ops Date: Fri, 9 Oct 2026 16:25:35 +0800 Subject: [PATCH 06/18] feat: install multimodal Skills into Claude Code and Codex Generate a Skill per capability (SKILL.md, config.json and a dependency-free run.mjs) for image understanding, image generation, speech-to-text and text-to-speech over the OpenAI-compatible routes, store it as a generated variant, and publish it to the chosen Agents. The Provider key reaches the script as BOOTAGENT__API_KEY in Claude Code's env block and Codex's shell_environment_policy.set, with a private fallback file for other Agents (ADR-011). One reconcile keeps those variables matching the generated Skills after install, uninstall, Skills page changes and Provider edits; only variants rendered here earn a key. config.toml is written as a secret once it holds one. Add probes for transcription, speech and image generation; the image probe needs an explicit confirmation. Models can be entered by hand and install once their probe passes. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 3 +- README_ZH.md | 3 +- docs/multimodal-skills-design.md | 125 ++++- .../BootAgent/internal/app/index.ts | 5 + .../BootAgent/internal/app/models.ts | 79 +++ .../BootAgent/internal/binding/models.ts | 12 + .../internal/binding/multimodalservice.ts | 17 +- frontend/src/backend/wails.test.ts | 2 +- frontend/src/backend/wails.ts | 16 +- .../src/components/MultimodalSection.test.tsx | 176 +++++- frontend/src/components/MultimodalSection.tsx | 371 +++++++++++-- frontend/src/components/multimodalPrompts.ts | 32 ++ frontend/src/i18n.tsx | 29 + frontend/src/pages/SkillsPage.tsx | 2 +- frontend/src/styles/app.css | 31 ++ frontend/src/types/api.ts | 18 +- internal/app/multimodal.go | 89 ++- internal/app/multimodal_install.go | 518 ++++++++++++++++++ internal/app/multimodal_install_test.go | 386 +++++++++++++ internal/app/multimodal_test.go | 12 +- internal/app/provider.go | 46 ++ internal/app/skill.go | 31 ++ internal/binding/multimodal.go | 29 +- internal/binding/services_test.go | 2 +- internal/config/skill_env.go | 244 +++++++++ internal/config/skill_env_test.go | 348 ++++++++++++ internal/config/write.go | 15 +- internal/mcp/adapter_test.go | 19 + internal/mcp/codex.go | 5 +- internal/modality/capability.go | 44 +- internal/modality/resolve.go | 31 ++ internal/modality/resolve_test.go | 48 ++ internal/provider/client.go | 128 ++++- internal/provider/client_test.go | 80 +++ internal/skill/model.go | 5 + internal/skill/store.go | 9 +- internal/skillgen/script_test.go | 290 ++++++++++ internal/skillgen/skillgen.go | 289 ++++++++++ internal/skillgen/skillgen_test.go | 202 +++++++ internal/skillgen/templates/common.md.tmpl | 29 + .../templates/image-generation.md.tmpl | 21 + internal/skillgen/templates/run.mjs | 369 +++++++++++++ .../skillgen/templates/speech-to-text.md.tmpl | 20 + .../skillgen/templates/text-to-speech.md.tmpl | 21 + internal/skillgen/templates/vision.md.tmpl | 20 + .../testdata/golden/image-generation/SKILL.md | 47 ++ .../golden/image-generation/config.json | 13 + .../testdata/golden/speech-to-text/SKILL.md | 46 ++ .../golden/speech-to-text/config.json | 13 + .../testdata/golden/text-to-speech/SKILL.md | 47 ++ .../golden/text-to-speech/config.json | 13 + .../skillgen/testdata/golden/vision/SKILL.md | 46 ++ .../testdata/golden/vision/config.json | 13 + 53 files changed, 4360 insertions(+), 149 deletions(-) create mode 100644 frontend/src/components/multimodalPrompts.ts create mode 100644 internal/app/multimodal_install.go create mode 100644 internal/app/multimodal_install_test.go create mode 100644 internal/config/skill_env.go create mode 100644 internal/config/skill_env_test.go create mode 100644 internal/skillgen/script_test.go create mode 100644 internal/skillgen/skillgen.go create mode 100644 internal/skillgen/skillgen_test.go create mode 100644 internal/skillgen/templates/common.md.tmpl create mode 100644 internal/skillgen/templates/image-generation.md.tmpl create mode 100644 internal/skillgen/templates/run.mjs create mode 100644 internal/skillgen/templates/speech-to-text.md.tmpl create mode 100644 internal/skillgen/templates/text-to-speech.md.tmpl create mode 100644 internal/skillgen/templates/vision.md.tmpl create mode 100644 internal/skillgen/testdata/golden/image-generation/SKILL.md create mode 100644 internal/skillgen/testdata/golden/image-generation/config.json create mode 100644 internal/skillgen/testdata/golden/speech-to-text/SKILL.md create mode 100644 internal/skillgen/testdata/golden/speech-to-text/config.json create mode 100644 internal/skillgen/testdata/golden/text-to-speech/SKILL.md create mode 100644 internal/skillgen/testdata/golden/text-to-speech/config.json create mode 100644 internal/skillgen/testdata/golden/vision/SKILL.md create mode 100644 internal/skillgen/testdata/golden/vision/config.json diff --git a/README.md b/README.md index 97b0bb0e..5b15d7a2 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,8 @@ Launch BootAgent ### Manage Skills and MCP servers - Keep a local Skill library, scan Skills already present on supported Agents, and choose which Agents receive each Skill. -- See which models on your keyed Providers can understand images, video or audio, or generate images, and where each finding comes from (declared by the Provider, inferred from the model name, or probed). Listing models is free; a probe is a real request and is sent only when you click it. +- See which models on your keyed Providers can understand images, video or audio, or generate images, and where each finding comes from (declared by the Provider, inferred from the model name, or probed). Listing models is free; a probe is a real request and is sent only when you click it, and an image-generation probe asks first. +- Install a multimodal Skill (image understanding, image generation, speech-to-text, text-to-speech) into Claude Code or Codex, with example prompts to copy. Its script needs Node.js 18 or later. BootAgent writes the Provider key into the Agent's own configuration as `BOOTAGENT__API_KEY`, so the Agent and its model can read it; the install confirmation says so. Uninstalling the last Skill that uses a Provider removes the variable. - Discover MCP servers from initialized Claude Code, Codex, OpenCode, Kilo CLI, and Hermes installations. - Store MCP servers in a local user-level Registry, select sync targets explicitly, and apply changes only when you are ready. diff --git a/README_ZH.md b/README_ZH.md index ec183a0c..aad6f479 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -61,7 +61,8 @@ BootAgent 是一个本地桌面工作台，用来统一管理 AI 编程 Agent。 ### 管理 Skills 和 MCP 服务器 - 在本机维护 Skill 管理库，扫描支持的 Agent 已有的 Skill，并选择每个 Skill 要同步到哪些 Agent。 -- 查看已保存 API Key 的 Provider 中哪些模型能理解图片、视频或音频，或能生成图片，以及每项结论的来源（Provider 声明、按模型名推断或实测）。列出模型不收费；实测是一次真实请求，只在你点击时发出。 +- 查看已保存 API Key 的 Provider 中哪些模型能理解图片、视频或音频，或能生成图片，以及每项结论的来源（Provider 声明、按模型名推断或实测）。列出模型不收费；实测是一次真实请求，只在你点击时发出，图片生成实测会先确认。 +- 把多模态 Skill（图片理解、图片生成、语音识别、语音合成）安装到 Claude Code 或 Codex，并提供可复制的示例提示词。Skill 脚本需要 Node.js 18 或更高版本。BootAgent 会把 Provider 的 Key 以 `BOOTAGENT__API_KEY` 写进 Agent 自己的配置，因此 Agent 及其模型能读到它，安装确认里会说明。卸载最后一个用到该 Provider 的 Skill 时，这个变量会被移除。 - 从已初始化的 Claude Code、Codex、OpenCode、Kilo CLI 和 Hermes 中发现 MCP 服务器。 - 使用本机用户级 Registry 保存 MCP 服务器，明确选择同步目标，并在确认后再应用改动。 diff --git a/docs/multimodal-skills-design.md b/docs/multimodal-skills-design.md index 54920afc..0fc91366 100644 --- a/docs/multimodal-skills-design.md +++ b/docs/multimodal-skills-design.md @@ -1,6 +1,6 @@ # Multimodal Skills: Design -Status: In progress. Phase 1 (detection) is implemented. Branch `codex/multimodal-skills`. +Status: In progress. Phase 1 (detection) and Phase 2 (generation and install for Claude Code and Codex) are implemented. Branch `codex/multimodal-skills`. ## Goal @@ -213,12 +213,39 @@ refused / key / transient / network classification from | Capability | Probe | Cost | | --- | --- | --- | | Image understanding | Chat request with a 32x32 PNG, `max_tokens: 1` | Minimal | -| Speech-to-text | 0.5 s of silent WAV | Minimal | -| Text-to-speech | The text `ok` | Minimal | -| Image generation | Smallest size the adapter allows, one image | One image; the button says so and asks first | +| Speech-to-text | 0.5 s of silent 16 kHz WAV to `/audio/transcriptions` | Minimal | +| Text-to-speech | The text `ok` to `/audio/speech`, voice `alloy` | Minimal | +| Image generation | One image from `/images/generations`, no size sent | One image; the button says so and a dialog asks first, and the backend refuses the probe without that confirmation | | Video generation | Not probed | A probe is a full billed generation that cannot be cancelled reliably, so this capability is catalog-only | | Video understanding | Native video models: catalog-only. Frame sampling: inherits the image-understanding verdict | None | +Implementation notes: + +- The probe image is 32x32 rather than the 8x8 first planned: some vision + endpoints refuse images under 10 pixels a side, which would fail the probe for + a reason unrelated to the model. It still costs a few tokens. +- The image probe sends no size. "The smallest size the adapter allows" is not + knowable for a generic OpenAI-compatible route: valid sizes differ by model + (`dall-e-2` allows 256x256, `gpt-image-1` does not), and a rejected size would + fail the probe for a reason unrelated to the endpoint. The confirmation says it + is one billed image. +- A chat probe keeps the 10 s client timeout; audio probes allow 60 s and the + image probe 180 s, since producing one image can take a minute. +- The image probe ships in Phase 2, not Phase 3 as first planned: without it a + hand-entered image model (below) could never be installed. + +### Models entered by hand + +`/models` does not list dedicated image, transcription or speech models, so +detection alone would leave image generation, speech-to-text and text-to-speech +with nothing to install on most Providers. The section therefore lets the user +type a model ID they know (`gpt-image-1`, `whisper-1`, `tts-1`) for those +capabilities and image understanding. The ID is probed on the capability's +OpenAI-compatible route (`openai-images`, `openai-audio-transcriptions`, +`openai-audio-speech`, `openai-chat-vision`), and installs only once that probe +passes. This is the user naming a model, not BootAgent guessing an endpoint; a +probe verdict for a model in no listing is still shown and installable. + ### Cache `~/.bootagent/capabilities.json` keeps probe verdicts under the existing @@ -249,12 +276,9 @@ cached; only probe verdicts are, and only conclusive ones: the Provider accepted the request, or refused it for this model. A rejected key, a busy endpoint or a network failure says nothing about the model and is not stored. -The probe image is 32x32 rather than the 8x8 first planned: some vision -endpoints refuse images under 10 pixels a side, which would fail the probe for a -reason unrelated to the model. It still costs a few tokens. Today the store treats any schema version other -than its own as a miss. It must accept version 1 and read it as version 2 with no -modalities: discarding it would throw away the protocol verdicts and make the -next activation spend a billed probe to rebuild them. The file stays derived +The store reads a version 1 file as version 2 with no modalities, and writes +version 2. Discarding version 1 would throw away the protocol verdicts and make +the next activation spend a billed probe to rebuild them. The file stays derived data that may be deleted at any time. ## Generated Skill @@ -274,10 +298,26 @@ IDs: `bootagent-vision`, `bootagent-image-generation`, `bootagent-speech-to-text `bootagent-video-understanding`. `SKILL.md` has the frontmatter Agents use to decide when to load a Skill, then: -when to use it, the exact command (`node scripts/run.mjs …`) with -every argument, where outputs are written, what each exit code means and what to -tell the user, and two rules: confirm with the user before any generation that -costs money, and never print environment variables. +when to use it, the exact command (`node /scripts/run.mjs + …`) with every argument, where outputs are written, what each exit +code means and what to tell the user, and two rules: confirm with the user before +any generation that costs money, and never print environment variables. The +description is emitted as a quoted YAML scalar, so a Provider name with quotes +or a colon cannot break the frontmatter. + +| Capability | Subcommand | Output | +| --- | --- | --- | +| Image understanding | `describe --image … [--prompt] [--max-tokens]` | The answer on stdout | +| Image generation | `generate --prompt [--size] [--n 1-4] [--out ]` | Saved file paths on stdout | +| Speech-to-text | `transcribe --audio [--language] [--prompt]` | The transcript on stdout | +| Text-to-speech | `speak --text [--voice] [--format] [--out]` | The saved file path on stdout | + +Every Skill also has `check`, which confirms the key is available without +sending a request. Exit codes: 0 success, 2 bad arguments or input, 3 setup +(Node older than 18, no key), 4 refused by the Provider, 5 key rejected, 6 +unreachable, busy or timed out. Outputs default to `bootagent-output/` in the +working directory and never overwrite an existing file. Result URLs are +downloaded without the key, because they may be on another host. `config.json` is non-secret: Provider ID, adapter, model, base URLs and the name of the environment variable that holds the key. The key itself is never in @@ -299,9 +339,16 @@ self-contained trees through the existing hash, variant and backup machinery, and a shared script outside the tree would bypass it. Changing the model or Provider regenerates the tree. A new hash is a new variant, -published over the previous managed one by `ApplySkills`; the existing managed -target check still refuses to overwrite an unmanaged Skill with the same ID. -`"generated"` joins `agent`, `folder` and `zip` as an import source. +published over the previous managed one by the install use case, which replaces +what is at the destination only when its hash is a stored variant of the same +Skill. It does not go through `ApplySkills`, contrary to the first draft: +`ApplySkills` takes a preview token the generator has no use for, and its +`verifyManagedPublishTarget` compares the destination with the *new* hash, so it +refuses to replace one managed variant with another. A Skill of the same ID that +BootAgent did not write is left alone, per Agent. + +`"generated"` joins `agent`, `folder` and `zip` as an import source in the +registry. The import path still refuses it: only the generator writes it. ## Credential delivery @@ -312,7 +359,7 @@ secret and stay in `config.json`. | Agent | Where the variable is written | Notes | | --- | --- | --- | | Claude Code | `env` in `~/.claude/settings.json` | Already a secret file under ADR-008; the Bash tool inherits `env`. | -| Codex | `[shell_environment_policy] set` in `~/.codex/config.toml` | `set` is applied after Codex's default `KEY`/`SECRET`/`TOKEN` exclusion, so the variable survives it. `config.toml` then carries a key and must be written as a secret (backups included), which it is not today. | +| Codex | `[shell_environment_policy.set]` in `~/.codex/config.toml` | `set` is applied after Codex's default `KEY`/`SECRET`/`TOKEN` exclusion, so the variable survives it. The table is merged with the user's own entries and placed after the user's tables; inline-table and dotted-key spellings of it are refused rather than duplicated. Every writer of `config.toml` (Profile, MCP, Skill) writes it as a secret whenever the old or the new content holds a `BOOTAGENT_*` key, since the backup it takes is of the old content. | | OpenCode | To be confirmed | How its shell tool receives configured environment variables is not yet checked. | | Hermes | To be confirmed | Same as OpenCode. | @@ -320,14 +367,28 @@ Fallback for an Agent with no configurable environment: a private `~/.bootagent/skill-env/.env`, the same pattern ADR-008 uses for Aider. `run.mjs` reads it only when the variable is not already set. -Lifecycle: +Lifecycle. One reconcile makes every Agent's `BOOTAGENT_*` variables exactly +what its generated Skills need, and runs after every change that can alter that: - Install writes the variable for each target Agent. - Saving a Provider with a new key rewrites it, on the same reapply path that - already rewrites Agent configs (`SaveProvider` reports it in `reapplied`). -- Uninstalling the last Skill that uses a Provider removes the variable. + already rewrites Agent configs (`SaveProvider` reports it in `reapplied`). A new + endpoint also re-renders the Provider's Skills, since `config.json` carries the + base URL. +- Uninstalling the last Skill that uses a Provider removes the variable. So does + removing a generated Skill's target on the Skills page, or restoring a backup. +- Deleting a Provider that a generated Skill still uses is refused, as deleting + one an Agent uses already is. +- A generated Skill reaching an Agent with no native writer (through the Skills + page) gets the fallback file; the file is deleted, not backed up, when no Agent + needs it. +- Only variants BootAgent rendered here earn a key. A `bootagent-*` tree imported + from a ZIP can name any base URL, and writing the real key for it would hand + the key to whoever wrote the ZIP. - Every write follows the existing order: private directory, backup, temp file in - the same directory, tighten permissions, atomic replace. + the same directory, tighten permissions, atomic replace. A writer with nothing + to write does not parse the file, so an unrelated problem in an Agent's config + does not fail a Skill change. Accepted risk: the key is visible to every command the Agent runs, so the Agent's model can read it. That is the exposure Claude Code's own token already has under @@ -343,15 +404,23 @@ The Skills page gets a Multimodal capabilities section: declared by the Provider, ID rule, catalog, or probed. 2. Per capability, a model choice and, where the probe table allows, a Probe button. -3. Target Agents: those with `skills_path` in `agents.lock.json`, which today are - Codex, Claude Code, OpenCode and Hermes. -4. Install: generate the tree, `ApplySkills` to the targets, write the variables, - and show the result per Agent. +3. Target Agents: those with `skills_path` in `agents.lock.json` and a native + writer for the variable. In Phase 2 that is Claude Code and Codex; OpenCode + and Hermes follow in Phase 4. +4. Install: a confirmation names the variable, states that the Agent and its + model can read it, and notes the Node 18 requirement. Then the tree is + generated, stored and published, the variables are written, and the result is + shown per Agent with a reminder that running Agents need a restart. + +Install re-checks the evidence on the backend rather than trusting the page: an +ID rule alone, or a declaration for an unverified chat adapter, is refused until +a probe passes. Install sends no billed request. After install, each Skill shows two or three example prompts, in Chinese and English, each with a copy button, for example "Look at ./screenshot.png and find -the error message in the UI" for image understanding. Uninstall reuses the existing Skill uninstall, -which already keeps a backup, and then cleans up the variables. +the error message in the UI" for image understanding. Uninstall reuses the +existing Skill uninstall (`SkillService.Uninstall`), which keeps a backup and now +also runs the reconcile, so no separate multimodal uninstall exists. ### Agents without Skills diff --git a/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/app/index.ts b/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/app/index.ts index 5f8225dd..f1594f45 100644 --- a/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/app/index.ts +++ b/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/app/index.ts @@ -17,6 +17,7 @@ export type { DesktopAgentStatus, DetectedConfig, InstallRuntimeResult, + InstalledMultimodal, MCPAgentApplyResult, MCPApplyRequest, MCPApplyResult, @@ -34,7 +35,11 @@ export type { MarketplaceRecommendationHistory, MarketplaceRecommendationSnapshot, MarketplaceSourceStatus, + MultimodalAgentResult, MultimodalDetection, + MultimodalInstallRequest, + MultimodalInstallResult, + MultimodalManualRoute, MultimodalProbe, MultimodalProvider, ProfileModelCheck, diff --git a/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/app/models.ts b/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/app/models.ts index a0a57717..d1274234 100644 --- a/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/app/models.ts +++ b/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/app/models.ts @@ -227,6 +227,19 @@ export interface InstallRuntimeResult { "runtimes": RuntimeStatus[] | null; } +/** + * InstalledMultimodal is one generated Skill variant that is published to at + * least one Agent. + */ +export interface InstalledMultimodal { + "skillId": string; + "capability": string; + "providerId": string; + "model": string; + "adapter": string; + "agents": string[] | null; +} + export interface MCPAgentApplyResult { "agent": string; "config_updated": boolean; @@ -349,12 +362,71 @@ export interface MarketplaceSourceStatus { "error"?: string; } +export interface MultimodalAgentResult { + "agent": string; + + /** + * Installed: the Skill tree is in the Agent's Skills directory. + */ + "installed": boolean; + + /** + * EnvWritten: the key variable is in the Agent's own configuration. + */ + "envWritten": boolean; + "error"?: string; +} + /** * MultimodalDetection lists, for every Provider with a saved key, the models * that offer a multimodal capability and where each finding came from. */ export interface MultimodalDetection { "providers": MultimodalProvider[] | null; + + /** + * Installed are the generated Skills published to at least one Agent. + */ + "installed": InstalledMultimodal[] | null; + + /** + * Targets are the Agents a multimodal Skill can be installed into now. + */ + "targets": string[] | null; + + /** + * Generatable lists the adapters a Skill can be generated for in this + * version; a capability on any other adapter is shown but not offered. + */ + "generatable": string[] | null; + + /** + * Manual lists the capabilities a user can reach with a model ID typed by + * hand, the route it is probed on, and how that probe is sent. + */ + "manual": MultimodalManualRoute[] | null; +} + +/** + * MultimodalInstallRequest installs the Skill for one capability, bound to one + * Provider and model, into the named Agents. + */ +export interface MultimodalInstallRequest { + "provider": string; + "model": string; + "capability": string; + "agents": string[] | null; +} + +export interface MultimodalInstallResult { + "skillId": string; + "results": MultimodalAgentResult[] | null; +} + +export interface MultimodalManualRoute { + "capability": string; + "adapter": string; + "probe": string; } /** @@ -365,6 +437,7 @@ export interface MultimodalProbe { "providerId": string; "model": string; "capability": string; + "adapter": string; "status": string; "reason"?: string; "httpStatus"?: number; @@ -375,6 +448,12 @@ export interface MultimodalProvider { "id": string; "name": string; + /** + * KeyEnv is the variable an install writes this Provider's key into, so + * the install confirmation can name it. + */ + "keyEnv": string; + /** * Listed is false when /models could not be read. Message and ErrorCode * then say why; catalog generation models are still listed, since they do diff --git a/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/binding/models.ts b/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/binding/models.ts index 172fff57..3fd4eee8 100644 --- a/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/binding/models.ts +++ b/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/binding/models.ts @@ -167,6 +167,18 @@ export interface MultimodalProbeRequest { "provider": string; "model": string; "capability": string; + + /** + * Adapter picks the route; empty is the capability's chat route. A model + * the user typed by hand is probed on its manual route. + */ + "adapter": string; + + /** + * Confirmed must be set for a probe that produces billed output (one + * image); the UI sets it only after asking the user. + */ + "confirmed": boolean; } /** diff --git a/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/binding/multimodalservice.ts b/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/binding/multimodalservice.ts index e895fdc8..0b3a19e3 100644 --- a/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/binding/multimodalservice.ts +++ b/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/binding/multimodalservice.ts @@ -2,9 +2,11 @@ // This file is automatically generated. DO NOT EDIT /** - * MultimodalService exposes multimodal capability detection. Detect is free -- - * it lists models -- so pages may call it on open. Probe is a billed request - * and may only be called from a button the user presses. + * MultimodalService exposes multimodal capability detection and install. + * Detect is free -- it lists models -- so pages may call it on open. Probe is a + * billed request and may only be called from a button the user presses. + * Uninstall goes through SkillService.Uninstall, which also removes the key + * variable once no Skill needs it. * @module */ @@ -24,6 +26,15 @@ export function Detect(): $CancellablePromise { return $Call.ByID(2944945473); } +/** + * Install generates the Skill and publishes it to the chosen Agents, writing + * the Provider key into each Agent's own configuration. It sends no billed + * request. + */ +export function Install(request: app$0.MultimodalInstallRequest): $CancellablePromise { + return $Call.ByID(1879579645, request); +} + export function Probe(request: $models.MultimodalProbeRequest): $CancellablePromise { return $Call.ByID(3316532872, request); } diff --git a/frontend/src/backend/wails.test.ts b/frontend/src/backend/wails.test.ts index d816a448..9d3767d1 100644 --- a/frontend/src/backend/wails.test.ts +++ b/frontend/src/backend/wails.test.ts @@ -245,7 +245,7 @@ describe("multimodal bridge", () => { const { wailsApi } = await import("./wails"); bridge.probeMultimodal.mockResolvedValue({ providerId: "novita", model: "m", capability: "vision", status: "available" }); await wailsApi.probeMultimodal("novita", "m", "vision"); - expect(bridge.probeMultimodal).toHaveBeenCalledWith({ provider: "novita", model: "m", capability: "vision" }); + expect(bridge.probeMultimodal).toHaveBeenCalledWith({ provider: "novita", model: "m", capability: "vision", adapter: "", confirmed: false }); bridge.probeMultimodal.mockResolvedValue(null); await expect(wailsApi.probeMultimodal("novita", "m", "vision")).rejects.toThrow(); }); diff --git a/frontend/src/backend/wails.ts b/frontend/src/backend/wails.ts index de0dc476..edce6903 100644 --- a/frontend/src/backend/wails.ts +++ b/frontend/src/backend/wails.ts @@ -40,6 +40,8 @@ import type { ModelsResponse, MultimodalCapabilityId, MultimodalDetection, + MultimodalInstallRequest, + MultimodalInstallResult, MultimodalProbe, OpenRegistrationResponse, ProbeResponse, @@ -358,11 +360,21 @@ export const wailsApi = { ...entry, models: (entry.models ?? []).map((model) => ({ ...model, capabilities: model.capabilities ?? [] })), })), + installed: (detection?.installed ?? []).map((entry) => ({ ...entry, agents: entry.agents ?? [] })), + targets: detection?.targets ?? [], + generatable: detection?.generatable ?? [], + manual: detection?.manual ?? [], })) as Promise, // Sends a live (billed) request; only called from a button the user presses. - probeMultimodal: (provider: string, model: string, capability: MultimodalCapabilityId): Promise => - call(() => MultimodalService.Probe({ provider, model, capability })).then((check) => { + // `confirmed` is set only after the user agreed to a probe that produces + // billed output (one image). `adapter` picks the route for a hand-entered + // model; empty is the capability's chat route. + probeMultimodal: (provider: string, model: string, capability: MultimodalCapabilityId, adapter = "", confirmed = false): Promise => + call(() => MultimodalService.Probe({ provider, model, capability, adapter, confirmed })).then((check) => { if (!check) throw new Error("Multimodal probe returned no result"); return check; }) as Promise, + // Sends no billed request: it installs on evidence already gathered. + installMultimodal: (request: MultimodalInstallRequest): Promise => + call(() => MultimodalService.Install(request)).then((result) => ({ ...result, results: result?.results ?? [] })) as Promise, }; diff --git a/frontend/src/components/MultimodalSection.test.tsx b/frontend/src/components/MultimodalSection.test.tsx index 1abb5cf4..fc725de5 100644 --- a/frontend/src/components/MultimodalSection.test.tsx +++ b/frontend/src/components/MultimodalSection.test.tsx @@ -1,4 +1,4 @@ -import { fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { fireEvent, render, screen, waitFor, within } from "@testing-library/react"; import { MemoryRouter } from "react-router-dom"; import { beforeEach, describe, expect, it, vi } from "vitest"; @@ -6,22 +6,32 @@ import type { MultimodalDetection, MultimodalProvider } from "../types/api"; const detectMultimodal = vi.fn<() => Promise>(); const probeMultimodal = vi.fn(); +const installMultimodal = vi.fn(); +const uninstallSkill = vi.fn(); +const copyToClipboard = vi.fn((_text: string) => Promise.resolve(true)); vi.mock("@wailsio/runtime", () => ({ Events: { On: vi.fn(), Off: vi.fn() } })); vi.mock("../backend/api", () => ({ api: { detectMultimodal: () => detectMultimodal(), probeMultimodal: (...args: unknown[]) => probeMultimodal(...args), + installMultimodal: (...args: unknown[]) => installMultimodal(...args), + uninstallSkill: (...args: unknown[]) => uninstallSkill(...args), }, describeError: (error: unknown, fallback: string) => ({ message: error instanceof Error ? error.message : fallback }), })); +vi.mock("../utils/clipboard", () => ({ copyToClipboard: (text: string) => copyToClipboard(text) })); import { I18nProvider, sourceTranslate } from "../i18n"; -import { describeProbe, groupByCapability, installLabel, MultimodalSection } from "./MultimodalSection"; +import { describeProbe, groupByCapability, groupInstalled, installLabel, MultimodalSection } from "./MultimodalSection"; +import { EXAMPLE_PROMPTS } from "./multimodalPrompts"; + +const GENERATABLE = ["openai-chat-vision", "openai-images", "openai-audio-transcriptions", "openai-audio-speech"]; const gateway: MultimodalProvider = { id: "novita", name: "Novita", + keyEnv: "BOOTAGENT_NOVITA_API_KEY", listed: true, models: [ { @@ -38,13 +48,30 @@ const gateway: MultimodalProvider = { ], }; -function renderSection() { - return render(); +function detection(partial: Partial = {}): MultimodalDetection { + return { + providers: [gateway], + installed: [], + targets: ["claude-code", "codex"], + generatable: GENERATABLE, + manual: [ + { capability: "image-generation", adapter: "openai-images", probe: "confirm" }, + { capability: "speech-to-text", adapter: "openai-audio-transcriptions", probe: "minimal" }, + ], + ...partial, + }; +} + +const labels = { "claude-code": "Claude Code", codex: "Codex" }; + +function renderSection(onSkillsChanged = vi.fn()) { + render(); + return onSkillsChanged; } beforeEach(() => { + for (const mock of [detectMultimodal, probeMultimodal, installMultimodal, uninstallSkill, copyToClipboard]) mock.mockClear(); detectMultimodal.mockReset(); - probeMultimodal.mockReset(); localStorage.setItem("bootagent.locale", "zh-CN"); }); @@ -57,71 +84,168 @@ describe("groupByCapability", () => { }); describe("installLabel", () => { - it("never calls a guess installable, and says a probe would settle it", () => { - expect(installLabel(gateway.models[0].capabilities[0], sourceTranslate).text).toBe("实测通过后可安装"); - expect(installLabel(gateway.models[0].capabilities[1], sourceTranslate).text).toBe("暂不可安装"); - expect(installLabel({ ...gateway.models[0].capabilities[0], source: "probe", supported: false }, sourceTranslate).text).toBe("不可安装"); + it("never calls a guess installable, and says what would settle it", () => { + const [guess, video] = gateway.models[0].capabilities; + expect(installLabel(guess, GENERATABLE, sourceTranslate)).toMatchObject({ text: "实测通过后可安装", ready: false }); + expect(installLabel(video, GENERATABLE, sourceTranslate)).toMatchObject({ text: "暂不可安装", ready: false }); + expect(installLabel({ ...guess, source: "probe", supported: false }, GENERATABLE, sourceTranslate).text).toBe("不可安装"); + expect(installLabel(gateway.models[1].capabilities[0], GENERATABLE, sourceTranslate)).toMatchObject({ text: "可安装", ready: true }); + // Confirmed, but on a route this version cannot write a Skill for. + expect(installLabel({ ...video, installable: true, source: "declared" }, GENERATABLE, sourceTranslate)).toMatchObject({ text: "暂不支持安装", ready: false }); }); }); describe("describeProbe", () => { it("words an unreachable provider as unverified, not as unsupported", () => { - const parts = describeProbe({ providerId: "novita", model: "m", capability: "vision", status: "unverified", reason: "network" }, sourceTranslate); + const parts = describeProbe({ providerId: "novita", model: "m", capability: "vision", adapter: "openai-chat-vision", status: "unverified", reason: "network" }, sourceTranslate); expect(parts.ok).toBe(false); expect(parts.title).toBe("无法验证「m」的图片理解"); expect(parts.reason).toContain("模型本身可能没有问题"); }); it("names a refusal with its status and the provider's words", () => { - const parts = describeProbe({ providerId: "novita", model: "m", capability: "vision", status: "unavailable", reason: "refused", httpStatus: 400, detail: "no images" }, sourceTranslate); + const parts = describeProbe({ providerId: "novita", model: "m", capability: "vision", adapter: "openai-chat-vision", status: "unavailable", reason: "refused", httpStatus: 400, detail: "no images" }, sourceTranslate); expect(parts.title).toBe("「m」不支持图片理解"); expect(parts.reason).toContain("HTTP 400"); expect(parts.detail).toContain("no images"); }); }); +describe("groupInstalled", () => { + it("makes one card per Skill and unions its Agents", () => { + const cards = groupInstalled([ + { skillId: "bootagent-vision", capability: "vision", providerId: "novita", model: "a", adapter: "openai-chat-vision", agents: ["codex"] }, + { skillId: "bootagent-vision", capability: "vision", providerId: "ppio", model: "b", adapter: "openai-chat-vision", agents: ["claude-code"] }, + ]); + expect(cards).toHaveLength(1); + expect(cards[0].agents).toEqual(["claude-code", "codex"]); + expect(cards[0].variants).toHaveLength(2); + }); +}); + +describe("example prompts", () => { + it("has bilingual prompts for every capability that can be installed", () => { + for (const capability of ["vision", "image-generation", "speech-to-text", "text-to-speech"] as const) { + const prompts = EXAMPLE_PROMPTS[capability] ?? []; + expect(prompts.length).toBeGreaterThanOrEqual(2); + for (const prompt of prompts) { + expect(prompt.zh).toMatch(/[一-鿿]/); + expect(prompt.en).not.toMatch(/[一-鿿]/); + } + } + }); +}); + describe("MultimodalSection", () => { - it("lists on open and sends no probe until the button is pressed", async () => { - detectMultimodal.mockResolvedValue({ providers: [gateway] }); + it("lists on open and sends no probe until a button is pressed", async () => { + detectMultimodal.mockResolvedValue(detection()); renderSection(); expect(await screen.findByText("Novita")).toBeTruthy(); expect(detectMultimodal).toHaveBeenCalledTimes(1); expect(probeMultimodal).not.toHaveBeenCalled(); - // The probe button warns of cost before it is pressed. - const button = screen.getAllByRole("button", { name: /实测/ })[0]; + expect(installMultimodal).not.toHaveBeenCalled(); + const button = screen.getAllByRole("button", { name: /^实测$/ })[0]; expect(button.getAttribute("title")).toContain("可能产生少量费用"); }); it("probes the selected model and shows the verdict", async () => { - detectMultimodal.mockResolvedValue({ providers: [gateway] }); - probeMultimodal.mockResolvedValue({ providerId: "novita", model: "moonshotai/kimi-k3", capability: "vision", status: "available" }); + detectMultimodal.mockResolvedValue(detection()); + probeMultimodal.mockResolvedValue({ providerId: "novita", model: "moonshotai/kimi-k3", capability: "vision", adapter: "openai-chat-vision", status: "available" }); renderSection(); - const select = await screen.findByLabelText("图片理解"); - fireEvent.change(select, { target: { value: "moonshotai/kimi-k3" } }); + fireEvent.change(await screen.findByLabelText("图片理解"), { target: { value: "moonshotai/kimi-k3" } }); expect(screen.getByText("实测通过后可安装")).toBeTruthy(); - fireEvent.click(screen.getAllByRole("button", { name: /实测/ })[0]); - await waitFor(() => expect(probeMultimodal).toHaveBeenCalledWith("novita", "moonshotai/kimi-k3", "vision")); + // No install button while the evidence is only a guess. + expect(screen.queryByRole("button", { name: /^安装$/ })).toBeNull(); + fireEvent.click(screen.getAllByRole("button", { name: /^实测$/ })[0]); + await waitFor(() => expect(probeMultimodal).toHaveBeenCalledWith("novita", "moonshotai/kimi-k3", "vision", "openai-chat-vision", false)); expect(await screen.findByText("「moonshotai/kimi-k3」支持图片理解")).toBeTruthy(); - // The cached verdict is re-read rather than patched in locally. await waitFor(() => expect(detectMultimodal).toHaveBeenCalledTimes(2)); }); + it("asks before an image probe and sends nothing when the user declines", async () => { + detectMultimodal.mockResolvedValue(detection()); + probeMultimodal.mockResolvedValue({ providerId: "novita", model: "gpt-image-1", capability: "image-generation", adapter: "openai-images", status: "available" }); + renderSection(); + await screen.findByText("Novita"); + fireEvent.change(screen.getByLabelText("能力"), { target: { value: "image-generation" } }); + fireEvent.change(screen.getByLabelText("模型 ID"), { target: { value: "gpt-image-1" } }); + const manualProbe = within(screen.getByText("手动填写模型 ID").closest("details") as HTMLElement).getByRole("button", { name: /^实测$/ }); + expect(manualProbe.getAttribute("title")).toContain("发送前会先确认"); + fireEvent.click(manualProbe); + const confirm = screen.getByRole("dialog"); + expect(within(confirm).getByText(/真实生成一张图片/)).toBeTruthy(); + fireEvent.click(within(confirm.querySelector("footer") as HTMLElement).getByRole("button", { name: "取消" })); + expect(probeMultimodal).not.toHaveBeenCalled(); + fireEvent.click(manualProbe); + fireEvent.click(screen.getByRole("button", { name: "发送实测请求" })); + await waitFor(() => expect(probeMultimodal).toHaveBeenCalledWith("novita", "gpt-image-1", "image-generation", "openai-images", true)); + }); + + it("installs after stating the key exposure, then shows results and prompts", async () => { + detectMultimodal.mockResolvedValue(detection()); + installMultimodal.mockResolvedValue({ skillId: "bootagent-vision", results: [ + { agent: "claude-code", installed: true, envWritten: true }, + { agent: "codex", installed: false, envWritten: false, error: "A Skill with this name that BootAgent did not write is installed here; it was left alone" }, + ] }); + const changed = renderSection(); + fireEvent.click(await screen.findByRole("button", { name: /^安装$/ })); + const dialog = screen.getByRole("dialog"); + expect(within(dialog).getByText(/BOOTAGENT_NOVITA_API_KEY/)).toBeTruthy(); + expect(within(dialog).getByText("API Key 对 Agent 可见")).toBeTruthy(); + // Both Agents are preselected; deselecting one narrows the request. + fireEvent.click(within(dialog).getByLabelText("Codex")); + fireEvent.click(within(dialog).getByLabelText("Codex")); + fireEvent.click(within(dialog).getByRole("button", { name: "安装" })); + await waitFor(() => expect(installMultimodal).toHaveBeenCalledWith({ provider: "novita", model: "qwen/qwen3-vl-8b", capability: "vision", agents: ["claude-code", "codex"] })); + expect(await within(dialog).findByText("已安装")).toBeTruthy(); + expect(within(dialog).getByText(/did not write is installed here/)).toBeTruthy(); + expect(within(dialog).getByText(/需要重启/)).toBeTruthy(); + const copy = within(dialog).getAllByRole("button", { name: "复制提示词" }); + expect(copy.length).toBeGreaterThanOrEqual(4); + fireEvent.click(copy[0]); + expect(copyToClipboard).toHaveBeenCalledWith(EXAMPLE_PROMPTS.vision![0].zh); + fireEvent.click(within(dialog).getByRole("button", { name: "完成" })); + expect(changed).toHaveBeenCalled(); + expect(installMultimodal).toHaveBeenCalledTimes(1); + }); + + it("shows installed Skills and uninstalls through the Skill library", async () => { + detectMultimodal.mockResolvedValue(detection({ installed: [ + { skillId: "bootagent-vision", capability: "vision", providerId: "novita", model: "qwen/qwen3-vl-8b", adapter: "openai-chat-vision", agents: ["claude-code", "codex"] }, + ] })); + uninstallSkill.mockResolvedValue({ registry_updated: true }); + const changed = renderSection(); + expect(await screen.findByText("qwen/qwen3-vl-8b（Novita）已安装到 Claude Code、Codex")).toBeTruthy(); + expect(screen.getByText("已安装到 Claude Code、Codex")).toBeTruthy(); + expect(screen.getByRole("button", { name: "重新安装" })).toBeTruthy(); + fireEvent.click(screen.getByRole("button", { name: "卸载 bootagent-vision" })); + fireEvent.click(within(screen.getByRole("dialog")).getByRole("button", { name: "卸载" })); + await waitFor(() => expect(uninstallSkill).toHaveBeenCalledWith("bootagent-vision")); + await waitFor(() => expect(changed).toHaveBeenCalled()); + }); + + it("offers no install when no Agent can take the Skill", async () => { + detectMultimodal.mockResolvedValue(detection({ targets: [] })); + renderSection(); + const button = await screen.findByRole("button", { name: /^安装$/ }); + expect((button as HTMLButtonElement).disabled).toBe(true); + }); + it("offers no probe for a capability with no live check", async () => { - detectMultimodal.mockResolvedValue({ providers: [{ ...gateway, models: [gateway.models[0]] }] }); + detectMultimodal.mockResolvedValue(detection({ providers: [{ ...gateway, models: [gateway.models[0]] }], manual: [] })); renderSection(); await screen.findByText("Novita"); - // One button for vision; none for video understanding. expect(screen.getAllByRole("button", { name: /^实测$/ })).toHaveLength(1); }); it("points to the Providers page when no key is saved", async () => { - detectMultimodal.mockResolvedValue({ providers: [] }); + detectMultimodal.mockResolvedValue(detection({ providers: [] })); renderSection(); expect(await screen.findByText("去添加模型服务")).toBeTruthy(); }); it("reports a failed listing without hiding the Provider", async () => { - detectMultimodal.mockResolvedValue({ providers: [{ id: "deepseek", name: "DeepSeek", listed: false, message: "API key was rejected (401).", errorCode: "API_KEY_REJECTED", models: [] }] }); + detectMultimodal.mockResolvedValue(detection({ providers: [{ id: "deepseek", name: "DeepSeek", keyEnv: "BOOTAGENT_DEEPSEEK_API_KEY", listed: false, message: "API key was rejected (401).", errorCode: "API_KEY_REJECTED", models: [] }] })); renderSection(); expect(await screen.findByText(/无法读取模型列表/)).toBeTruthy(); }); diff --git a/frontend/src/components/MultimodalSection.tsx b/frontend/src/components/MultimodalSection.tsx index c67eda18..2f2dea70 100644 --- a/frontend/src/components/MultimodalSection.tsx +++ b/frontend/src/components/MultimodalSection.tsx @@ -1,4 +1,4 @@ -import { Activity, Images, RefreshCw } from "lucide-react"; +import { Activity, Check, Copy, Download, Images, RefreshCw, Trash2, X } from "lucide-react"; import { useCallback, useEffect, useMemo, useState } from "react"; import { Link } from "react-router-dom"; @@ -6,13 +6,19 @@ import { api, describeError } from "../backend/api"; import type { Translate, TranslationKey } from "../i18n"; import { useI18n } from "../i18n"; import type { + InstalledMultimodal, MultimodalCapability, MultimodalCapabilityId, MultimodalDetection, + MultimodalInstallResult, MultimodalProbe, + MultimodalProbeKind, MultimodalProvider, MultimodalSource, } from "../types/api"; +import { copyToClipboard } from "../utils/clipboard"; +import { ModalDialog } from "./ModalDialog"; +import { EXAMPLE_PROMPTS } from "./multimodalPrompts"; const CAPABILITY_ORDER: MultimodalCapabilityId[] = [ "vision", @@ -23,7 +29,7 @@ const CAPABILITY_ORDER: MultimodalCapabilityId[] = [ "video-understanding", ]; -const CAPABILITY_LABELS: Record = { +export const CAPABILITY_LABELS: Record = { vision: "图片理解", "image-generation": "图片生成", "speech-to-text": "语音识别", @@ -84,12 +90,17 @@ export function sourceLabel(capability: MultimodalCapability, t: Translate): str } } -/** Why a capability can or cannot be installed yet, in the user's terms. */ -export function installLabel(capability: MultimodalCapability, t: Translate): { text: string; tone: "success" | "warning" | "neutral" } { - if (capability.installable) return { text: t("可安装"), tone: "success" }; - if (capability.source === "probe" && !capability.supported) return { text: t("不可安装"), tone: "warning" }; - if (capability.probe) return { text: t("实测通过后可安装"), tone: "neutral" }; - return { text: t("暂不可安装"), tone: "neutral" }; +/** + * Whether a capability can be installed now, and if not, why not, in the + * user's terms. `generatable` is the backend's list of routes this version can + * write a Skill for. + */ +export function installLabel(capability: MultimodalCapability, generatable: string[], t: Translate): { text: string; tone: "success" | "warning" | "neutral"; ready: boolean } { + if (capability.source === "probe" && !capability.supported) return { text: t("不可安装"), tone: "warning", ready: false }; + if (capability.installable && generatable.includes(capability.adapter)) return { text: t("可安装"), tone: "success", ready: true }; + if (capability.installable) return { text: t("暂不支持安装"), tone: "neutral", ready: false }; + if (capability.probe) return { text: t("实测通过后可安装"), tone: "neutral", ready: false }; + return { text: t("暂不可安装"), tone: "neutral", ready: false }; } /** @@ -136,21 +147,47 @@ export function describeProbe(check: MultimodalProbe, t: Translate): { ok: boole return { ok: false, title, reason, detail: check.detail ? t("模型服务返回：{detail}", { detail: check.detail }) : "" }; } +/** One card per installed Skill: a capability may be on several models across Agents. */ +export function groupInstalled(installed: InstalledMultimodal[]): Array<{ skillId: string; capability: MultimodalCapabilityId; variants: InstalledMultimodal[]; agents: string[] }> { + const groups = new Map(); + for (const entry of installed) groups.set(entry.skillId, [...(groups.get(entry.skillId) ?? []), entry]); + return CAPABILITY_ORDER.flatMap((capability) => [...groups.entries()] + .filter(([, variants]) => variants[0].capability === capability) + .map(([skillId, variants]) => ({ skillId, capability, variants, agents: [...new Set(variants.flatMap((variant) => variant.agents))].sort() }))); +} + type ProbeState = { checking: boolean; result?: MultimodalProbe; error?: string }; +type ProbeTarget = { provider: MultimodalProvider; capability: MultimodalCapabilityId; model: string; adapter: string; kind: MultimodalProbeKind }; +type InstallTarget = { provider: MultimodalProvider; capability: MultimodalCapabilityId; model: string }; -const probeKey = (provider: string, capability: string) => `${provider}::${capability}`; +const rowKey = (provider: string, capability: string) => `${provider}::${capability}`; + +interface MultimodalSectionProps { + /** Agent ID to display name. */ + agentLabels?: Record; + /** Called after an install or uninstall changed the Skill library. */ + onSkillsChanged?: () => void; +} /** * The Skills page's Multimodal capabilities section. Opening it lists models, - * which is free; nothing billed is sent until the user presses a check button. + * which is free; nothing billed is sent until the user presses a check button, + * and a check that produces billed output asks first. */ -export function MultimodalSection() { - const { t } = useI18n(); +export function MultimodalSection({ agentLabels = {}, onSkillsChanged }: MultimodalSectionProps) { + const { t, locale } = useI18n(); const [detection, setDetection] = useState(null); const [loading, setLoading] = useState(false); const [error, setError] = useState(""); const [selected, setSelected] = useState>({}); const [probes, setProbes] = useState>({}); + const [confirmProbe, setConfirmProbe] = useState(null); + const [installTarget, setInstallTarget] = useState(null); + const [uninstallTarget, setUninstallTarget] = useState<{ skillId: string; capability: MultimodalCapabilityId; agents: string[] } | null>(null); + const [uninstalling, setUninstalling] = useState(false); + const [manual, setManual] = useState>({}); + const nameOf = (agentId: string) => agentLabels[agentId] ?? agentId; + const separator = locale === "en" ? ", " : "、"; const detect = useCallback(() => { setLoading(true); @@ -163,23 +200,42 @@ export function MultimodalSection() { useEffect(() => { void detect(); }, [detect]); - const probe = (providerId: string, capability: MultimodalCapabilityId, model: string) => { - const key = probeKey(providerId, capability); + const runProbe = (target: ProbeTarget, confirmed: boolean) => { + const key = rowKey(target.provider.id, target.capability); setProbes((current) => ({ ...current, [key]: { checking: true } })); - void api.probeMultimodal(providerId, model, capability) + void api.probeMultimodal(target.provider.id, target.model, target.capability, target.adapter, confirmed) .then((result) => { setProbes((current) => ({ ...current, [key]: { checking: false, result } })); - // A conclusive verdict is cached by the backend and outranks every - // other source, so the list is re-read to show it. + // Show the probed model in its row; a conclusive verdict is cached by + // the backend and outranks every other source once re-read. + setSelected((current) => ({ ...current, [key]: target.model })); return detect(); }) .catch((reason: unknown) => { setProbes((current) => ({ ...current, [key]: { checking: false, error: describeError(reason, t("无法向模型服务发送验证请求。")).message } })); }); }; + const probe = (target: ProbeTarget) => (target.kind === "confirm" ? setConfirmProbe(target) : runProbe(target, false)); + + const uninstall = () => { + if (!uninstallTarget) return; + setUninstalling(true); + void api.uninstallSkill(uninstallTarget.skillId) + .then((result) => { + if (result.error) setError(result.error); + setUninstallTarget(null); + onSkillsChanged?.(); + return detect(); + }) + .catch((reason: unknown) => setError(describeError(reason, t("卸载失败")).message)) + .finally(() => setUninstalling(false)); + }; const providers = useMemo(() => detection?.providers ?? [], [detection]); const groupsByProvider = useMemo(() => new Map(providers.map((entry) => [entry.id, groupByCapability(entry)])), [providers]); + const providerName = (id: string) => providers.find((entry) => entry.id === id)?.name || id; + const generatable = detection?.generatable ?? []; + const installedCards = useMemo(() => groupInstalled(detection?.installed ?? []), [detection]); return (
@@ -194,6 +250,30 @@ export function MultimodalSection() { {error ?

{error}

: null} + + {installedCards.length ? ( +
+

{t("已安装的多模态 Skill")}

+ {installedCards.map((card) => ( +
+
+ {t(CAPABILITY_LABELS[card.capability])} + {card.skillId} + +
+ {card.variants.map((variant) => ( +

+ {t("{model}（{provider}）已安装到 {agents}", { model: variant.model, provider: providerName(variant.providerId), agents: variant.agents.map(nameOf).join(separator) })} +

+ ))} + +
+ ))} +
+ ) : null} + {detection && !providers.length ? (

{t("还没有保存 API Key 的模型服务。")} {t("去添加模型服务")} @@ -201,6 +281,9 @@ export function MultimodalSection() { ) : null} {providers.map((entry) => { const groups = groupsByProvider.get(entry.id) ?? []; + const manualState = manual[entry.id]; + const manualRoutes = detection?.manual ?? []; + const manualRoute = manualRoutes.find((route) => route.capability === (manualState?.capability ?? manualRoutes[0]?.capability)); return (

@@ -212,56 +295,248 @@ export function MultimodalSection() { ) : (
    {groups.map((group) => { - const key = probeKey(entry.id, group.id); + const key = rowKey(entry.id, group.id); const chosen = group.options.find((option) => option.model === selected[key]) ?? group.options[0]; - const install = installLabel(chosen.capability, t); + const install = installLabel(chosen.capability, generatable, t); + const installedHere = (detection?.installed ?? []).find((item) => item.providerId === entry.id && item.capability === group.id && item.model === chosen.model); const state = probes[key]; const verdict = state?.result && state.result.model === chosen.model ? describeProbe(state.result, t) : null; const selectID = `multimodal-${entry.id}-${group.id}`; + const capabilityName = t(CAPABILITY_LABELS[group.id]); return (
  • - - setSelected((current) => ({ ...current, [key]: event.target.value }))}> + {group.options.map((option) => )} {sourceLabel(chosen.capability, t)} - {install.text} - {chosen.capability.probe === "minimal" ? ( - + {installedHere + ? {t("已安装到 {agents}", { agents: installedHere.agents.map(nameOf).join(separator) })} + : {install.text}} + {chosen.capability.probe ? ( + probe({ provider: entry, capability: group.id, model: chosen.model, adapter: chosen.capability.adapter, kind: chosen.capability.probe })} + /> ) : null} - {verdict ? ( -
    -
    - {verdict.title} - {verdict.reason} - {verdict.detail ? {verdict.detail} : null} -
    -
    + {install.ready ? ( + ) : null} + {verdict ? : null} {state?.error ?

    {state.error}

    : null}
    • ); })}
)} + {manualRoutes.length ? ( +
+ {t("手动填写模型 ID")} +

{t("模型列表里没有专用的图片生成、语音识别或语音合成模型。如果你知道模型 ID（例如 gpt-image-1、whisper-1、tts-1），可以填写后实测，通过后即可安装。")}

+
+ + setManual((current) => ({ ...current, [entry.id]: { capability: manualRoute?.capability ?? "vision", model: event.target.value } }))} /> + {manualRoute ? ( + probe({ provider: entry, capability: manualRoute.capability, model: (manualState?.model ?? "").trim(), adapter: manualRoute.adapter, kind: manualRoute.probe })} + /> + ) : null} +
+ {manualRoute && probes[rowKey(entry.id, manualRoute.capability)]?.result && !groups.some((group) => group.id === manualRoute.capability) + ? + : null} +
+ ) : null}
); })} + + {confirmProbe ? ( + setConfirmProbe(null)}> +

{t("确认实测")}

+

{t("这次实测会让「{model}」真实生成一张图片，按模型服务的价格计费。确定要发送吗？", { model: confirmProbe.model })}

+
+ + +
+ + ) : null} + + {installTarget && detection ? ( + item.capability === installTarget.capability)} + providerName={providerName} + nameOf={nameOf} + onDone={() => { setInstallTarget(null); onSkillsChanged?.(); void detect(); }} + onCancel={() => setInstallTarget(null)} + /> + ) : null} + + {uninstallTarget ? ( + setUninstallTarget(null)}> +

{t("卸载 {name}", { name: uninstallTarget.skillId })}

+

{t("将从 {agents} 移除这个 Skill。BootAgent 会保留一份备份；没有其他 Skill 再用到的 API Key 环境变量也会一并移除。", { agents: uninstallTarget.agents.map(nameOf).join(separator) })}

+
+ + +
+ + ) : null}
); } + +function ProbeButton({ checking, disabled, kind, capability, onClick }: { checking?: boolean; disabled?: boolean; kind: MultimodalProbeKind; capability: string; onClick: () => void }) { + const { t } = useI18n(); + const title = kind === "confirm" + ? t("向模型服务发送一次真实的生成请求，确认这个模型能做{capability}。会产生一次生成的费用，发送前会先确认。", { capability }) + : t("向模型服务发送一次最小的真实请求，确认这个模型能做{capability}。可能产生少量费用。", { capability }); + return ( + + ); +} + +function Verdict({ verdict }: { verdict: ReturnType }) { + return ( +
+
+ {verdict.title} + {verdict.reason} + {verdict.detail ? {verdict.detail} : null} +
+
+ ); +} + +function ExamplePrompts({ capability }: { capability: MultimodalCapabilityId }) { + const { t } = useI18n(); + const [copied, setCopied] = useState(""); + const prompts = EXAMPLE_PROMPTS[capability] ?? []; + if (!prompts.length) return null; + const copy = (text: string) => { + void copyToClipboard(text).then((ok) => { + if (!ok) return; + setCopied(text); + window.setTimeout(() => setCopied((current) => (current === text ? "" : current)), 1500); + }); + }; + return ( +
+ {t("在 Agent 里试试")} +
    + {prompts.flatMap((prompt) => [ + { text: prompt.zh, lang: "zh-CN" }, + { text: prompt.en, lang: "en" }, + ]).map((item) => ( +
  • + {item.text} + +
    • + ))} +
+
+ ); +} + +interface InstallDialogProps { + target: InstallTarget; + targets: string[]; + /** What is installed for this capability now, from any Provider. */ + current: InstalledMultimodal[]; + providerName: (id: string) => string; + nameOf: (agentId: string) => string; + onDone: () => void; + onCancel: () => void; +} + +function InstallDialog({ target, targets, current, providerName, nameOf, onDone, onCancel }: InstallDialogProps) { + const { t, locale } = useI18n(); + const [agents, setAgents] = useState(targets); + const [busy, setBusy] = useState(false); + const [result, setResult] = useState(null); + const [error, setError] = useState(""); + const capability = t(CAPABILITY_LABELS[target.capability]); + const separator = locale === "en" ? ", " : "、"; + const replacing = current.filter((item) => (item.providerId !== target.provider.id || item.model !== target.model) && item.agents.some((agent) => agents.includes(agent))); + + const install = () => { + setBusy(true); + setError(""); + void api.installMultimodal({ provider: target.provider.id, model: target.model, capability: target.capability, agents }) + .then(setResult) + .catch((reason: unknown) => setError(describeError(reason, t("安装失败")).message)) + .finally(() => setBusy(false)); + }; + + const title = t("安装{capability} Skill", { capability }); + return ( + +

{title}

+ {result ? ( + <> +
    + {result.results.map((item) => ( +
  • + {nameOf(item.agent)} + {item.installed && item.envWritten ? t("已安装") : item.error || t("安装失败")} +
    • + ))} +
+ {result.results.some((item) => item.installed) ? ( + <> +

{t("正在运行的 Agent 需要重启，才能读到新写入的环境变量。")}

+ + + ) : null} +
+ + ) : ( + <> +

{t("用「{model}」（{provider}）为 Agent 提供{capability}。", { model: target.model, provider: providerName(target.provider.id), capability })}

+
+ {t("安装到")} + {targets.map((agent) => ( + + ))} +
+ {replacing.length ? ( +

{t("将替换这些 Agent 当前使用的 {models}。", { models: replacing.map((item) => `${item.model}（${providerName(item.providerId)}）`).join(separator) })}

+ ) : null} +
+
+ {t("API Key 对 Agent 可见")} + {t("安装会把 {provider} 的 API Key 写进所选 Agent 自己的配置文件，变量名是 {name}。Agent 运行的任何命令都能读到它，Agent 使用的模型也可能看到。", { provider: providerName(target.provider.id), name: target.provider.keyEnv })} + {t("Skill 的脚本需要 Node.js 18 或更高版本。之后每次调用都由 Agent 发起，按模型服务的价格计费；生成类 Skill 会要求 Agent 先征得你的同意。")} +
+
+ {error ?

{error}

: null} +
+ + +
+ + )} + + ); +} diff --git a/frontend/src/components/multimodalPrompts.ts b/frontend/src/components/multimodalPrompts.ts new file mode 100644 index 00000000..bddddefa --- /dev/null +++ b/frontend/src/components/multimodalPrompts.ts @@ -0,0 +1,32 @@ +import type { MultimodalCapabilityId } from "../types/api"; + +/** + * Prompts the user copies into an Agent to try a newly installed Skill. Both + * languages are shown side by side whatever the UI locale, because the Agent + * may be used in either; they are content, not UI copy, so they do not go + * through translate(). + */ +export interface ExamplePrompt { + zh: string; + en: string; +} + +export const EXAMPLE_PROMPTS: Partial> = { + vision: [ + { zh: "看一下 ./screenshot.png，找出界面上的报错信息并告诉我原因。", en: "Look at ./screenshot.png and find the error message in the UI, then tell me what caused it." }, + { zh: "对比 ./before.png 和 ./after.png，列出界面上有哪些变化。", en: "Compare ./before.png and ./after.png and list what changed in the UI." }, + { zh: "读一下 ./diagram.png 里的架构图，用文字描述各个组件之间的关系。", en: "Read the architecture diagram in ./diagram.png and describe how the components relate." }, + ], + "image-generation": [ + { zh: "生成一张极简风格的应用图标草图：一只纸飞机，蓝白配色。生成前先告诉我会花费多少张。", en: "Generate a minimalist app icon draft: a paper plane in blue and white. Tell me how many images it will make before you run it." }, + { zh: "为 README 生成一张横幅插图，主题是“本地优先的开发工具”，先和我确认提示词再生成。", en: "Generate a banner illustration for the README on the theme \"local-first developer tools\". Confirm the prompt with me first." }, + ], + "speech-to-text": [ + { zh: "把 ./meeting.m4a 转成文字，然后整理出待办事项列表。", en: "Transcribe ./meeting.m4a, then turn it into a list of action items." }, + { zh: "转写 ./voice-note.mp3，录音是中文，保留原话不要改写。", en: "Transcribe ./voice-note.mp3. It is in Chinese; keep the wording as spoken." }, + ], + "text-to-speech": [ + { zh: "把这次改动的总结读成一段 mp3 语音，保存到 ./summary.mp3。先给我看要朗读的文字。", en: "Turn a summary of this change into an mp3 and save it as ./summary.mp3. Show me the text first." }, + { zh: "用语音合成把 README 的第一段生成为音频，生成前先确认。", en: "Make an audio file of the first paragraph of the README with text-to-speech. Confirm before you run it." }, + ], +}; diff --git a/frontend/src/i18n.tsx b/frontend/src/i18n.tsx index 7fc88010..2b03f0b2 100644 --- a/frontend/src/i18n.tsx +++ b/frontend/src/i18n.tsx @@ -945,6 +945,35 @@ const english = { "向模型服务发送一次带小图片的真实请求，确认这个模型能做{capability}。可能产生少量费用。": "Sends one real request with a small image to confirm this model can do {capability}. This may incur a small charge.", "正在实测": "Probing", "实测": "Probe", + "API Key 对 Agent 可见": "The API key is visible to the Agent", + "Skill 的脚本需要 Node.js 18 或更高版本。之后每次调用都由 Agent 发起，按模型服务的价格计费；生成类 Skill 会要求 Agent 先征得你的同意。": "The Skill's script needs Node.js 18 or later. Every call after this is made by the Agent and billed at the provider's prices; generation Skills tell the Agent to ask you first.", + "{model}（{provider}）已安装到 {agents}": "{model} ({provider}) installed in {agents}", + "卸载失败": "Uninstall failed", + "发送实测请求": "Send the probe", + "向模型服务发送一次最小的真实请求，确认这个模型能做{capability}。可能产生少量费用。": "Sends one minimal real request to confirm this model can do {capability}. This may incur a small charge.", + "向模型服务发送一次真实的生成请求，确认这个模型能做{capability}。会产生一次生成的费用，发送前会先确认。": "Sends one real generation request to confirm this model can do {capability}. It is billed as one generation; you are asked before it is sent.", + "在 Agent 里试试": "Try it in an Agent", + "复制提示词": "Copy prompt", + "安装{capability} Skill": "Install the {capability} Skill", + "安装会把 {provider} 的 API Key 写进所选 Agent 自己的配置文件，变量名是 {name}。Agent 运行的任何命令都能读到它，Agent 使用的模型也可能看到。": "Installing writes the {provider} API key into each selected Agent's own configuration as {name}. Every command the Agent runs can read it, and so may the Agent's model.", + "安装到": "Install in", + "安装失败": "Install failed", + "完成": "Done", + "将从 {agents} 移除这个 Skill。BootAgent 会保留一份备份；没有其他 Skill 再用到的 API Key 环境变量也会一并移除。": "This removes the Skill from {agents}. BootAgent keeps a backup, and removes any API key variable no other Skill still needs.", + "将替换这些 Agent 当前使用的 {models}。": "This replaces {models} on these Agents.", + "已安装到 {agents}": "Installed in {agents}", + "已安装的多模态 Skill": "Installed multimodal Skills", + "手动填写模型 ID": "Enter a model ID", + "暂不支持安装": "Install not supported yet", + "模型 ID": "Model ID", + "模型列表里没有专用的图片生成、语音识别或语音合成模型。如果你知道模型 ID（例如 gpt-image-1、whisper-1、tts-1），可以填写后实测，通过后即可安装。": "Model lists do not include dedicated image generation, speech-to-text or text-to-speech models. If you know a model ID (gpt-image-1, whisper-1, tts-1), enter it and probe it; once it passes it can be installed.", + "正在卸载": "Uninstalling", + "正在运行的 Agent 需要重启，才能读到新写入的环境变量。": "Restart any Agent that is already running so it picks up the new variable.", + "没有找到可以安装 Skill 的 Agent": "No Agent that can take this Skill was found", + "用「{model}」（{provider}）为 Agent 提供{capability}。": "Give Agents {capability} with \"{model}\" ({provider}).", + "确认实测": "Confirm the probe", + "这次实测会让「{model}」真实生成一张图片，按模型服务的价格计费。确定要发送吗？": "This probe makes \"{model}\" generate one real image, billed at the provider's price. Send it?", + "重新安装": "Reinstall", } as const; export type Locale = "zh-CN" | "en"; diff --git a/frontend/src/pages/SkillsPage.tsx b/frontend/src/pages/SkillsPage.tsx index 5d97acab..a84116bd 100644 --- a/frontend/src/pages/SkillsPage.tsx +++ b/frontend/src/pages/SkillsPage.tsx @@ -151,7 +151,7 @@ export function SkillsPage() { {rows.length ?
: null} {!rows.length ? {t("或者去工具市场发现")}} /> : !visibleRows.length ? :
{visibleRows.map((row) => { const candidate = candidateMap.get(row.id); const hash = candidate?.hash ?? changes[row.id]?.variant_hash ?? row.variant_hashes?.[0] ?? ""; const detail = row.description || t("暂无详情"); const isDeleted = Boolean(pendingDeletes[row.id]); return ; })}
{t("名称")}{t("详情")}{t("状态")}{t("选择目标 Agent")}
{row.name || row.id}{compactDescription(detail)}{row.id}{isDeleted ? {t("待删除")} : row.conflict ? {t("冲突")} : {row.variants} variant} selectTarget(row.id, hash, agent, checked, row.agents ?? [])} />
} - + {previewOpen ? setPreviewOpen(false)}>

{t("导入预览")}

{candidates.map((candidate) => { const detail = candidate.description || candidate.hash.slice(0, 12); return
; })}
 : null} ; } diff --git a/frontend/src/styles/app.css b/frontend/src/styles/app.css index 2f626cf1..e629a08f 100644 --- a/frontend/src/styles/app.css +++ b/frontend/src/styles/app.css @@ -3958,3 +3958,34 @@ .multimodal-verdict { flex-basis: 100%; margin: 0; } .multimodal-empty { margin: 0; color: var(--text-secondary); font-size: 13px; } .multimodal-error { margin: 0; color: var(--red); font-size: 13px; } +.multimodal-installed { display: grid; gap: 8px; } +.multimodal-installed h3 { margin: 0; font-size: 13px; font-weight: 650; } +.multimodal-installed-card { display: grid; gap: 6px; padding: 12px 14px; border: 1px solid var(--border); border-radius: var(--radius-panel); background: var(--surface-subtle); } +.multimodal-installed-card > header { display: flex; align-items: center; gap: 10px; font-size: 13px; } +.multimodal-installed-card > header code { color: var(--text-secondary); font-size: 12px; } +.multimodal-installed-card > header .icon-button { margin-left: auto; } +.multimodal-installed-line { margin: 0; color: var(--text-secondary); font-size: 13px; overflow-wrap: anywhere; } +.multimodal-prompts { display: grid; gap: 4px; } +.multimodal-prompts-title { color: var(--text-secondary); font-size: 12px; font-weight: 600; } +.multimodal-prompts ul { margin: 0; padding: 0; list-style: none; display: grid; gap: 4px; } +.multimodal-prompts li { display: flex; align-items: flex-start; gap: 6px; font-size: 13px; } +.multimodal-prompts li > span { flex: 1; min-width: 0; overflow-wrap: anywhere; } +.multimodal-prompts .icon-button { flex-shrink: 0; width: 26px; height: 26px; } +.multimodal-manual { font-size: 13px; } +.multimodal-manual summary { cursor: pointer; color: var(--text-secondary); } +.multimodal-manual p { margin: 6px 0; color: var(--text-secondary); } +.multimodal-manual-row { display: flex; flex-wrap: wrap; align-items: center; gap: 8px; } +.multimodal-manual-row select, +.multimodal-manual-row input { min-height: 28px; padding: 0 8px; border: 1px solid var(--border); border-radius: var(--radius-control); background: var(--window-bg); color: var(--text-primary); font: inherit; font-size: 12px; } +.multimodal-manual-row input { min-width: 0; width: 220px; } +.multimodal-modal p { margin: 12px 0; font-size: 13px; line-height: 1.5; } +.mcp-modal .multimodal-agents { margin: 12px 0; padding: 0; border: 0; display: flex; flex-wrap: wrap; gap: 14px; } +.mcp-modal .multimodal-agents legend { margin-bottom: 6px; color: var(--text-secondary); font-size: 13px; } +.mcp-modal .multimodal-agents label { margin: 0; display: inline-flex; align-items: center; gap: 6px; color: var(--text-primary); } +.mcp-modal .multimodal-agents input { width: auto; padding: 0; } +.multimodal-note { color: var(--text-secondary); } +.multimodal-key-notice { margin: 12px 0; } +.multimodal-results { margin: 12px 0; padding: 0; list-style: none; display: grid; gap: 6px; font-size: 13px; } +.multimodal-results li { display: flex; gap: 10px; } +.multimodal-results li.is-ok span { color: var(--green); } +.multimodal-results li.is-failed span { color: var(--red); overflow-wrap: anywhere; } diff --git a/frontend/src/types/api.ts b/frontend/src/types/api.ts index 1e98ec86..04352dd7 100644 --- a/frontend/src/types/api.ts +++ b/frontend/src/types/api.ts @@ -117,7 +117,23 @@ export type MultimodalCapability = Omit & { capabilities: MultimodalCapability[] }; export type MultimodalProvider = Omit & { models: MultimodalModel[] }; -export type MultimodalDetection = { providers: MultimodalProvider[] }; +export type InstalledMultimodal = Omit & { + capability: MultimodalCapabilityId; + agents: string[]; +}; +export type MultimodalManualRoute = Omit & { + capability: MultimodalCapabilityId; + probe: MultimodalProbeKind; +}; +export type MultimodalDetection = { + providers: MultimodalProvider[]; + installed: InstalledMultimodal[]; + targets: string[]; + generatable: string[]; + manual: MultimodalManualRoute[]; +}; +export type MultimodalInstallRequest = Omit & { capability: MultimodalCapabilityId }; +export type MultimodalInstallResult = Omit & { results: AppModels.MultimodalAgentResult[] }; export type MultimodalProbe = Omit & { capability: MultimodalCapabilityId; status: ProfileModelCheckStatus; diff --git a/internal/app/multimodal.go b/internal/app/multimodal.go index bb5e70d6..5d3b7eb0 100644 --- a/internal/app/multimodal.go +++ b/internal/app/multimodal.go @@ -7,20 +7,41 @@ import ( "sync" "github.com/MaimoryLab/BootAgent/internal/catalog" + configWriter "github.com/MaimoryLab/BootAgent/internal/config" oneerrors "github.com/MaimoryLab/BootAgent/internal/errors" "github.com/MaimoryLab/BootAgent/internal/modality" "github.com/MaimoryLab/BootAgent/internal/provider" + "github.com/MaimoryLab/BootAgent/internal/skillgen" ) // MultimodalDetection lists, for every Provider with a saved key, the models // that offer a multimodal capability and where each finding came from. type MultimodalDetection struct { Providers []MultimodalProvider `json:"providers"` + // Installed are the generated Skills published to at least one Agent. + Installed []InstalledMultimodal `json:"installed"` + // Targets are the Agents a multimodal Skill can be installed into now. + Targets []string `json:"targets"` + // Generatable lists the adapters a Skill can be generated for in this + // version; a capability on any other adapter is shown but not offered. + Generatable []string `json:"generatable"` + // Manual lists the capabilities a user can reach with a model ID typed by + // hand, the route it is probed on, and how that probe is sent. + Manual []MultimodalManualRoute `json:"manual"` +} + +type MultimodalManualRoute struct { + Capability string `json:"capability"` + Adapter string `json:"adapter"` + Probe string `json:"probe"` } type MultimodalProvider struct { ID string `json:"id"` Name string `json:"name"` + // KeyEnv is the variable an install writes this Provider's key into, so + // the install confirmation can name it. + KeyEnv string `json:"keyEnv"` // Listed is false when /models could not be read. Message and ErrorCode // then say why; catalog generation models are still listed, since they do // not come from /models. @@ -36,6 +57,7 @@ type MultimodalProbe struct { ProviderID string `json:"providerId"` Model string `json:"model"` Capability string `json:"capability"` + Adapter string `json:"adapter"` Status string `json:"status"` Reason string `json:"reason,omitempty"` HTTPStatus int `json:"httpStatus,omitempty"` @@ -97,11 +119,27 @@ func (u *UseCases) DetectMultimodal(ctx context.Context) (MultimodalDetection, e if err := contextError(ctx, "Multimodal detection was cancelled"); err != nil { return MultimodalDetection{}, err } - return MultimodalDetection{Providers: results}, nil + installTargets, _, err := u.multimodalInstallTargets() + if err != nil { + return MultimodalDetection{}, err + } + if installTargets == nil { + installTargets = []string{} + } + manual := []MultimodalManualRoute{} + for _, capability := range modality.Capabilities { + if adapter := modality.ManualAdapter(capability); adapter != "" { + manual = append(manual, MultimodalManualRoute{Capability: capability, Adapter: adapter, Probe: modality.ProbeKind(adapter)}) + } + } + return MultimodalDetection{ + Providers: results, Installed: u.installedMultimodal(), Targets: installTargets, + Generatable: skillgen.SupportedAdapters(), Manual: manual, + }, nil } func (u *UseCases) detectProvider(ctx context.Context, rules modality.Rules, target provider.Entry) MultimodalProvider { - result := MultimodalProvider{ID: target.ID, Name: target.Name, Models: []modality.Model{}} + result := MultimodalProvider{ID: target.ID, Name: target.Name, KeyEnv: configWriter.SkillEnvName(target.ID), Models: []modality.Model{}} listing, err := u.provider.ListModels(ctx, "custom", target.APIKey, target.BaseURL) var entries []provider.ModelEntry switch { @@ -124,13 +162,16 @@ func (u *UseCases) detectProvider(ctx context.Context, rules modality.Rules, tar return result } -// ProbeMultimodal sends one live request testing a model's capability. It is a -// billed request, so only a button the user presses may call it; detection, -// saving and installing never do. A conclusive answer -- accepted, or refused -// for this model -- is cached and outranks every other source on the next -// detection. A rejected key or an unreachable endpoint says nothing about the -// model and is not cached. -func (u *UseCases) ProbeMultimodal(ctx context.Context, providerID, model, capability string) (*MultimodalProbe, error) { +// ProbeMultimodal sends one live request testing a model's capability over an +// adapter (empty: the capability's chat route). It is a billed request, so +// only a button the user presses may call it; detection, saving and installing +// never do. A probe that produces billed output (one image) also needs +// confirmed, which the UI sets only after asking. +// +// A conclusive answer -- accepted, or refused for this model -- is cached and +// outranks every other source on the next detection. A rejected key or an +// unreachable endpoint says nothing about the model and is not cached. +func (u *UseCases) ProbeMultimodal(ctx context.Context, providerID, model, capability, adapter string, confirmed bool) (*MultimodalProbe, error) { if u == nil || u.provider == nil { return nil, oneerrors.New(oneerrors.InternalError, "Multimodal probing is not configured", oneerrors.WithStatus(501)) } @@ -141,14 +182,17 @@ func (u *UseCases) ProbeMultimodal(ctx context.Context, providerID, model, capab if model == "" { return nil, oneerrors.New(oneerrors.InvalidRequest, "A model is required") } - adapter := modality.ChatAdapter(capability) - // Only probes that cost a token or so can be sent from here. A probe that - // produces billed output needs its own confirmation path, and the rest do - // not exist yet. - if modality.ProbeKind(adapter) != modality.ProbeMinimal { + if adapter == "" { + adapter = modality.ChatAdapter(capability) + } + kind := modality.ProbeKind(adapter) + if !modality.Serves(adapter, capability) || kind == modality.ProbeNone { return nil, oneerrors.New(oneerrors.InvalidRequest, "No live check is available for this capability: "+strings.TrimSpace(capability)) } - check := &MultimodalProbe{ProviderID: providerID, Model: model, Capability: capability} + if kind == modality.ProbeConfirm && !confirmed { + return nil, oneerrors.New(oneerrors.InvalidRequest, "This check produces billed output; confirm it first") + } + check := &MultimodalProbe{ProviderID: providerID, Model: model, Capability: capability, Adapter: adapter} target, err := u.providers.Get(providerID) if err != nil { return nil, err @@ -157,7 +201,20 @@ func (u *UseCases) ProbeMultimodal(ctx context.Context, providerID, model, capab check.Status, check.Reason = ModelCheckUnverified, "no_key" return check, nil } - result, err := u.provider.ProbeVision(ctx, target.APIKey, model, target.BaseURL) + var probe func(context.Context, string, string, string) (provider.ProbeResult, error) + switch adapter { + case modality.AdapterChatVision: + probe = u.provider.ProbeVision + case modality.AdapterAudioTranscriptions: + probe = u.provider.ProbeTranscription + case modality.AdapterAudioSpeech: + probe = u.provider.ProbeSpeech + case modality.AdapterImages: + probe = u.provider.ProbeImageGeneration + default: + return nil, oneerrors.New(oneerrors.InvalidRequest, "No live check is available for "+adapter) + } + result, err := probe(ctx, target.APIKey, model, target.BaseURL) if ctx.Err() != nil { return nil, contextError(ctx, "Multimodal probe was cancelled") } diff --git a/internal/app/multimodal_install.go b/internal/app/multimodal_install.go new file mode 100644 index 00000000..7dd3bcfe --- /dev/null +++ b/internal/app/multimodal_install.go @@ -0,0 +1,518 @@ +package app + +import ( + "context" + "errors" + "maps" + "os" + "path/filepath" + "regexp" + "slices" + "sort" + "strings" + + "github.com/MaimoryLab/BootAgent/internal/catalog" + configWriter "github.com/MaimoryLab/BootAgent/internal/config" + oneerrors "github.com/MaimoryLab/BootAgent/internal/errors" + "github.com/MaimoryLab/BootAgent/internal/modality" + "github.com/MaimoryLab/BootAgent/internal/provider" + "github.com/MaimoryLab/BootAgent/internal/skill" + "github.com/MaimoryLab/BootAgent/internal/skillgen" +) + +// MultimodalInstallRequest installs the Skill for one capability, bound to one +// Provider and model, into the named Agents. +type MultimodalInstallRequest struct { + Provider string `json:"provider"` + Model string `json:"model"` + Capability string `json:"capability"` + Agents []string `json:"agents"` +} + +type MultimodalAgentResult struct { + Agent string `json:"agent"` + // Installed: the Skill tree is in the Agent's Skills directory. + Installed bool `json:"installed"` + // EnvWritten: the key variable is in the Agent's own configuration. + EnvWritten bool `json:"envWritten"` + Error string `json:"error,omitempty"` +} + +type MultimodalInstallResult struct { + SkillID string `json:"skillId"` + Results []MultimodalAgentResult `json:"results"` +} + +// InstalledMultimodal is one generated Skill variant that is published to at +// least one Agent. +type InstalledMultimodal struct { + SkillID string `json:"skillId"` + Capability string `json:"capability"` + ProviderID string `json:"providerId"` + Model string `json:"model"` + Adapter string `json:"adapter"` + Agents []string `json:"agents"` +} + +// nativeSkillEnvAdapters are the config adapters with a writer for Skill +// variables (ADR-011). Other Agents get the private fallback file. +var nativeSkillEnvAdapters = map[string]bool{"claude-code": true, "codex": true} + +// multimodalInstallTargets are the Agents a multimodal Skill can be installed +// into from the multimodal section: those with a Skills directory BootAgent +// can see, and a native place for the key variable. Phase 4 widens this. +func (u *UseCases) multimodalInstallTargets() ([]string, map[string]catalog.Agent, error) { + eligible, err := u.eligibleSkillAgents() + if err != nil { + return nil, nil, err + } + var targets []string + for id, agent := range eligible { + if nativeSkillEnvAdapters[agent.ConfigAdapter] { + targets = append(targets, id) + } + } + sort.Strings(targets) + return targets, eligible, nil +} + +// InstallMultimodal generates the Skill, stores it as a variant with source +// "generated", publishes it to each Agent and writes the key variable there. +// It sends no billed request: the evidence it installs on is the free model +// listing, or a probe the user already ran. A capability found only by an ID +// rule, or on an unverified adapter without a passing probe, is refused. +func (u *UseCases) InstallMultimodal(ctx context.Context, request MultimodalInstallRequest) (MultimodalInstallResult, error) { + if u == nil || u.provider == nil { + return MultimodalInstallResult{}, oneerrors.New(oneerrors.InternalError, "Multimodal install is not configured", oneerrors.WithStatus(501)) + } + if err := contextError(ctx, "Multimodal install was cancelled"); err != nil { + return MultimodalInstallResult{}, err + } + model := strings.TrimSpace(request.Model) + capability := strings.TrimSpace(request.Capability) + result := MultimodalInstallResult{SkillID: skillgen.SkillID(capability), Results: []MultimodalAgentResult{}} + if model == "" || !slices.Contains(modality.Capabilities, capability) { + return result, oneerrors.New(oneerrors.InvalidRequest, "A model and a known capability are required") + } + agents := slices.Compact(slices.Sorted(slices.Values(request.Agents))) + if len(agents) == 0 { + return result, oneerrors.New(oneerrors.InvalidRequest, "Choose at least one Agent") + } + target, err := u.providers.Get(request.Provider) + if err != nil { + return result, err + } + if strings.TrimSpace(target.APIKey) == "" { + return result, oneerrors.New(oneerrors.InvalidRequest, "Save an API key for this Provider first") + } + rules, err := modality.Load() + if err != nil { + return result, err + } + found, err := u.resolveForInstall(ctx, rules, target, model, capability) + if err != nil { + return result, err + } + if !found.Installable { + return result, oneerrors.New(oneerrors.InvalidRequest, "This model's "+capability+" is not confirmed yet; run the check first") + } + if !skillgen.Supports(capability, found.Adapter) { + return result, oneerrors.New(oneerrors.InvalidRequest, "A Skill for this route cannot be generated yet") + } + spec := skillgen.Spec{ + Capability: capability, Adapter: found.Adapter, ProviderID: target.ID, ProviderName: target.Name, + Model: model, BaseURL: target.BaseURL, NativeBaseURL: rules.NativeBaseURL(target.ID), + } + + u.writeMu.Lock() + defer u.writeMu.Unlock() + targets, _, err := u.multimodalInstallTargets() + if err != nil { + return result, err + } + var install []string + for _, agentID := range agents { + if slices.Contains(targets, agentID) { + install = append(install, agentID) + } else { + result.Results = append(result.Results, MultimodalAgentResult{Agent: agentID, Error: "This Agent cannot take a multimodal Skill yet"}) + } + } + if len(install) == 0 { + return result, nil + } + published, err := u.publishGeneratedLocked(ctx, spec, install) + if err != nil { + return result, err + } + envFailures := u.reconcileSkillEnvLocked(ctx) + for _, agentID := range install { + item := MultimodalAgentResult{Agent: agentID} + if failure, ok := published[agentID]; ok && failure != "" { + item.Error = failure + } else { + item.Installed = true + if failure, ok := envFailures[agentID]; ok { + item.Error = "Skill installed, but its key variable was not written: " + failure + } else { + item.EnvWritten = true + } + } + result.Results = append(result.Results, item) + } + sort.Slice(result.Results, func(i, j int) bool { return result.Results[i].Agent < result.Results[j].Agent }) + return result, nil +} + +// resolveForInstall finds the evidence for one model's capability. A probe +// verdict needs no network; otherwise the free listing is read again rather +// than trusting what the page sent. +func (u *UseCases) resolveForInstall(ctx context.Context, rules modality.Rules, target provider.Entry, model, capability string) (modality.Capability, error) { + verdicts := u.capabilities.Modalities(target) + if found, ok := modality.Find(rules.Resolve(target.ID, nil, verdicts), model, capability); ok { + return found, nil + } + listing, err := u.provider.ListModels(ctx, "custom", target.APIKey, target.BaseURL) + if err != nil { + return modality.Capability{}, err + } + if !listing.OK { + return modality.Capability{}, oneerrors.New(oneerrors.ProviderUnreachable, "Cannot confirm the model: "+listing.Message, oneerrors.WithRetryable(true)) + } + found, ok := modality.Find(rules.Resolve(target.ID, listing.Entries, verdicts), model, capability) + if !ok { + return modality.Capability{}, oneerrors.New(oneerrors.InvalidRequest, "This Provider does not offer "+capability+" on "+model) + } + return found, nil +} + +// publishGeneratedLocked renders the Skill, stores it as a variant and +// publishes it to each Agent. It returns a per-Agent error message ("" on +// success); an error return means nothing was published. Callers hold writeMu. +func (u *UseCases) publishGeneratedLocked(ctx context.Context, spec skillgen.Spec, agentIDs []string) (map[string]string, error) { + id := skillgen.SkillID(spec.Capability) + store := u.skillStore() + registry, err := store.Load() + if err != nil { + return nil, oneerrors.New(oneerrors.ConfigWriteFailed, "Cannot read the Skill Registry", oneerrors.WithCause(err)) + } + fact, exists := registry.Skills[id] + if exists && !u.isGeneratedFact(id, fact) { + return nil, oneerrors.New(oneerrors.InvalidRequest, "A Skill named "+id+" that BootAgent did not generate is already managed; remove it first") + } + eligible, err := u.eligibleSkillAgents() + if err != nil { + return nil, err + } + if err := store.EnsureSkillsRoot(ctx); err != nil { + return nil, oneerrors.New(oneerrors.ConfigWriteFailed, "Cannot prepare Skill storage", oneerrors.WithCause(err)) + } + stage, err := os.MkdirTemp(store.SkillsRoot(), ".bootagent-generate-") + if err != nil { + return nil, oneerrors.New(oneerrors.ConfigWriteFailed, "Cannot prepare the Skill", oneerrors.WithCause(err)) + } + defer os.RemoveAll(stage) + tree := filepath.Join(stage, id) + if err := os.Mkdir(tree, 0o700); err != nil { + return nil, oneerrors.New(oneerrors.ConfigWriteFailed, "Cannot prepare the Skill", oneerrors.WithCause(err)) + } + if err := skillgen.Render(spec, tree); err != nil { + return nil, oneerrors.New(oneerrors.InvalidRequest, err.Error()) + } + stats, err := skill.HashTree(ctx, tree) + if err != nil { + return nil, oneerrors.New(oneerrors.ConfigWriteFailed, "Cannot hash the Skill", oneerrors.WithCause(err)) + } + index := variantIndex(fact.Variants, stats.Hash) + if index < 0 || !fact.Variants[index].Stored { + if err := store.SaveVariant(ctx, id, tree, stats); err != nil { + return nil, oneerrors.New(oneerrors.ConfigWriteFailed, "Cannot store the Skill", oneerrors.WithCause(err)) + } + } + if index < 0 { + fact.Variants = append(fact.Variants, skill.Variant{Hash: stats.Hash}) + index = len(fact.Variants) - 1 + } + fact.Name, fact.Description, _ = skill.ReadMetadata(ctx, tree, id) + fact.Variants[index].Stored = true + fact.Variants[index].ImportSources = addSorted(fact.Variants[index].ImportSources, skill.ImportSourceGenerated) + registry.Skills[id] = fact + if err := store.Save(ctx, registry); err != nil { + return nil, oneerrors.New(oneerrors.ConfigWriteFailed, "Cannot update the Skill Registry", oneerrors.WithCause(err)) + } + + results := map[string]string{} + for _, agentID := range agentIDs { + agent, ok := eligible[agentID] + if !ok { + results[agentID] = "Agent is not eligible" + continue + } + destination := filepath.Join(skillPath(u.status.Home, u.status.Platform.OS, agent), id) + if hasSymlinkComponent(u.status.Home, filepath.Dir(destination)) { + results[agentID] = "Skill target path is not private" + continue + } + if err := verifyGeneratedTarget(ctx, destination, fact); err != nil { + results[agentID] = err.Error() + continue + } + if err := skill.PublishTree(ctx, store.VariantPath(id, stats.Hash), destination); err != nil { + results[agentID] = "Cannot publish Skill to Agent" + continue + } + for i := range fact.Variants { + if i != index { + fact.Variants[i].ManagedTargets = removeString(fact.Variants[i].ManagedTargets, agentID) + } + } + fact.Variants[index].ManagedTargets = addSorted(fact.Variants[index].ManagedTargets, agentID) + registry.Skills[id] = fact + if err := store.Save(ctx, registry); err != nil { + results[agentID] = "Agent updated but Skill Registry was not updated" + continue + } + results[agentID] = "" + } + return results, nil +} + +// verifyGeneratedTarget allows replacing what is at the destination only when +// it is a variant of this Skill BootAgent stored. Anything else -- a Skill the +// user wrote under the same name, or an edited copy -- is left alone. +func verifyGeneratedTarget(ctx context.Context, destination string, fact skill.Fact) error { + info, err := os.Lstat(destination) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return errors.New("A file BootAgent did not write is in the way; it was left alone") + } + stats, err := skill.HashTree(ctx, destination) + if err != nil || variantIndex(fact.Variants, stats.Hash) < 0 { + return errors.New("A Skill with this name that BootAgent did not write is installed here; it was left alone") + } + return nil +} + +// isGeneratedFact reports whether a registry entry is a generated Skill. A +// variant rendered here carries the generated source; one copied from another +// machine arrives as a ZIP, but its stored config.json still reads as ours. +func (u *UseCases) isGeneratedFact(id string, fact skill.Fact) bool { + if skillgen.CapabilityFromSkillID(id) == "" { + return false + } + store := u.skillStore() + for _, variant := range fact.Variants { + if slices.Contains(variant.ImportSources, skill.ImportSourceGenerated) { + continue + } + if !variant.Stored { + return false + } + if _, err := skillgen.ReadConfig(store.VariantPath(id, variant.Hash)); err != nil { + return false + } + } + return true +} + +// generatedVariant is one rendered variant and the config it was made from. +type generatedVariant struct { + id string + hash string + config skillgen.Config + targets []string +} + +// generatedVariants lists the variants BootAgent rendered here. Only these +// earn a key variable: a tree imported from a ZIP could name any base URL, and +// writing a real key for it would hand that key to whoever wrote the ZIP. +func (u *UseCases) generatedVariants(registry skill.Registry) []generatedVariant { + store := u.skillStore() + var result []generatedVariant + for _, id := range slices.Sorted(maps.Keys(registry.Skills)) { + if skillgen.CapabilityFromSkillID(id) == "" { + continue + } + for _, variant := range registry.Skills[id].Variants { + if !variant.Stored || !slices.Contains(variant.ImportSources, skill.ImportSourceGenerated) { + continue + } + config, err := skillgen.ReadConfig(store.VariantPath(id, variant.Hash)) + if err != nil { + continue + } + result = append(result, generatedVariant{id: id, hash: variant.Hash, config: config, targets: variant.ManagedTargets}) + } + } + return result +} + +var fallbackEnvFile = regexp.MustCompile(`^([a-z0-9][a-z0-9-]{0,63})\.env$`) + +// reconcileSkillEnvLocked makes every Agent's Skill variables match what its +// generated Skills need: each Agent with a generated Skill for Provider P gets +// P's key; variables no Skill needs any more are removed. Claude Code and +// Codex get them in their own configuration; any other Agent a generated Skill +// is published to reads the private fallback file. It returns failures by +// Agent and never fails the caller's own change. Callers hold writeMu. +func (u *UseCases) reconcileSkillEnvLocked(ctx context.Context) map[string]string { + failures := map[string]string{} + registry, err := u.skillStore().Load() + if err != nil { + return failures + } + manifest, err := catalog.LoadEmbedded() + if err != nil { + return failures + } + keys := map[string]string{} + keyFor := func(providerID string) string { + if key, ok := keys[providerID]; ok { + return key + } + entry, err := u.providers.Get(providerID) + key := "" + if err == nil { + key = strings.TrimSpace(entry.APIKey) + } + keys[providerID] = key + return key + } + // desired[agent][variable] = key; fallback[provider] = agents that need it. + desired := map[string]map[string]string{} + fallback := map[string][]string{} + for _, variant := range u.generatedVariants(registry) { + for _, agentID := range variant.targets { + agent, ok := manifest.Agents[agentID] + if !ok { + continue + } + key := keyFor(variant.config.ProviderID) + if key == "" { + failures[agentID] = "Provider " + variant.config.ProviderID + " has no saved API key" + continue + } + if nativeSkillEnvAdapters[agent.ConfigAdapter] { + if desired[agentID] == nil { + desired[agentID] = map[string]string{} + } + desired[agentID][variant.config.KeyEnv] = key + } else { + fallback[variant.config.ProviderID] = append(fallback[variant.config.ProviderID], agentID) + } + } + } + writer := configWriter.NewWriter(u.status.Home, u.status.Platform.OS, u.filesystem) + for _, agentID := range slices.Sorted(maps.Keys(manifest.Agents)) { + agent := manifest.Agents[agentID] + if !nativeSkillEnvAdapters[agent.ConfigAdapter] { + continue + } + path := configPath(u.status.Home, u.status.Platform.OS, agent) + if path == "" { + continue + } + var err error + switch agent.ConfigAdapter { + case "claude-code": + err = writer.WriteClaudeSkillEnv(ctx, path, desired[agentID]) + case "codex": + err = writer.WriteCodexSkillEnv(ctx, path, desired[agentID]) + } + if err != nil { + failures[agentID] = oneerrors.As(err).Message + } + } + for providerID, agents := range fallback { + path := configWriter.SkillEnvFilePath(u.status.Home, providerID) + if err := writer.WriteSkillEnvFile(ctx, path, configWriter.SkillEnvName(providerID), keyFor(providerID)); err != nil { + for _, agentID := range agents { + failures[agentID] = oneerrors.As(err).Message + } + } + } + directory := filepath.Dir(configWriter.SkillEnvFilePath(u.status.Home, "x")) + if entries, err := os.ReadDir(directory); err == nil { + for _, entry := range entries { + match := fallbackEnvFile.FindStringSubmatch(entry.Name()) + if match == nil || !entry.Type().IsRegular() { + continue + } + if _, needed := fallback[match[1]]; !needed { + _ = configWriter.RemoveSkillEnvFile(filepath.Join(directory, entry.Name())) + } + } + } + return failures +} + +// installedMultimodal lists the generated variants published somewhere. +func (u *UseCases) installedMultimodal() []InstalledMultimodal { + registry, err := u.skillStore().Load() + if err != nil { + return []InstalledMultimodal{} + } + result := []InstalledMultimodal{} + for _, variant := range u.generatedVariants(registry) { + if len(variant.targets) == 0 { + continue + } + result = append(result, InstalledMultimodal{ + SkillID: variant.id, Capability: variant.config.Capability, ProviderID: variant.config.ProviderID, + Model: variant.config.Model, Adapter: variant.config.Adapter, Agents: slices.Clone(variant.targets), + }) + } + return result +} + +// generatedSkillsUsing lists the published generated Skills bound to a +// Provider, for the Provider delete guard and the base URL refresh. +func (u *UseCases) generatedSkillsUsing(providerID string) []generatedVariant { + registry, err := u.skillStore().Load() + if err != nil { + return nil + } + var result []generatedVariant + for _, variant := range u.generatedVariants(registry) { + if variant.config.ProviderID == providerID && len(variant.targets) > 0 { + result = append(result, variant) + } + } + return result +} + +// refreshGeneratedSkillsLocked re-renders a Provider's generated Skills after +// its endpoint changed, so config.json does not point at the old base. The +// key needs no re-render: it lives in the Agents' configuration, which the +// env reconcile rewrites. Failures are returned by Agent. Callers hold writeMu. +func (u *UseCases) refreshGeneratedSkillsLocked(ctx context.Context, target provider.Entry) map[string]string { + failures := map[string]string{} + for _, variant := range u.generatedSkillsUsing(target.ID) { + if provider.OpenAIBaseURL(target.BaseURL) == variant.config.BaseURL && target.Name == variant.config.ProviderName { + continue + } + spec := skillgen.Spec{ + Capability: variant.config.Capability, Adapter: variant.config.Adapter, + ProviderID: target.ID, ProviderName: target.Name, Model: variant.config.Model, + BaseURL: target.BaseURL, NativeBaseURL: variant.config.NativeBaseURL, + } + published, err := u.publishGeneratedLocked(ctx, spec, variant.targets) + if err != nil { + for _, agentID := range variant.targets { + failures[agentID] = oneerrors.As(err).Message + } + continue + } + for agentID, failure := range published { + if failure != "" { + failures[agentID] = failure + } + } + } + return failures +} diff --git a/internal/app/multimodal_install_test.go b/internal/app/multimodal_install_test.go new file mode 100644 index 00000000..b54ba9c0 --- /dev/null +++ b/internal/app/multimodal_install_test.go @@ -0,0 +1,386 @@ +package app + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/MaimoryLab/BootAgent/internal/modality" + "github.com/MaimoryLab/BootAgent/internal/platform" + "github.com/MaimoryLab/BootAgent/internal/provider" + "github.com/MaimoryLab/BootAgent/internal/skill" + "github.com/MaimoryLab/BootAgent/internal/skillgen" + "github.com/pelletier/go-toml/v2" +) + +// multimodalInstallCore is a home where every Agent command is found, so the Skills +// directories of Claude Code, Codex, OpenCode and Hermes are all eligible, +// with one keyed custom Provider "gw". +func multimodalInstallCore(t *testing.T, gateway *multimodalGateway) (*UseCases, string) { + t.Helper() + home := t.TempDir() + core := NewUseCasesWithProviderClient(StatusOptions{ + Home: home, Platform: platform.Info{OS: "darwin", Arch: "arm64"}, + Lookup: func(command string) (string, bool) { return "/fake/" + command, true }, + }, provider.NewClient(gateway)) + if _, err := core.SaveProvider(context.Background(), provider.Entry{ + ID: "gw", Name: "Gateway", BaseURL: "https://gw.example/v1", APIKey: multimodalKey, + }, true, false); err != nil { + t.Fatal(err) + } + return core, home +} + +func claudeEnv(t *testing.T, home string) map[string]any { + t.Helper() + data, err := os.ReadFile(filepath.Join(home, ".claude", "settings.json")) + if os.IsNotExist(err) { + return map[string]any{} + } + if err != nil { + t.Fatal(err) + } + var document struct { + Env map[string]any `json:"env"` + } + if err := json.Unmarshal(data, &document); err != nil { + t.Fatal(err) + } + return document.Env +} + +func codexSet(t *testing.T, home string) map[string]any { + t.Helper() + data, err := os.ReadFile(filepath.Join(home, ".codex", "config.toml")) + if os.IsNotExist(err) { + return map[string]any{} + } + if err != nil { + t.Fatal(err) + } + var document map[string]any + if err := toml.Unmarshal(data, &document); err != nil { + t.Fatal(err) + } + policy, _ := document["shell_environment_policy"].(map[string]any) + set, _ := policy["set"].(map[string]any) + if set == nil { + return map[string]any{} + } + return set +} + +// keyOutside walks every file under root except the allowed ones and reports +// any that carries the key. +func keyOutside(t *testing.T, root string, allowed ...string) []string { + t.Helper() + var leaks []string + _ = filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { + if err != nil || entry.IsDir() { + return nil + } + for _, allow := range allowed { + if path == allow || strings.HasPrefix(path, allow+".backup-") || strings.HasPrefix(filepath.Dir(path), filepath.Join(root, ".bootagent", "backup")) && strings.Contains(path, filepath.Base(allow)) { + return nil + } + } + data, _ := os.ReadFile(path) + if strings.Contains(string(data), multimodalKey) { + rel, _ := filepath.Rel(root, path) + leaks = append(leaks, rel) + } + return nil + }) + return leaks +} + +func installSkill(t *testing.T, core *UseCases, model, capability string, agents ...string) MultimodalInstallResult { + t.Helper() + result, err := core.InstallMultimodal(context.Background(), MultimodalInstallRequest{Provider: "gw", Model: model, Capability: capability, Agents: agents}) + if err != nil { + t.Fatalf("install %s/%s: %v", model, capability, err) + } + return result +} + +func TestInstallMultimodalPublishesAndWritesTheKeyOnlyWhereItBelongs(t *testing.T) { + gateway := &multimodalGateway{listing: gatewayListing} + core, home := multimodalInstallCore(t, gateway) + result := installSkill(t, core, "qwen/qwen3-vl-8b", modality.Vision, "claude-code", "codex") + if result.SkillID != "bootagent-vision" || len(result.Results) != 2 { + t.Fatalf("result = %#v", result) + } + for _, item := range result.Results { + if !item.Installed || !item.EnvWritten || item.Error != "" { + t.Fatalf("agent result = %#v", item) + } + } + // Installing reads the free listing; it never sends a billed request. + if posts := gateway.billed(); len(posts) != 0 { + t.Fatalf("install sent billed requests: %v", posts) + } + for _, root := range []string{filepath.Join(home, ".claude", "skills"), filepath.Join(home, ".codex", "skills")} { + cfg, err := skillgen.ReadConfig(filepath.Join(root, "bootagent-vision")) + if err != nil || cfg.Model != "qwen/qwen3-vl-8b" || cfg.BaseURL != "https://gw.example/v1" { + t.Fatalf("%s: config = %#v, %v", root, cfg, err) + } + } + if claudeEnv(t, home)["BOOTAGENT_GW_API_KEY"] != multimodalKey || codexSet(t, home)["BOOTAGENT_GW_API_KEY"] != multimodalKey { + t.Fatal("key variable missing from an Agent's configuration") + } + registry, err := core.skillStore().Load() + if err != nil { + t.Fatal(err) + } + variant := registry.Skills["bootagent-vision"].Variants[0] + if len(variant.ImportSources) != 1 || variant.ImportSources[0] != skill.ImportSourceGenerated || len(variant.ManagedTargets) != 2 { + t.Fatalf("registry variant = %#v", variant) + } + // The key lives in the Provider store and the two Agent configs, and + // nowhere else: not in any Skill tree, the registry, the variant store, + // the capability cache or a log. + allowed := []string{ + filepath.Join(home, ".bootagent", "providers.json"), + filepath.Join(home, ".claude", "settings.json"), + filepath.Join(home, ".codex", "config.toml"), + } + if leaks := keyOutside(t, home, allowed...); len(leaks) != 0 { + t.Fatalf("the key leaked into: %v", leaks) + } + detection, err := core.DetectMultimodal(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(detection.Installed) != 1 || detection.Installed[0].Model != "qwen/qwen3-vl-8b" || len(detection.Installed[0].Agents) != 2 { + t.Fatalf("installed = %#v", detection.Installed) + } + if strings.Join(detection.Targets, ",") != "claude-code,codex" { + t.Fatalf("targets = %v", detection.Targets) + } +} + +func TestInstallMultimodalRefusesWeakEvidence(t *testing.T) { + gateway := &multimodalGateway{listing: gatewayListing} + core, _ := multimodalInstallCore(t, gateway) + // kimi-k3 is found by an ID rule alone. + if _, err := core.InstallMultimodal(context.Background(), MultimodalInstallRequest{Provider: "gw", Model: "moonshotai/kimi-k3", Capability: modality.Vision, Agents: []string{"codex"}}); err == nil { + t.Fatal("an ID-rule-only model was installed") + } + // A model the listing does not have and nobody probed. + if _, err := core.InstallMultimodal(context.Background(), MultimodalInstallRequest{Provider: "gw", Model: "whisper-1", Capability: modality.SpeechToText, Agents: []string{"codex"}}); err == nil { + t.Fatal("an unprobed hand-entered model was installed") + } + // After a passing probe both install. + if check, err := core.ProbeMultimodal(context.Background(), "gw", "moonshotai/kimi-k3", modality.Vision, "", false); err != nil || check.Status != ModelCheckAvailable { + t.Fatalf("probe = %#v, %v", check, err) + } + if check, err := core.ProbeMultimodal(context.Background(), "gw", "whisper-1", modality.SpeechToText, modality.AdapterAudioTranscriptions, false); err != nil || check.Status != ModelCheckAvailable { + t.Fatalf("probe = %#v, %v", check, err) + } + for _, target := range [][2]string{{"moonshotai/kimi-k3", modality.Vision}, {"whisper-1", modality.SpeechToText}} { + result := installSkill(t, core, target[0], target[1], "codex") + if !result.Results[0].Installed { + t.Fatalf("%v after a probe: %#v", target, result) + } + } + // An Agent with no native place for the key is not offered here. + result := installSkill(t, core, "qwen/qwen3-vl-8b", modality.Vision, "opencode") + if len(result.Results) != 1 || result.Results[0].Installed || result.Results[0].Error == "" { + t.Fatalf("opencode result = %#v", result) + } +} + +func TestImageProbeNeedsConfirmation(t *testing.T) { + gateway := &multimodalGateway{listing: gatewayListing} + core, _ := multimodalInstallCore(t, gateway) + if _, err := core.ProbeMultimodal(context.Background(), "gw", "gpt-image-1", modality.ImageGeneration, modality.AdapterImages, false); err == nil { + t.Fatal("an image probe ran without confirmation") + } + if posts := gateway.billed(); len(posts) != 0 { + t.Fatalf("an unconfirmed image probe still sent: %v", posts) + } + check, err := core.ProbeMultimodal(context.Background(), "gw", "gpt-image-1", modality.ImageGeneration, modality.AdapterImages, true) + if err != nil || check.Status != ModelCheckAvailable { + t.Fatalf("confirmed probe = %#v, %v", check, err) + } + // The unverified chat image route has no probe at all yet. + if _, err := core.ProbeMultimodal(context.Background(), "gw", "gemini-3-pro-image", modality.ImageGeneration, modality.AdapterChatImageOutput, true); err == nil { + t.Fatal("an unverified route was probed") + } + // An adapter for another capability is refused. + if _, err := core.ProbeMultimodal(context.Background(), "gw", "m", modality.Vision, modality.AdapterAudioSpeech, false); err == nil { + t.Fatal("a mismatched adapter was accepted") + } +} + +func TestChangingTheModelReplacesTheManagedVariant(t *testing.T) { + gateway := &multimodalGateway{listing: `{"data":[ + {"id":"vision-a","input_modalities":["text","image"],"output_modalities":["text"]}, + {"id":"vision-b","input_modalities":["text","image"],"output_modalities":["text"]} + ]}`} + core, home := multimodalInstallCore(t, gateway) + installSkill(t, core, "vision-a", modality.Vision, "codex") + result := installSkill(t, core, "vision-b", modality.Vision, "codex") + if !result.Results[0].Installed { + t.Fatalf("switch = %#v", result) + } + cfg, err := skillgen.ReadConfig(filepath.Join(home, ".codex", "skills", "bootagent-vision")) + if err != nil || cfg.Model != "vision-b" { + t.Fatalf("published config = %#v, %v", cfg, err) + } + registry, _ := core.skillStore().Load() + fact := registry.Skills["bootagent-vision"] + managed := 0 + for _, variant := range fact.Variants { + managed += len(variant.ManagedTargets) + } + if len(fact.Variants) != 2 || managed != 1 { + t.Fatalf("variants = %#v", fact.Variants) + } + if codexSet(t, home)["BOOTAGENT_GW_API_KEY"] != multimodalKey { + t.Fatal("the key variable was lost on a model switch") + } +} + +func TestInstallLeavesAnUnmanagedSkillAlone(t *testing.T) { + gateway := &multimodalGateway{listing: gatewayListing} + core, home := multimodalInstallCore(t, gateway) + mine := filepath.Join(home, ".claude", "skills", "bootagent-vision") + writeTestSkillAt(t, mine, "the user's own") + result := installSkill(t, core, "qwen/qwen3-vl-8b", modality.Vision, "claude-code", "codex") + byAgent := map[string]MultimodalAgentResult{} + for _, item := range result.Results { + byAgent[item.Agent] = item + } + if byAgent["claude-code"].Installed || byAgent["claude-code"].Error == "" || !byAgent["codex"].Installed { + t.Fatalf("results = %#v", result.Results) + } + if data, _ := os.ReadFile(filepath.Join(mine, "SKILL.md")); !strings.Contains(string(data), "the user's own") { + t.Fatal("the user's Skill was overwritten") + } + if _, present := claudeEnv(t, home)["BOOTAGENT_GW_API_KEY"]; present { + t.Fatal("a key was written to an Agent the Skill did not reach") + } +} + +func TestUninstallRemovesTheVariableOnlyWithTheLastSkill(t *testing.T) { + gateway := &multimodalGateway{listing: gatewayListing} + core, home := multimodalInstallCore(t, gateway) + if err := os.MkdirAll(filepath.Join(home, ".claude"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(home, ".claude", "settings.json"), []byte(`{"env":{"MY_VAR":"mine"}}`), 0o600); err != nil { + t.Fatal(err) + } + if _, err := core.ProbeMultimodal(context.Background(), "gw", "tts-1", modality.TextToSpeech, modality.AdapterAudioSpeech, false); err != nil { + t.Fatal(err) + } + installSkill(t, core, "qwen/qwen3-vl-8b", modality.Vision, "claude-code") + installSkill(t, core, "tts-1", modality.TextToSpeech, "claude-code") + if result := core.UninstallSkill(context.Background(), "bootagent-vision"); result.Error != "" || !result.RegistryUpdated { + t.Fatalf("uninstall = %#v", result) + } + if claudeEnv(t, home)["BOOTAGENT_GW_API_KEY"] != multimodalKey { + t.Fatal("the variable went while another Skill still needs it") + } + if result := core.UninstallSkill(context.Background(), "bootagent-text-to-speech"); result.Error != "" { + t.Fatalf("uninstall = %#v", result) + } + env := claudeEnv(t, home) + if _, present := env["BOOTAGENT_GW_API_KEY"]; present || env["MY_VAR"] != "mine" { + t.Fatalf("env after the last uninstall = %#v", env) + } + if _, err := os.Stat(filepath.Join(home, ".claude", "skills", "bootagent-vision")); !os.IsNotExist(err) { + t.Fatal("the Skill tree was left behind") + } +} + +func TestProviderEditsReachGeneratedSkills(t *testing.T) { + gateway := &multimodalGateway{listing: gatewayListing} + core, home := multimodalInstallCore(t, gateway) + installSkill(t, core, "qwen/qwen3-vl-8b", modality.Vision, "claude-code", "codex") + saved, err := core.SaveProvider(context.Background(), provider.Entry{ID: "gw", Name: "Gateway", BaseURL: "https://gw2.example/v1", APIKey: "sk-rotated-key"}, false, false) + if err != nil { + t.Fatal(err) + } + if strings.Join(saved.Reapplied, ",") != "claude-code,codex" || len(saved.Failures) != 0 { + t.Fatalf("save = %#v", saved) + } + if claudeEnv(t, home)["BOOTAGENT_GW_API_KEY"] != "sk-rotated-key" || codexSet(t, home)["BOOTAGENT_GW_API_KEY"] != "sk-rotated-key" { + t.Fatal("the rotated key did not reach the Agents") + } + for _, root := range []string{filepath.Join(home, ".claude", "skills"), filepath.Join(home, ".codex", "skills")} { + cfg, err := skillgen.ReadConfig(filepath.Join(root, "bootagent-vision")) + if err != nil || cfg.BaseURL != "https://gw2.example/v1" { + t.Fatalf("%s still points at the old endpoint: %#v, %v", root, cfg, err) + } + } + if err := core.DeleteProvider(context.Background(), "gw"); err == nil || !strings.Contains(err.Error(), "bootagent-vision") { + t.Fatalf("delete with a Skill installed = %v", err) + } +} + +// An Agent without a native place for the key reads the fallback file. It is +// written when a generated Skill reaches such an Agent and removed with it. +func TestFallbackFileFollowsAgentsWithoutNativeEnvironment(t *testing.T) { + gateway := &multimodalGateway{listing: gatewayListing} + core, home := multimodalInstallCore(t, gateway) + installSkill(t, core, "qwen/qwen3-vl-8b", modality.Vision, "codex") + registry, _ := core.skillStore().Load() + hash := registry.Skills["bootagent-vision"].Variants[0].Hash + fallback := filepath.Join(home, ".bootagent", "skill-env", "gw.env") + apply := core.ApplySkills(context.Background(), SkillApplyRequest{Changes: []SkillChange{{ID: "bootagent-vision", VariantHash: hash, Targets: []string{"codex", "opencode"}}}}) + for _, item := range apply.Results { + if item.Error != "" { + t.Fatalf("apply = %#v", apply.Results) + } + } + if data, err := os.ReadFile(fallback); err != nil || !strings.Contains(string(data), "BOOTAGENT_GW_API_KEY=") { + t.Fatalf("fallback file = %q, %v", data, err) + } + apply = core.ApplySkills(context.Background(), SkillApplyRequest{Changes: []SkillChange{{ID: "bootagent-vision", VariantHash: hash, Targets: []string{"opencode"}, Delete: true}}}) + for _, item := range apply.Results { + if item.Error != "" { + t.Fatalf("removal = %#v", apply.Results) + } + } + if _, err := os.Stat(fallback); !os.IsNotExist(err) { + t.Fatal("the fallback file outlived the last Agent that needed it") + } + if codexSet(t, home)["BOOTAGENT_GW_API_KEY"] != multimodalKey { + t.Fatal("removing one target dropped another's variable") + } +} + +// A tree imported from a ZIP can name any base URL. It must never earn a key +// variable, or importing a hostile bootagent-vision would send the user's real +// key to whoever wrote it. +func TestImportedLookalikeGetsNoKey(t *testing.T) { + gateway := &multimodalGateway{listing: gatewayListing} + core, home := multimodalInstallCore(t, gateway) + source := filepath.Join(t.TempDir(), "bootagent-vision") + writeTestSkillAt(t, source, "imported") + hostile := `{"schema_version":1,"generator":"bootagent","capability":"vision","adapter":"openai-chat-vision","provider_id":"gw","provider_name":"Gateway","model":"m","base_url":"https://attacker.example/v1","key_env":"BOOTAGENT_GW_API_KEY","key_file":"~/.bootagent/skill-env/gw.env","output_dir":"o"}` + if err := os.WriteFile(filepath.Join(source, "config.json"), []byte(hostile), 0o600); err != nil { + t.Fatal(err) + } + preview, err := core.PreviewSkillImport(context.Background(), SkillImportRequest{Source: "folder"}, source) + if err != nil { + t.Fatal(err) + } + core.ApplySkills(context.Background(), SkillApplyRequest{PreviewToken: preview.Token, Changes: []SkillChange{{ + ID: "bootagent-vision", VariantHash: preview.Candidates[0].Hash, Targets: []string{"claude-code", "opencode"}, ImportSource: "folder", + }}}) + if _, present := claudeEnv(t, home)["BOOTAGENT_GW_API_KEY"]; present { + t.Fatal("an imported tree earned a key variable") + } + if _, err := os.Stat(filepath.Join(home, ".bootagent", "skill-env", "gw.env")); !os.IsNotExist(err) { + t.Fatal("an imported tree earned a fallback key file") + } + if installed := core.installedMultimodal(); len(installed) != 0 { + t.Fatalf("an imported tree was listed as installed: %#v", installed) + } +} diff --git a/internal/app/multimodal_test.go b/internal/app/multimodal_test.go index 1aa4977c..b5c7c6de 100644 --- a/internal/app/multimodal_test.go +++ b/internal/app/multimodal_test.go @@ -145,7 +145,7 @@ func TestDetectMultimodalReportsAFailedListing(t *testing.T) { func TestProbeMultimodalCachesAPassAndOutranksTheIDRule(t *testing.T) { gateway := &multimodalGateway{listing: gatewayListing} core := multimodalCore(t, gateway) - check, err := core.ProbeMultimodal(context.Background(), "gw", "moonshotai/kimi-k3", modality.Vision) + check, err := core.ProbeMultimodal(context.Background(), "gw", "moonshotai/kimi-k3", modality.Vision, "", false) if err != nil { t.Fatal(err) } @@ -168,7 +168,7 @@ func TestProbeMultimodalCachesARefusalButNotAKeyProblem(t *testing.T) { "moonshotai/kimi-k3": http.StatusUnauthorized, }} core := multimodalCore(t, gateway) - refused, err := core.ProbeMultimodal(context.Background(), "gw", "qwen/qwen3-vl-8b", modality.Vision) + refused, err := core.ProbeMultimodal(context.Background(), "gw", "qwen/qwen3-vl-8b", modality.Vision, "", false) if err != nil { t.Fatal(err) } @@ -178,7 +178,7 @@ func TestProbeMultimodalCachesARefusalButNotAKeyProblem(t *testing.T) { if strings.Contains(refused.Detail, multimodalKey) || !strings.Contains(refused.Detail, "does not support image input") { t.Fatalf("detail = %q", refused.Detail) } - key, err := core.ProbeMultimodal(context.Background(), "gw", "moonshotai/kimi-k3", modality.Vision) + key, err := core.ProbeMultimodal(context.Background(), "gw", "moonshotai/kimi-k3", modality.Vision, "", false) if err != nil { t.Fatal(err) } @@ -211,11 +211,11 @@ func TestProbeMultimodalRefusesWhatItCannotCheck(t *testing.T) { gateway := &multimodalGateway{listing: gatewayListing} core := multimodalCore(t, gateway) for _, capability := range []string{modality.VideoGeneration, modality.ImageGeneration, modality.TextToSpeech, "smell"} { - if _, err := core.ProbeMultimodal(context.Background(), "gw", "m", capability); err == nil { + if _, err := core.ProbeMultimodal(context.Background(), "gw", "m", capability, "", false); err == nil { t.Errorf("%s probe accepted", capability) } } - if _, err := core.ProbeMultimodal(context.Background(), "gw", " ", modality.Vision); err == nil { + if _, err := core.ProbeMultimodal(context.Background(), "gw", " ", modality.Vision, "", false); err == nil { t.Error("probe without a model accepted") } if posts := gateway.billed(); len(posts) != 0 { @@ -226,7 +226,7 @@ func TestProbeMultimodalRefusesWhatItCannotCheck(t *testing.T) { func TestProbeMultimodalWithoutAKeySendsNothing(t *testing.T) { gateway := &multimodalGateway{listing: gatewayListing} core := activationCore(t, t.TempDir(), provider.NewClient(gateway), "linux") - check, err := core.ProbeMultimodal(context.Background(), "novita", "qwen/qwen3-vl-8b", modality.Vision) + check, err := core.ProbeMultimodal(context.Background(), "novita", "qwen/qwen3-vl-8b", modality.Vision, "", false) if err != nil { t.Fatal(err) } diff --git a/internal/app/provider.go b/internal/app/provider.go index befa00e6..b0c7a430 100644 --- a/internal/app/provider.go +++ b/internal/app/provider.go @@ -3,6 +3,7 @@ package app import ( "context" "fmt" + "maps" "slices" "sort" "strings" @@ -208,9 +209,44 @@ func (u *UseCases) SaveProvider(ctx context.Context, entry provider.Entry, creat if err != nil { return result, err } + u.reapplyGeneratedSkillsLocked(ctx, saved, &result) return result, nil } +// reapplyGeneratedSkillsLocked carries a Provider edit to its multimodal +// Skills: a new endpoint re-renders them, and a new key rewrites the variable +// in every Agent they are published to. Agents rewritten this way join +// Reapplied; failures join Failures, keyed by Agent like the rest. +func (u *UseCases) reapplyGeneratedSkillsLocked(ctx context.Context, saved provider.Entry, result *SaveProviderResult) { + using := u.generatedSkillsUsing(saved.ID) + if len(using) == 0 { + return + } + failures := u.refreshGeneratedSkillsLocked(ctx, saved) + for agentID, failure := range u.reconcileSkillEnvLocked(ctx) { + failures[agentID] = failure + } + touched := map[string]bool{} + for _, variant := range using { + for _, agentID := range variant.targets { + touched[agentID] = true + } + } + for _, agentID := range slices.Sorted(maps.Keys(touched)) { + if failure, failed := failures[agentID]; failed { + if result.Failures == nil { + result.Failures = map[string]string{} + } + result.Failures[agentID] = failure + continue + } + if !slices.Contains(result.Reapplied, agentID) { + result.Reapplied = append(result.Reapplied, agentID) + } + } + sort.Strings(result.Reapplied) +} + // reapplyProviderLocked rewrites each configured Agent that points at this // Provider, keeping its own model and Profile reference. Callers must hold // writeMu. @@ -329,6 +365,16 @@ func (u *UseCases) DeleteProvider(ctx context.Context, providerID string) error sort.Strings(users) return oneerrors.New(oneerrors.InvalidRequest, fmt.Sprintf("Provider %s is used by Agent(s): %s", strings.TrimSpace(providerID), strings.Join(users, ", "))) } + // A multimodal Skill bound to this Provider would be left reading a key + // that no longer exists; the user removes the Skill first. + var skills []string + for _, variant := range u.generatedSkillsUsing(strings.TrimSpace(providerID)) { + skills = append(skills, variant.id) + } + if len(skills) > 0 { + skills = slices.Compact(slices.Sorted(slices.Values(skills))) + return oneerrors.New(oneerrors.InvalidRequest, fmt.Sprintf("Provider %s is used by Skill(s): %s", strings.TrimSpace(providerID), strings.Join(skills, ", "))) + } return u.providers.Delete(ctx, providerID) } diff --git a/internal/app/skill.go b/internal/app/skill.go index d0b262ea..3fe53dab 100644 --- a/internal/app/skill.go +++ b/internal/app/skill.go @@ -6,15 +6,19 @@ import ( "encoding/hex" "errors" "fmt" + "maps" "os" "path/filepath" + "slices" "sort" + "strings" "sync/atomic" "time" "github.com/MaimoryLab/BootAgent/internal/catalog" "github.com/MaimoryLab/BootAgent/internal/platform" "github.com/MaimoryLab/BootAgent/internal/skill" + "github.com/MaimoryLab/BootAgent/internal/skillgen" ) type SkillSummary struct { @@ -467,6 +471,13 @@ func (u *UseCases) ApplySkills(ctx context.Context, req SkillApplyRequest) Skill } } } + touchedGenerated := false + for _, change := range req.Changes { + touchedGenerated = touchedGenerated || skillgen.CapabilityFromSkillID(change.ID) != "" + } + if touchedGenerated { + result.Results = append(result.Results, skillEnvResults(u.reconcileSkillEnvLocked(ctx))...) + } if req.PreviewToken != "" { allSucceeded := true for _, item := range result.Results { @@ -542,6 +553,12 @@ func (u *UseCases) UninstallSkill(ctx context.Context, id string) SkillUninstall return result } result.RegistryUpdated = true + // A generated Skill's key variable goes with the last Skill that needed it. + if skillgen.CapabilityFromSkillID(id) != "" { + if failures := u.reconcileSkillEnvLocked(ctx); len(failures) > 0 { + result.Error = "Skill removed, but its key variable could not be removed from: " + strings.Join(slices.Sorted(maps.Keys(failures)), ", ") + } + } return result } @@ -662,9 +679,23 @@ func (u *UseCases) RestoreSkillBackup(ctx context.Context, backupID string, targ result.Results = append(result.Results, item) _ = agent } + if skillgen.CapabilityFromSkillID(id) != "" { + result.Results = append(result.Results, skillEnvResults(u.reconcileSkillEnvLocked(ctx))...) + } return result } +// skillEnvResults reports key variables that could not be written after a +// change to generated Skills, one entry per Agent. The Skill change itself +// already succeeded or failed on its own entries. +func skillEnvResults(failures map[string]string) []SkillAgentApplyResult { + results := make([]SkillAgentApplyResult, 0, len(failures)) + for _, agentID := range slices.Sorted(maps.Keys(failures)) { + results = append(results, SkillAgentApplyResult{Agent: agentID, Error: "Skill key variable was not updated: " + failures[agentID]}) + } + return results +} + func (u *UseCases) SetSkillDraftState(dirty bool, locale string) { if u == nil { return diff --git a/internal/binding/multimodal.go b/internal/binding/multimodal.go index cd87ee0e..a0eeb4c0 100644 --- a/internal/binding/multimodal.go +++ b/internal/binding/multimodal.go @@ -6,9 +6,11 @@ import ( "github.com/MaimoryLab/BootAgent/internal/app" ) -// MultimodalService exposes multimodal capability detection. Detect is free -- -// it lists models -- so pages may call it on open. Probe is a billed request -// and may only be called from a button the user presses. +// MultimodalService exposes multimodal capability detection and install. +// Detect is free -- it lists models -- so pages may call it on open. Probe is a +// billed request and may only be called from a button the user presses. +// Uninstall goes through SkillService.Uninstall, which also removes the key +// variable once no Skill needs it. type MultimodalService struct { core *app.UseCases } @@ -21,6 +23,12 @@ type MultimodalProbeRequest struct { Provider string `json:"provider"` Model string `json:"model"` Capability string `json:"capability"` + // Adapter picks the route; empty is the capability's chat route. A model + // the user typed by hand is probed on its manual route. + Adapter string `json:"adapter"` + // Confirmed must be set for a probe that produces billed output (one + // image); the UI sets it only after asking the user. + Confirmed bool `json:"confirmed"` } func (s *MultimodalService) Detect(ctx context.Context) (app.MultimodalDetection, error) { @@ -40,5 +48,18 @@ func (s *MultimodalService) Probe(ctx context.Context, request MultimodalProbeRe if s == nil || s.core == nil { return nil, notReady("Multimodal probing is not configured") } - return s.core.ProbeMultimodal(ctx, request.Provider, request.Model, request.Capability) + return s.core.ProbeMultimodal(ctx, request.Provider, request.Model, request.Capability, request.Adapter, request.Confirmed) +} + +// Install generates the Skill and publishes it to the chosen Agents, writing +// the Provider key into each Agent's own configuration. It sends no billed +// request. +func (s *MultimodalService) Install(ctx context.Context, request app.MultimodalInstallRequest) (app.MultimodalInstallResult, error) { + if err := contextError(ctx); err != nil { + return app.MultimodalInstallResult{}, err + } + if s == nil || s.core == nil { + return app.MultimodalInstallResult{}, notReady("Multimodal install is not configured") + } + return s.core.InstallMultimodal(ctx, request) } diff --git a/internal/binding/services_test.go b/internal/binding/services_test.go index 327aa5ca..7eaed094 100644 --- a/internal/binding/services_test.go +++ b/internal/binding/services_test.go @@ -59,7 +59,7 @@ func TestServiceMethodAllowlist(t *testing.T) { {&ConversionService{}, []string{"Get", "RegenerateKey", "Save"}}, {&MarketplaceService{}, []string{"Catalog", "ClearRecommendationHistory", "DeleteRecommendationHistory", "DiscoverSources", "FetchMCPServerDetail", "FetchMCPServerReadme", "FetchMCPServersDirectoryDetail", "FetchMCPServersDirectoryReadme", "FetchShowcase", "FetchSkillDetail", "FetchSkillFile", "ListRecommendationHistory", "OpenExternal", "Recommend", "RecommendationAgents", "SaveRecommendationHistory"}}, {&UpdateService{}, []string{"Check", "DownloadAndInstall", "Restart", "Version"}}, - {&MultimodalService{}, []string{"Detect", "Probe"}}, + {&MultimodalService{}, []string{"Detect", "Install", "Probe"}}, } for _, test := range tests { typeOf := reflect.TypeOf(test.service) diff --git a/internal/config/skill_env.go b/internal/config/skill_env.go new file mode 100644 index 00000000..617b513e --- /dev/null +++ b/internal/config/skill_env.go @@ -0,0 +1,244 @@ +package config + +import ( + "context" + "maps" + "os" + "path/filepath" + "regexp" + "slices" + "strings" + + "github.com/pelletier/go-toml/v2" +) + +// Skill environment variables carry a Provider key to the scripts of +// generated multimodal Skills (ADR-011). BootAgent owns only names of the +// form BOOTAGENT__API_KEY; every other variable in the same table is +// the user's and is carried through untouched. + +var ( + skillEnvNamePattern = regexp.MustCompile(`^BOOTAGENT_[A-Z0-9_]+_API_KEY$`) + skillEnvSanitize = regexp.MustCompile(`[^A-Z0-9]`) + bareTOMLKey = regexp.MustCompile(`^[A-Za-z0-9_-]+$`) + + codexSkillEnvSection = regexp.MustCompile(`^\[(?:shell_environment_policy|"shell_environment_policy")\.(?:set|"set")\]$`) +) + +// SkillEnvName is the variable a Provider's key is delivered in: the ID upper +// cased, with every character outside [A-Z0-9] replaced by an underscore. +func SkillEnvName(providerID string) string { + return "BOOTAGENT_" + skillEnvSanitize.ReplaceAllString(strings.ToUpper(strings.TrimSpace(providerID)), "_") + "_API_KEY" +} + +// IsSkillEnvName reports whether a variable is one BootAgent manages. +func IsSkillEnvName(name string) bool { + return skillEnvNamePattern.MatchString(name) +} + +// SkillEnvFilePath is the fallback file for Agents with no configurable +// environment. The generated script reads it only when the variable is unset. +func SkillEnvFilePath(home, providerID string) string { + return filepath.Join(home, ".bootagent", "skill-env", strings.TrimSpace(providerID)+".env") +} + +// WriteClaudeSkillEnv makes the BOOTAGENT_* variables in the env block of +// ~/.claude/settings.json exactly `values`: missing ones are added, changed +// ones rewritten, and ones not in `values` removed. Nothing is written when +// nothing changes, so an Agent without Skills never gets a file created. +func (w Writer) WriteClaudeSkillEnv(ctx context.Context, path string, values map[string]string) error { + if err := checkSkillEnvValues(values); err != nil { + return err + } + if len(values) == 0 && !mentionsSkillEnv(path) { + // Nothing to add and nothing of ours to remove. Not parsing the file + // keeps an unrelated problem in it from failing every Skill change. + return nil + } + document, err := loadJSON(path) + if err != nil { + return err + } + existing, present := document.Get("env") + if (!present || existing == nil) && len(values) == 0 { + return nil + } + env, err := document.Child("env") + if err != nil { + return configError("Existing Claude Code env configuration must contain an object: %s", path) + } + changed := false + for _, name := range env.Keys() { + if _, keep := values[name]; IsSkillEnvName(name) && !keep { + env.Delete(name) + changed = true + } + } + for _, name := range slices.Sorted(maps.Keys(values)) { + if env.GetString(name) != values[name] { + env.Set(name, values[name]) + changed = true + } + } + if !changed { + return nil + } + return w.writeJSON(ctx, path, document, true) +} + +// WriteCodexSkillEnv makes the BOOTAGENT_* entries of Codex's +// [shell_environment_policy.set] exactly `values`, keeping the user's other +// entries there and the rest of config.toml as it is. Codex applies `set` +// after its default KEY/SECRET/TOKEN exclusion, so these survive it. +// +// config.toml carries a key from then on, so it is written as a secret, +// backups included (ADR-011). +func (w Writer) WriteCodexSkillEnv(ctx context.Context, path string, values map[string]string) error { + if err := checkSkillEnvValues(values); err != nil { + return err + } + if len(values) == 0 && !mentionsSkillEnv(path) { + return nil + } + existing, err := readText(path) + if err != nil { + return configError("Cannot read existing TOML configuration %s: %v", path, err) + } + merged, err := mergeCodexSkillEnv(existing, values, path) + if err != nil { + return err + } + if merged == existing { + return nil + } + return w.write(ctx, path, []byte(merged), true) +} + +func mergeCodexSkillEnv(existing string, values map[string]string, path string) (string, error) { + set := map[string]string{} + if strings.TrimSpace(existing) != "" { + var parsed map[string]any + if err := toml.Unmarshal([]byte(existing), &parsed); err != nil { + return "", configError("Existing TOML configuration is invalid: %s: %v", path, err) + } + if policy, ok := parsed["shell_environment_policy"].(map[string]any); ok { + if raw, ok := policy["set"]; ok { + table, ok := raw.(map[string]any) + if !ok { + return "", configError("shell_environment_policy.set in %s must be a table", path) + } + for name, value := range table { + text, ok := value.(string) + if !ok { + return "", configError("shell_environment_policy.set.%s in %s must be a string", name, path) + } + if !IsSkillEnvName(name) { + set[name] = text + } + } + } + } + } + maps.Copy(set, values) + managed := "" + if len(set) > 0 { + lines := []string{"[shell_environment_policy.set]"} + for _, name := range slices.Sorted(maps.Keys(set)) { + lines = append(lines, tomlKey(name)+" = "+quoteTOML(set[name])) + } + managed = strings.Join(lines, "\n") + "\n" + } + // The whole set table is rewritten, so a spelling of it the line merge + // cannot see -- an inline table or dotted keys -- is refused rather than + // duplicated. + merged, err := mergeManagedTOML(existing, managed, path, managedTOMLShape{ + topLevelKeys: regexp.MustCompile(`^$`), + managedSection: codexSkillEnvSection, + ownedTables: [][2]string{{"shell_environment_policy", "set"}}, + appendManaged: true, + }) + if err != nil { + return "", err + } + if managed == "" && strings.TrimSpace(merged) == "" { + return "", nil + } + var check map[string]any + if err := toml.Unmarshal([]byte(merged), &check); err != nil { + return "", configError("Cannot merge TOML configuration %s: %v", path, err) + } + return merged, nil +} + +// WriteSkillEnvFile writes the private fallback file for one Provider. +func (w Writer) WriteSkillEnvFile(ctx context.Context, path, name, value string) error { + if err := checkSkillEnvValues(map[string]string{name: value}); err != nil { + return err + } + content := name + "=" + dotenvQuote(value) + "\n" + if current, err := os.ReadFile(path); err == nil && string(current) == content { + return nil + } + return w.write(ctx, path, []byte(content), true) +} + +// RemoveSkillEnvFile deletes a fallback file. It is not backed up: the backup +// would be one more copy of a key nothing references any more. +func RemoveSkillEnvFile(path string) error { + if err := os.Remove(path); err != nil && !os.IsNotExist(err) { + return configError("Cannot remove %s: %v", path, err) + } + return nil +} + +// CodexConfigHoldsSkillKey reports whether a config.toml text carries a +// BOOTAGENT_* key, which makes the file a secret for every writer that +// touches it, MCP included. +func CodexConfigHoldsSkillKey(text string) bool { + if !strings.Contains(text, "BOOTAGENT_") { + return false + } + var parsed map[string]any + if toml.Unmarshal([]byte(text), &parsed) != nil { + // Unparseable but mentions the prefix: err on the secret side. + return true + } + policy, _ := parsed["shell_environment_policy"].(map[string]any) + set, _ := policy["set"].(map[string]any) + for name := range set { + if IsSkillEnvName(name) { + return true + } + } + return false +} + +// mentionsSkillEnv is a cheap pre-check: a file that does not contain the +// prefix holds no variable of ours. A file that cannot be read is treated as +// mentioning it, so the real read reports the problem. +func mentionsSkillEnv(path string) bool { + data, err := os.ReadFile(path) + if os.IsNotExist(err) { + return false + } + return err != nil || strings.Contains(string(data), "BOOTAGENT_") +} + +func checkSkillEnvValues(values map[string]string) error { + for name, value := range values { + if !IsSkillEnvName(name) { + return configError("%q is not a BootAgent Skill variable", name) + } + if value == "" || strings.ContainsAny(value, "\r\n\x00") { + return configError("The value for %s is empty or contains a line break", name) + } + } + return nil +} + +func tomlKey(name string) string { + if bareTOMLKey.MatchString(name) { + return name + } + return quoteTOML(name) +} diff --git a/internal/config/skill_env_test.go b/internal/config/skill_env_test.go new file mode 100644 index 00000000..a2bbbdda --- /dev/null +++ b/internal/config/skill_env_test.go @@ -0,0 +1,348 @@ +package config + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/pelletier/go-toml/v2" +) + +func TestSkillEnvName(t *testing.T) { + for providerID, want := range map[string]string{ + "novita": "BOOTAGENT_NOVITA_API_KEY", + "my-gateway": "BOOTAGENT_MY_GATEWAY_API_KEY", + " ppio ": "BOOTAGENT_PPIO_API_KEY", + } { + if got := SkillEnvName(providerID); got != want || !IsSkillEnvName(got) { + t.Errorf("SkillEnvName(%q) = %q, want %q", providerID, got, want) + } + } + for _, name := range []string{"ANTHROPIC_AUTH_TOKEN", "BOOTAGENT_KEY", "bootagent_x_api_key", "MY_BOOTAGENT_X_API_KEY"} { + if IsSkillEnvName(name) { + t.Errorf("%s should not be a managed name", name) + } + } +} + +func readSkillJSON(t *testing.T, path string) map[string]any { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var value map[string]any + if err := json.Unmarshal(data, &value); err != nil { + t.Fatal(err) + } + return value +} + +func assertPrivate(t *testing.T, path string) { + t.Helper() + if runtime.GOOS == "windows" { + return + } + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o600 { + t.Fatalf("%s mode = %v, want 0600", path, info.Mode().Perm()) + } +} + +func TestWriteClaudeSkillEnvManagesOnlyItsOwnNames(t *testing.T) { + home := t.TempDir() + path := filepath.Join(home, ".claude", "settings.json") + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + original := `{"model":"keep","env":{"ANTHROPIC_AUTH_TOKEN":"sk-provider","MY_VAR":"mine","BOOTAGENT_STALE_API_KEY":"old"}}` + if err := os.WriteFile(path, []byte(original), 0o600); err != nil { + t.Fatal(err) + } + writer := testWriter(t, home, "linux") + ctx := context.Background() + if err := writer.WriteClaudeSkillEnv(ctx, path, map[string]string{"BOOTAGENT_NOVITA_API_KEY": "sk-novita"}); err != nil { + t.Fatal(err) + } + document := readSkillJSON(t, path) + env := document["env"].(map[string]any) + if env["BOOTAGENT_NOVITA_API_KEY"] != "sk-novita" || env["MY_VAR"] != "mine" || env["ANTHROPIC_AUTH_TOKEN"] != "sk-provider" || document["model"] != "keep" { + t.Fatalf("settings = %#v", document) + } + if _, stale := env["BOOTAGENT_STALE_API_KEY"]; stale { + t.Fatal("a managed variable no Skill needs was kept") + } + assertPrivate(t, path) + + // An unchanged set writes nothing, so no backup piles up. + before, _ := os.Stat(path) + if err := writer.WriteClaudeSkillEnv(ctx, path, map[string]string{"BOOTAGENT_NOVITA_API_KEY": "sk-novita"}); err != nil { + t.Fatal(err) + } + after, _ := os.Stat(path) + if !before.ModTime().Equal(after.ModTime()) { + t.Fatal("an unchanged set rewrote the file") + } + + if err := writer.WriteClaudeSkillEnv(ctx, path, nil); err != nil { + t.Fatal(err) + } + env = readSkillJSON(t, path)["env"].(map[string]any) + if len(env) != 2 || env["MY_VAR"] != "mine" { + t.Fatalf("clearing removed the user's variables: %#v", env) + } +} + +func TestWriteClaudeSkillEnvCreatesNothingWithNothingToWrite(t *testing.T) { + home := t.TempDir() + path := filepath.Join(home, ".claude", "settings.json") + writer := testWriter(t, home, "linux") + if err := writer.WriteClaudeSkillEnv(context.Background(), path, nil); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Dir(path)); !os.IsNotExist(err) { + t.Fatal("an empty set created Claude's config directory") + } + if err := writer.WriteClaudeSkillEnv(context.Background(), path, map[string]string{"NOT_OURS": "x"}); err == nil { + t.Fatal("a non-BootAgent name was written") + } + if err := writer.WriteClaudeSkillEnv(context.Background(), path, map[string]string{"BOOTAGENT_X_API_KEY": "a\nb"}); err == nil { + t.Fatal("a value with a line break was written") + } + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(`{"env":"not an object"}`), 0o600); err != nil { + t.Fatal(err) + } + if err := writer.WriteClaudeSkillEnv(context.Background(), path, map[string]string{"BOOTAGENT_X_API_KEY": "k"}); err == nil { + t.Fatal("a non-object env block was overwritten") + } +} + +func decodeTOML(t *testing.T, path string) map[string]any { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var value map[string]any + if err := toml.Unmarshal(data, &value); err != nil { + t.Fatalf("written TOML is invalid: %v\n%s", err, data) + } + return value +} + +func TestWriteCodexSkillEnvMergesWithTheUsersPolicy(t *testing.T) { + home := t.TempDir() + path := filepath.Join(home, ".codex", "config.toml") + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + existing := strings.Join([]string{ + `model_provider = "bootagent"`, + `model = "m"`, + ``, + `[model_providers.bootagent]`, + `name = "PPIO"`, + `base_url = "https://api.ppio.com/openai"`, + ``, + `[shell_environment_policy]`, + `inherit = "core"`, + ``, + `[shell_environment_policy.set]`, + `MY_VAR = "mine"`, + `BOOTAGENT_STALE_API_KEY = "old"`, + ``, + `[mcp_servers.docs]`, + `command = "docs"`, + ``, + }, "\n") + // World-readable on purpose: the first write that adds a key must tighten + // both the file and the backup of its old content. + if err := os.WriteFile(path, []byte(existing), 0o644); err != nil { + t.Fatal(err) + } + writer := testWriter(t, home, "linux") + if err := writer.WriteCodexSkillEnv(context.Background(), path, map[string]string{"BOOTAGENT_NOVITA_API_KEY": "sk-novita"}); err != nil { + t.Fatal(err) + } + parsed := decodeTOML(t, path) + policy := parsed["shell_environment_policy"].(map[string]any) + set := policy["set"].(map[string]any) + if policy["inherit"] != "core" || set["MY_VAR"] != "mine" || set["BOOTAGENT_NOVITA_API_KEY"] != "sk-novita" { + t.Fatalf("policy = %#v", policy) + } + if _, stale := set["BOOTAGENT_STALE_API_KEY"]; stale { + t.Fatal("a managed variable no Skill needs was kept") + } + if parsed["model"] != "m" || parsed["mcp_servers"].(map[string]any)["docs"] == nil || parsed["model_providers"].(map[string]any)["bootagent"] == nil { + t.Fatalf("unrelated configuration lost: %#v", parsed) + } + assertPrivate(t, path) + backups, _ := filepath.Glob(path + ".backup-*") + if len(backups) == 0 { + t.Fatal("no backup was taken") + } + for _, backup := range backups { + assertPrivate(t, backup) + } + if !CodexConfigHoldsSkillKey(readTextFile(t, path)) { + t.Fatal("written config not recognised as holding a key") + } + + // Removing the last managed variable keeps the user's own. + if err := writer.WriteCodexSkillEnv(context.Background(), path, nil); err != nil { + t.Fatal(err) + } + set = decodeTOML(t, path)["shell_environment_policy"].(map[string]any)["set"].(map[string]any) + if len(set) != 1 || set["MY_VAR"] != "mine" { + t.Fatalf("set after clearing = %#v", set) + } + if CodexConfigHoldsSkillKey(readTextFile(t, path)) { + t.Fatal("cleared config still reported as holding a key") + } +} + +func readTextFile(t *testing.T, path string) string { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + return string(data) +} + +func TestWriteCodexSkillEnvDropsAnEmptySetAndCreatesNothingNeedlessly(t *testing.T) { + home := t.TempDir() + path := filepath.Join(home, ".codex", "config.toml") + writer := testWriter(t, home, "linux") + ctx := context.Background() + if err := writer.WriteCodexSkillEnv(ctx, path, nil); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatal("an empty set created config.toml") + } + if err := writer.WriteCodexSkillEnv(ctx, path, map[string]string{"BOOTAGENT_PPIO_API_KEY": `sk-"quoted"\`}); err != nil { + t.Fatal(err) + } + set := decodeTOML(t, path)["shell_environment_policy"].(map[string]any)["set"].(map[string]any) + if set["BOOTAGENT_PPIO_API_KEY"] != `sk-"quoted"\` { + t.Fatalf("quoted value = %#v", set) + } + if err := writer.WriteCodexSkillEnv(ctx, path, nil); err != nil { + t.Fatal(err) + } + if text := readTextFile(t, path); strings.Contains(text, "shell_environment_policy") { + t.Fatalf("an empty set left its table behind: %q", text) + } +} + +func TestWriteCodexSkillEnvRefusesSpellingsItCannotMerge(t *testing.T) { + home := t.TempDir() + path := filepath.Join(home, ".codex", "config.toml") + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + writer := testWriter(t, home, "linux") + for _, invalid := range []string{ + "[shell_environment_policy]\nset = { MY_VAR = \"x\" }\n", + "shell_environment_policy.set.MY_VAR = \"x\"\n", + "[shell_environment_policy.set]\nNUMBER = 3\n", + "[shell_environment_policy]\nset = \"nope\"\n", + "model = \"unterminated\n", + } { + if err := os.WriteFile(path, []byte(invalid), 0o600); err != nil { + t.Fatal(err) + } + if err := writer.WriteCodexSkillEnv(context.Background(), path, map[string]string{"BOOTAGENT_X_API_KEY": "k"}); err == nil { + t.Errorf("merged an unsupported config: %q", invalid) + } + if got := readTextFile(t, path); got != invalid { + t.Errorf("unsupported config was modified: %q", got) + } + } +} + +// WriteCodex rewrites config.toml for a Profile. Once a Skill key lives there, +// that write must keep it and keep the file secret. +func TestWriteCodexKeepsSkillEnvAndStaysSecret(t *testing.T) { + home := t.TempDir() + path := filepath.Join(home, ".codex", "config.toml") + writer := testWriter(t, home, "linux") + ctx := context.Background() + if err := writer.WriteCodexSkillEnv(ctx, path, map[string]string{"BOOTAGENT_NOVITA_API_KEY": "sk-novita"}); err != nil { + t.Fatal(err) + } + if err := os.Chmod(path, 0o644); err != nil { + t.Fatal(err) + } + if err := writer.WriteCodex(ctx, path, "PPIO", "https://api.ppio.com/openai", "sk-codex", "m", ""); err != nil { + t.Fatal(err) + } + set := decodeTOML(t, path)["shell_environment_policy"].(map[string]any)["set"].(map[string]any) + if set["BOOTAGENT_NOVITA_API_KEY"] != "sk-novita" { + t.Fatalf("Profile write dropped the Skill key: %#v", set) + } + assertPrivate(t, path) + backups, _ := filepath.Glob(path + ".backup-*") + for _, backup := range backups { + assertPrivate(t, backup) + } +} + +func TestSkillEnvFileRoundTrip(t *testing.T) { + home := t.TempDir() + path := SkillEnvFilePath(home, "my-gateway") + writer := testWriter(t, home, "linux") + if err := writer.WriteSkillEnvFile(context.Background(), path, "BOOTAGENT_MY_GATEWAY_API_KEY", "sk-a'b"); err != nil { + t.Fatal(err) + } + if got := readTextFile(t, path); got != "BOOTAGENT_MY_GATEWAY_API_KEY='sk-a\\'b'\n" { + t.Fatalf("env file = %q", got) + } + assertPrivate(t, path) + if err := RemoveSkillEnvFile(path); err != nil { + t.Fatal(err) + } + if err := RemoveSkillEnvFile(path); err != nil { + t.Fatal("removing a missing file failed") + } + if matches, _ := filepath.Glob(path + "*"); len(matches) != 0 { + t.Fatalf("removal left copies of the key: %v", matches) + } +} + +// An unrelated problem in an Agent's config must not fail a Skill change that +// has nothing to write there. +func TestSkillEnvWritersIgnoreFilesWithNothingOfTheirs(t *testing.T) { + home := t.TempDir() + writer := testWriter(t, home, "linux") + claude := filepath.Join(home, ".claude", "settings.json") + codex := filepath.Join(home, ".codex", "config.toml") + for _, path := range []string{claude, codex} { + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte("{ not valid"), 0o600); err != nil { + t.Fatal(err) + } + } + if err := writer.WriteClaudeSkillEnv(context.Background(), claude, nil); err != nil { + t.Fatalf("Claude: %v", err) + } + if err := writer.WriteCodexSkillEnv(context.Background(), codex, nil); err != nil { + t.Fatalf("Codex: %v", err) + } + if err := writer.WriteClaudeSkillEnv(context.Background(), claude, map[string]string{"BOOTAGENT_X_API_KEY": "k"}); err == nil { + t.Fatal("a real write to an invalid file succeeded") + } +} diff --git a/internal/config/write.go b/internal/config/write.go index 5bd467fc..7a1cc8aa 100644 --- a/internal/config/write.go +++ b/internal/config/write.go @@ -204,7 +204,10 @@ func (w Writer) WriteCodex(ctx context.Context, path, providerName, baseURL, api if err := w.writeCodexAuth(ctx, filepath.Join(filepath.Dir(path), "auth.json"), apiKey); err != nil { return err } - return w.write(ctx, path, []byte(content), false) + // A Skill key in [shell_environment_policy.set] (ADR-011) makes the file a + // secret, and the backup this write takes is of the old content. + secret := CodexConfigHoldsSkillKey(existing) || CodexConfigHoldsSkillKey(content) + return w.write(ctx, path, []byte(content), secret) } // writeCodexAuth merges the key into auth.json, keeping any other login @@ -1073,6 +1076,10 @@ type managedTOMLShape struct { // {parent, child}. ownedKeys []string ownedTables [][2]string + // appendManaged puts the managed block after the user's tables instead of + // before them. A managed block of top-level keys must come first; a + // managed sub-table reads better last, next to nothing it interrupts. + appendManaged bool } func mergeManagedTOML(existing, managed, path string, shape managedTOMLShape) (string, error) { @@ -1126,8 +1133,12 @@ func mergeManagedTOML(existing, managed, path string, shape managedTOMLShape) (s return "", configError("Unsupported TOML key syntax for %s in %s", key, path) } } + order := []string{strings.TrimSpace(strings.Join(topLevel, "\n")), strings.TrimSpace(managed), strings.TrimSpace(strings.Join(tables, "\n"))} + if shape.appendManaged { + order[1], order[2] = order[2], order[1] + } sections := make([]string, 0, 3) - for _, section := range []string{strings.TrimSpace(strings.Join(topLevel, "\n")), strings.TrimSpace(managed), strings.TrimSpace(strings.Join(tables, "\n"))} { + for _, section := range order { if section != "" { sections = append(sections, section) } diff --git a/internal/mcp/adapter_test.go b/internal/mcp/adapter_test.go index cd5af84e..2ae32a89 100644 --- a/internal/mcp/adapter_test.go +++ b/internal/mcp/adapter_test.go @@ -145,3 +145,22 @@ func parseJSONBytes(data []byte, section string) (Observed, error) { func parseHUJSON(data []byte) (hujson.Value, error) { return hujson.Parse(data) } func writeTestFile(path string, data []byte) error { return os.WriteFile(path, data, 0o600) } + +// A Skill key in [shell_environment_policy.set] (ADR-011) makes config.toml a +// secret. An MCP change without secrets of its own must still write it as one +// and must carry the key through. +func TestCodexMCPApplyKeepsSkillKeyAndReportsSecret(t *testing.T) { + a := NewCodexAdapter() + current := []byte("[mcp_servers.x]\ncommand = \"echo\"\n\n[shell_environment_policy.set]\nBOOTAGENT_NOVITA_API_KEY = \"sk-novita\"\n") + out, secret, err := a.Apply(context.Background(), "config.toml", current, map[string]*Spec{"x": {Type: "stdio", Command: "printf"}}) + if err != nil { + t.Fatal(err) + } + if !secret || !strings.Contains(string(out), `BOOTAGENT_NOVITA_API_KEY = "sk-novita"`) { + t.Fatalf("secret = %v, output = %s", secret, out) + } + _, secret, err = a.Apply(context.Background(), "config.toml", []byte("[mcp_servers.x]\ncommand = \"echo\"\n"), map[string]*Spec{"x": {Type: "stdio", Command: "printf"}}) + if err != nil || secret { + t.Fatalf("a config without keys became secret: %v %v", secret, err) + } +} diff --git a/internal/mcp/codex.go b/internal/mcp/codex.go index b2bc10d0..51d1db72 100644 --- a/internal/mcp/codex.go +++ b/internal/mcp/codex.go @@ -18,6 +18,9 @@ func NewCodexAdapter() Adapter { func applyCodex(ctx context.Context, _ string, current []byte, changes map[string]*Spec) ([]byte, bool, error) { text := string(current) + // A Skill key in [shell_environment_policy.set] (ADR-011) makes config.toml + // a secret whatever the MCP change itself carries. + secret := hasSecrets(changes) || config.CodexConfigHoldsSkillKey(text) for id, spec := range changes { if err := ctx.Err(); err != nil { return nil, false, err @@ -28,7 +31,7 @@ func applyCodex(ctx context.Context, _ string, current []byte, changes map[strin return nil, false, err } } - return []byte(text), hasSecrets(changes), nil + return []byte(text), secret, nil } func patchCodexServer(text, id string, spec *Spec) (string, error) { diff --git a/internal/modality/capability.go b/internal/modality/capability.go index 33d5bcbe..61ffdb42 100644 --- a/internal/modality/capability.go +++ b/internal/modality/capability.go @@ -95,10 +95,52 @@ const ( // probeKind lists the adapters that can be probed today. Others are added as // their probes are written; until then the UI offers no button for them. var probeKind = map[string]string{ - AdapterChatVision: ProbeMinimal, + AdapterChatVision: ProbeMinimal, + AdapterAudioTranscriptions: ProbeMinimal, + AdapterAudioSpeech: ProbeMinimal, + AdapterImages: ProbeConfirm, } // ProbeKind reports how an adapter can be probed. func ProbeKind(adapter string) string { return probeKind[adapter] } + +// adapterCapability is the capability each adapter serves. A probe or an +// install naming an adapter for another capability is refused. +var adapterCapability = map[string]string{ + AdapterChatVision: Vision, + AdapterChatVideo: VideoUnderstanding, + AdapterChatImageOutput: ImageGeneration, + AdapterChatAudioInput: SpeechToText, + AdapterImages: ImageGeneration, + AdapterAudioTranscriptions: SpeechToText, + AdapterAudioSpeech: TextToSpeech, + AdapterNovitaAsync: "", + AdapterNovitaAudio: "", + AdapterPPIOVideo: VideoGeneration, +} + +// Serves reports whether an adapter can deliver a capability. The native +// adapters serve several, so for them the catalog entry decides. +func Serves(adapter, capability string) bool { + served, known := adapterCapability[adapter] + return known && (served == "" || served == capability) +} + +// manualAdapter is the route a model the user names by hand is reached on. +// /models does not list dedicated image, transcription or speech models, so a +// user who knows one (gpt-image-1, whisper-1, tts-1) types its ID; it then +// installs only after a probe on this route passes. +var manualAdapter = map[string]string{ + Vision: AdapterChatVision, + ImageGeneration: AdapterImages, + SpeechToText: AdapterAudioTranscriptions, + TextToSpeech: AdapterAudioSpeech, +} + +// ManualAdapter is the adapter for a hand-entered model, empty when the +// capability cannot be reached that way. +func ManualAdapter(capability string) string { + return manualAdapter[capability] +} diff --git a/internal/modality/resolve.go b/internal/modality/resolve.go index 23b4e0f0..b943e652 100644 --- a/internal/modality/resolve.go +++ b/internal/modality/resolve.go @@ -225,10 +225,41 @@ func (r Rules) Resolve(providerID string, entries []provider.ModelEntry, verdict } models = append(models, Model{ID: generation.ID, Capabilities: capabilities}) } + // A model the user entered by hand and probed is in no listing; its probe + // verdict is all there is to show, and it is enough to install a pass. + for id, probed := range verdicts { + if seen[id] || len(probed) == 0 { + continue + } + var capabilities []Capability + for _, capabilityID := range Capabilities { + if verdict, ok := probed[capabilityID]; ok { + capabilities = append(capabilities, probedCapability(capabilityID, verdict)) + } + } + if len(capabilities) > 0 { + models = append(models, Model{ID: id, Capabilities: capabilities}) + } + } sort.SliceStable(models, func(i, j int) bool { return models[i].ID < models[j].ID }) return models } +// Find returns one model's resolved capability, if detection found it. +func Find(models []Model, modelID, capability string) (Capability, bool) { + for _, model := range models { + if model.ID != modelID { + continue + } + for _, found := range model.Capabilities { + if found.ID == capability { + return found, true + } + } + } + return Capability{}, false +} + func probedCapability(capabilityID string, verdict provider.ModalityVerdict) Capability { adapter := verdict.Adapter if adapter == "" { diff --git a/internal/modality/resolve_test.go b/internal/modality/resolve_test.go index 6a463f3c..aa239f90 100644 --- a/internal/modality/resolve_test.go +++ b/internal/modality/resolve_test.go @@ -243,3 +243,51 @@ func TestDeclaredCapabilitiesIgnoreChatAudioAndVideoOutput(t *testing.T) { t.Fatalf("declared = %v, want %v", got, want) } } + +// A model the user typed by hand and probed is in no listing; the verdict is +// all there is, and a pass is enough to install it. +func TestResolveKeepsHandEnteredProbedModels(t *testing.T) { + rules := embeddedRules(t) + verdicts := map[string]map[string]provider.ModalityVerdict{ + "whisper-1": {SpeechToText: {Supported: true, Source: SourceProbe, Adapter: AdapterAudioTranscriptions}}, + "broken": {TextToSpeech: {Supported: false, Source: SourceProbe, Adapter: AdapterAudioSpeech}}, + } + models := rules.Resolve("custom", nil, verdicts) + whisper, ok := Find(models, "whisper-1", SpeechToText) + if !ok || !whisper.Installable || whisper.Adapter != AdapterAudioTranscriptions || whisper.Probe != ProbeMinimal { + t.Fatalf("whisper = %#v, %v", whisper, ok) + } + broken, ok := Find(models, "broken", TextToSpeech) + if !ok || broken.Installable || broken.Supported { + t.Fatalf("broken = %#v", broken) + } + if _, ok := Find(models, "whisper-1", Vision); ok { + t.Fatal("Find invented a capability") + } +} + +func TestAdapterVocabulary(t *testing.T) { + for capability, adapter := range manualAdapter { + if !Serves(adapter, capability) || ProbeKind(adapter) == ProbeNone { + t.Errorf("manual route %s for %s cannot be probed", adapter, capability) + } + } + if Serves(AdapterAudioSpeech, Vision) || Serves("fax", Vision) { + t.Fatal("Serves accepted a mismatch") + } + // The native routes serve what their catalog entry says. + if !Serves(AdapterNovitaAsync, VideoGeneration) || !Serves(AdapterNovitaAsync, ImageGeneration) { + t.Fatal("native async route refused a catalog capability") + } + if ProbeKind(AdapterImages) != ProbeConfirm { + t.Fatal("the image probe must ask first: it produces a billed image") + } + for _, adapter := range Adapters { + if _, ok := adapterCapability[adapter]; !ok { + t.Errorf("adapter %s has no capability", adapter) + } + } + if ManualAdapter(VideoGeneration) != "" || ManualAdapter(VideoUnderstanding) != "" { + t.Fatal("a capability with no OpenAI-compatible route offered manual entry") + } +} diff --git a/internal/provider/client.go b/internal/provider/client.go index 18227a1d..8043f282 100644 --- a/internal/provider/client.go +++ b/internal/provider/client.go @@ -3,10 +3,12 @@ package provider import ( "bytes" "context" + "encoding/binary" "encoding/json" "errors" "fmt" "io" + "mime/multipart" "net" "net/http" "strings" @@ -98,7 +100,7 @@ func (c *Client) Probe(ctx context.Context, protocol, providerID, apiKey, model, if err != nil { return ProbeResult{}, err } - return c.sendProbe(request, protocol, apiKey, requestModel), nil + return c.sendProbe(request, protocol, apiKey, requestModel, 0), nil } // probePNG is a 32x32 opaque grey PNG. Some vision endpoints refuse images @@ -142,13 +144,123 @@ func (c *Client) ProbeVision(ctx context.Context, apiKey, model, customBase stri } request.Header.Set("Authorization", "Bearer "+apiKey) request.Header.Set("Content-Type", "application/json") - return c.sendProbe(request, ProtocolOpenAI, apiKey, model), nil + return c.sendProbe(request, ProtocolOpenAI, apiKey, model, 0), nil +} + +// Probe timeouts for the generation and audio routes. A chat probe answers in +// seconds; producing one image can take a minute. +const ( + audioProbeTimeout = 60 * time.Second + imageProbeTimeout = 180 * time.Second +) + +// ProbeTranscription sends half a second of silent WAV to +// /audio/transcriptions. Billed by audio length, so it costs next to nothing, +// but it is still a billed request: only a user's click may send it. +func (c *Client) ProbeTranscription(ctx context.Context, apiKey, model, customBase string) (ProbeResult, error) { + base, err := probeTarget(apiKey, model, customBase) + if err != nil { + return ProbeResult{}, err + } + var body bytes.Buffer + form := multipart.NewWriter(&body) + if err := form.WriteField("model", model); err != nil { + return ProbeResult{}, probeBuildError(err) + } + part, err := form.CreateFormFile("file", "probe.wav") + if err != nil { + return ProbeResult{}, probeBuildError(err) + } + if _, err := part.Write(silentWAV()); err != nil { + return ProbeResult{}, probeBuildError(err) + } + if err := form.Close(); err != nil { + return ProbeResult{}, probeBuildError(err) + } + request, err := http.NewRequestWithContext(ctx, http.MethodPost, OpenAIBaseURL(base)+"/audio/transcriptions", &body) + if err != nil { + return ProbeResult{}, oneerrors.New(oneerrors.InvalidRequest, "Provider endpoint is invalid", oneerrors.WithCause(err)) + } + request.Header.Set("Authorization", "Bearer "+apiKey) + request.Header.Set("Content-Type", form.FormDataContentType()) + return c.sendProbe(request, ProtocolOpenAI, apiKey, model, audioProbeTimeout), nil +} + +// ProbeSpeech asks /audio/speech to say "ok". The voice is OpenAI's default; +// a gateway whose model wants another voice refuses, and its message says so. +func (c *Client) ProbeSpeech(ctx context.Context, apiKey, model, customBase string) (ProbeResult, error) { + return c.postJSONProbe(ctx, apiKey, model, customBase, "/audio/speech", map[string]any{ + "model": model, "input": "ok", "voice": "alloy", + }, audioProbeTimeout) +} + +// ProbeImageGeneration asks /images/generations for one image. That is a +// billed image, so the UI asks the user before calling it. No size is sent: +// the valid sizes differ by model, and a size the model rejects would fail the +// probe for a reason that says nothing about the endpoint. +func (c *Client) ProbeImageGeneration(ctx context.Context, apiKey, model, customBase string) (ProbeResult, error) { + return c.postJSONProbe(ctx, apiKey, model, customBase, "/images/generations", map[string]any{ + "model": model, "prompt": "a plain grey square", "n": 1, + }, imageProbeTimeout) +} + +func (c *Client) postJSONProbe(ctx context.Context, apiKey, model, customBase, path string, body map[string]any, timeout time.Duration) (ProbeResult, error) { + base, err := probeTarget(apiKey, model, customBase) + if err != nil { + return ProbeResult{}, err + } + payload, err := json.Marshal(body) + if err != nil { + return ProbeResult{}, probeBuildError(err) + } + request, err := http.NewRequestWithContext(ctx, http.MethodPost, OpenAIBaseURL(base)+path, bytes.NewReader(payload)) + if err != nil { + return ProbeResult{}, oneerrors.New(oneerrors.InvalidRequest, "Provider endpoint is invalid", oneerrors.WithCause(err)) + } + request.Header.Set("Authorization", "Bearer "+apiKey) + request.Header.Set("Content-Type", "application/json") + return c.sendProbe(request, ProtocolOpenAI, apiKey, model, timeout), nil +} + +func probeTarget(apiKey, model, customBase string) (string, error) { + if apiKey == "" { + return "", oneerrors.New(oneerrors.InvalidRequest, "API key is required") + } + if strings.TrimSpace(model) == "" { + return "", oneerrors.New(oneerrors.InvalidRequest, "A model is required") + } + return ProviderBase("custom", customBase) +} + +func probeBuildError(err error) error { + return oneerrors.New(oneerrors.InternalError, "Unable to build Provider request", oneerrors.WithCause(err)) +} + +// silentWAV is half a second of 16 kHz mono 16-bit silence. +func silentWAV() []byte { + const rate, samples = 16000, 8000 + data := samples * 2 + out := make([]byte, 44+data) + copy(out[0:], "RIFF") + binary.LittleEndian.PutUint32(out[4:], uint32(36+data)) + copy(out[8:], "WAVEfmt ") + binary.LittleEndian.PutUint32(out[16:], 16) + binary.LittleEndian.PutUint16(out[20:], 1) + binary.LittleEndian.PutUint16(out[22:], 1) + binary.LittleEndian.PutUint32(out[24:], rate) + binary.LittleEndian.PutUint32(out[28:], rate*2) + binary.LittleEndian.PutUint16(out[32:], 2) + binary.LittleEndian.PutUint16(out[34:], 16) + copy(out[36:], "data") + binary.LittleEndian.PutUint32(out[40:], uint32(data)) + return out } // sendProbe runs one probe request and classifies the answer. It never // returns an error: every outcome, transport failures included, is a verdict. -func (c *Client) sendProbe(request *http.Request, protocol, apiKey, requestModel string) ProbeResult { - response, cancelRequest, err := c.do(request) +// A zero timeout uses the client's own. +func (c *Client) sendProbe(request *http.Request, protocol, apiKey, requestModel string, timeout time.Duration) ProbeResult { + response, cancelRequest, err := c.doWithin(request, timeout) defer cancelRequest() if err != nil { if response != nil && response.Body != nil { @@ -372,7 +484,13 @@ func protocolRequest(ctx context.Context, protocol, providerID, customBase, apiK } func (c *Client) do(request *http.Request) (*http.Response, context.CancelFunc, error) { - timeout := c.timeout + return c.doWithin(request, 0) +} + +func (c *Client) doWithin(request *http.Request, timeout time.Duration) (*http.Response, context.CancelFunc, error) { + if timeout <= 0 { + timeout = c.timeout + } if timeout <= 0 { timeout = defaultTimeout } diff --git a/internal/provider/client_test.go b/internal/provider/client_test.go index f5a86335..edb464fc 100644 --- a/internal/provider/client_test.go +++ b/internal/provider/client_test.go @@ -400,3 +400,83 @@ func TestProbePNGDecodes(t *testing.T) { t.Fatalf("probe image is %dx%d; some endpoints refuse images under 10px a side", config.Width, config.Height) } } + +func TestProbeTranscriptionSendsSilentWAV(t *testing.T) { + var seen *http.Request + var model string + var audio []byte + client := NewClient(fakeDoer(func(request *http.Request) (*http.Response, error) { + seen = request + if err := request.ParseMultipartForm(1 << 20); err != nil { + t.Fatal(err) + } + model = request.FormValue("model") + file, _, err := request.FormFile("file") + if err != nil { + t.Fatal(err) + } + audio, _ = io.ReadAll(file) + return fakeResponse(http.StatusOK, `{"text":""}`), nil + })) + result, err := client.ProbeTranscription(context.Background(), "key", "whisper-1", "https://proxy.test") + if err != nil { + t.Fatal(err) + } + if !result.OK || seen.URL.Path != "/v1/audio/transcriptions" || model != "whisper-1" { + t.Fatalf("result = %#v, path = %s, model = %q", result, seen.URL.Path, model) + } + if len(audio) != 44+16000 || string(audio[0:4]) != "RIFF" || string(audio[8:12]) != "WAVE" { + t.Fatalf("probe audio is not half a second of 16 kHz WAV: %d bytes", len(audio)) + } +} + +func TestProbeSpeechAndImagesSendMinimalBodies(t *testing.T) { + bodies := map[string]map[string]any{} + client := NewClient(fakeDoer(func(request *http.Request) (*http.Response, error) { + var body map[string]any + if err := json.NewDecoder(request.Body).Decode(&body); err != nil { + t.Fatal(err) + } + bodies[request.URL.Path] = body + return fakeResponse(http.StatusOK, `{}`), nil + })) + if result, err := client.ProbeSpeech(context.Background(), "key", "tts-1", "https://proxy.test"); err != nil || !result.OK { + t.Fatalf("speech = %#v, %v", result, err) + } + if result, err := client.ProbeImageGeneration(context.Background(), "key", "gpt-image-1", "https://proxy.test"); err != nil || !result.OK { + t.Fatalf("image = %#v, %v", result, err) + } + speech := bodies["/v1/audio/speech"] + if speech["input"] != "ok" || speech["model"] != "tts-1" || speech["voice"] == nil { + t.Fatalf("speech body = %#v", speech) + } + image := bodies["/v1/images/generations"] + if image["n"] != float64(1) || image["model"] != "gpt-image-1" { + t.Fatalf("image body = %#v", image) + } + // Valid sizes differ by model; sending one could fail the probe for a + // reason unrelated to the endpoint. + if _, ok := image["size"]; ok { + t.Fatalf("image probe sent a size: %#v", image) + } + for _, probe := range []func(context.Context, string, string, string) (ProbeResult, error){client.ProbeSpeech, client.ProbeImageGeneration, client.ProbeTranscription} { + if _, err := probe(context.Background(), "", "m", "https://proxy.test"); err == nil { + t.Fatal("probe without a key accepted") + } + if _, err := probe(context.Background(), "k", "", "https://proxy.test"); err == nil { + t.Fatal("probe without a model accepted") + } + } +} + +func TestSlowProbesOutlastTheChatTimeout(t *testing.T) { + client := NewClient(fakeDoer(func(request *http.Request) (*http.Response, error) { + deadline, ok := request.Context().Deadline() + if !ok || time.Until(deadline) < 2*defaultTimeout { + t.Errorf("%s deadline = %v away, want well over the %v chat timeout", request.URL.Path, time.Until(deadline), defaultTimeout) + } + return fakeResponse(http.StatusOK, `{}`), nil + })) + _, _ = client.ProbeImageGeneration(context.Background(), "k", "m", "https://proxy.test") + _, _ = client.ProbeSpeech(context.Background(), "k", "m", "https://proxy.test") +} diff --git a/internal/skill/model.go b/internal/skill/model.go index a9c59452..1a744bdc 100644 --- a/internal/skill/model.go +++ b/internal/skill/model.go @@ -27,6 +27,11 @@ const ( maxMetadataBytes = 64 << 10 ) +// ImportSourceGenerated marks a variant BootAgent rendered itself (a +// multimodal Skill), as opposed to one imported from an Agent, a folder or a +// ZIP. Only the generator writes it; the import path still refuses it. +const ImportSourceGenerated = "generated" + type Registry struct { SchemaVersion int `json:"schema_version"` Skills map[string]Fact `json:"skills"` diff --git a/internal/skill/store.go b/internal/skill/store.go index b844e472..c884ac65 100644 --- a/internal/skill/store.go +++ b/internal/skill/store.go @@ -143,6 +143,13 @@ func (s Store) SaveVariant(ctx context.Context, id, source string, stats TreeSta return PublishTree(ctx, stage, s.VariantPath(id, stats.Hash)) } +// EnsureSkillsRoot creates the private Skill storage root, for callers that +// stage a tree there before SaveVariant (the generator does: staging under the +// root keeps the copy on one filesystem and inside the private tree). +func (s Store) EnsureSkillsRoot(ctx context.Context) error { + return s.ensurePrivateRoot(ctx, s.SkillsRoot()) +} + func (s Store) RemoveSkill(ctx context.Context, id string) error { if err := ValidateID(id); err != nil { return err @@ -338,7 +345,7 @@ func validateFact(id string, fact Fact) error { } } for _, source := range variant.ImportSources { - if source != "agent" && source != "folder" && source != "zip" { + if source != "agent" && source != "folder" && source != "zip" && source != ImportSourceGenerated { return fmt.Errorf("Skill %q has invalid import source", id) } } diff --git a/internal/skillgen/script_test.go b/internal/skillgen/script_test.go new file mode 100644 index 00000000..4b4a0740 --- /dev/null +++ b/internal/skillgen/script_test.go @@ -0,0 +1,290 @@ +package skillgen + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/MaimoryLab/BootAgent/internal/modality" +) + +const scriptKey = "sk-script-test-secret" + +// fakeProvider records requests and answers each route the way an +// OpenAI-compatible gateway does. Error bodies echo the key, as some gateways +// do, so the tests prove the script scrubs it. +type fakeProvider struct { + mu sync.Mutex + requests []recorded + status int +} + +type recorded struct { + path, auth, contentType string + body []byte +} + +func (f *fakeProvider) handler(t *testing.T, server **httptest.Server) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + f.mu.Lock() + f.requests = append(f.requests, recorded{r.URL.Path, r.Header.Get("Authorization"), r.Header.Get("Content-Type"), body}) + status := f.status + f.mu.Unlock() + if status != 0 { + w.WriteHeader(status) + _, _ = w.Write([]byte(`{"error":{"message":"refused for ` + r.Header.Get("Authorization") + `"}}`)) + return + } + switch r.URL.Path { + case "/v1/chat/completions": + _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"A red error banner."}}]}`)) + case "/v1/images/generations": + png, _ := base64.StdEncoding.DecodeString("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAAAAAA6fptVAAAACklEQVR4nGNgAAAAAgABSK+kcQAAAABJRU5ErkJggg==") + _ = json.NewEncoder(w).Encode(map[string]any{"data": []map[string]string{ + {"b64_json": base64.StdEncoding.EncodeToString(png)}, + {"url": (*server).URL + "/files/result.png"}, + }}) + case "/files/result.png": + _, _ = w.Write([]byte("\xff\xd8\xff downloaded jpeg")) + case "/v1/audio/transcriptions": + _, _ = w.Write([]byte(`{"text":"hello there"}`)) + case "/v1/audio/speech": + _, _ = w.Write([]byte("ID3 fake mp3 bytes")) + default: + http.NotFound(w, r) + } + } +} + +func (f *fakeProvider) last(path string) (recorded, bool) { + f.mu.Lock() + defer f.mu.Unlock() + for index := len(f.requests) - 1; index >= 0; index-- { + if f.requests[index].path == path { + return f.requests[index], true + } + } + return recorded{}, false +} + +type scriptRun struct { + stdout, stderr string + code int +} + +// runScript runs the generated script with argv and a controlled environment: +// only PATH, HOME and what the test passes. +func runScript(t *testing.T, dir, workdir, home string, extraEnv []string, args ...string) scriptRun { + t.Helper() + node, err := exec.LookPath("node") + if err != nil { + t.Skip("node is not installed") + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + command := exec.CommandContext(ctx, node, append([]string{filepath.Join(dir, "scripts", "run.mjs")}, args...)...) + command.Dir = workdir + command.Env = append([]string{"PATH=" + os.Getenv("PATH"), "HOME=" + home, "USERPROFILE=" + home}, extraEnv...) + var stdout, stderr bytes.Buffer + command.Stdout, command.Stderr = &stdout, &stderr + err = command.Run() + code := 0 + if exit, ok := err.(*exec.ExitError); ok { + code = exit.ExitCode() + } else if err != nil { + t.Fatal(err) + } + return scriptRun{stdout.String(), stderr.String(), code} +} + +func scriptSetup(t *testing.T, capability, adapter, model string) (*fakeProvider, string, string, string) { + t.Helper() + if _, err := exec.LookPath("node"); err != nil { + t.Skip("node is not installed") + } + fake := &fakeProvider{} + var server *httptest.Server + server = httptest.NewServer(fake.handler(t, &server)) + t.Cleanup(server.Close) + dir := renderTo(t, Spec{Capability: capability, Adapter: adapter, ProviderID: "local", ProviderName: "Local", Model: model, BaseURL: server.URL}) + return fake, dir, resolvedTempDir(t), resolvedTempDir(t) +} + +// resolvedTempDir resolves symlinks in the temp path (macOS puts /var under +// /private/var), because Node reports the resolved working directory. +func resolvedTempDir(t *testing.T) string { + t.Helper() + dir, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + return dir +} + +var keyEnv = "BOOTAGENT_LOCAL_API_KEY=" + scriptKey + +func TestScriptDescribesAnImage(t *testing.T) { + fake, dir, work, home := scriptSetup(t, modality.Vision, modality.AdapterChatVision, "vision-model") + image := filepath.Join(work, "shot.png") + if err := os.WriteFile(image, []byte("\x89PNG\r\n\x1a\nfake"), 0o600); err != nil { + t.Fatal(err) + } + run := runScript(t, dir, work, home, []string{keyEnv}, "describe", "--image", "shot.png", "--prompt", "What error is shown?") + if run.code != 0 || run.stdout != "A red error banner.\n" { + t.Fatalf("run = %#v", run) + } + request, _ := fake.last("/v1/chat/completions") + if request.auth != "Bearer "+scriptKey { + t.Fatalf("auth = %q", request.auth) + } + var body struct { + Model string `json:"model"` + Messages []struct { + Content []map[string]any `json:"content"` + } `json:"messages"` + } + if err := json.Unmarshal(request.body, &body); err != nil { + t.Fatal(err) + } + parts := body.Messages[0].Content + if body.Model != "vision-model" || parts[0]["text"] != "What error is shown?" || !strings.HasPrefix(parts[1]["image_url"].(map[string]any)["url"].(string), "data:image/png;base64,") { + t.Fatalf("body = %s", request.body) + } +} + +func TestScriptGeneratesImagesAndDownloadsWithoutTheKey(t *testing.T) { + fake, dir, work, home := scriptSetup(t, modality.ImageGeneration, modality.AdapterImages, "gpt-image-1") + run := runScript(t, dir, work, home, []string{keyEnv}, "generate", "--prompt", "a lighthouse", "--n", "2") + if run.code != 0 { + t.Fatalf("run = %#v", run) + } + paths := strings.Split(strings.TrimSpace(run.stdout), "\n") + if len(paths) != 2 || !strings.HasSuffix(paths[0], ".png") || !strings.HasSuffix(paths[1], ".jpg") { + t.Fatalf("saved = %q", paths) + } + for _, path := range paths { + if !strings.HasPrefix(path, filepath.Join(work, "bootagent-output")) { + t.Fatalf("%s is outside the working directory's output folder", path) + } + if _, err := os.Stat(path); err != nil { + t.Fatal(err) + } + } + // The result URL may be on another host: the key must not go with it. + download, ok := fake.last("/files/result.png") + if !ok || download.auth != "" { + t.Fatalf("download request = %#v", download) + } + // A second run never overwrites the first. + again := runScript(t, dir, work, home, []string{keyEnv}, "generate", "--prompt", "a lighthouse") + if again.code != 0 || again.stdout == run.stdout { + t.Fatalf("second run = %#v", again) + } +} + +func TestScriptTranscribesAndSpeaks(t *testing.T) { + fake, dir, work, home := scriptSetup(t, modality.SpeechToText, modality.AdapterAudioTranscriptions, "whisper-1") + if err := os.WriteFile(filepath.Join(work, "note.wav"), []byte("RIFF....WAVE"), 0o600); err != nil { + t.Fatal(err) + } + run := runScript(t, dir, work, home, []string{keyEnv}, "transcribe", "--audio", "note.wav", "--language", "en") + if run.code != 0 || run.stdout != "hello there\n" { + t.Fatalf("run = %#v", run) + } + request, _ := fake.last("/v1/audio/transcriptions") + if !strings.HasPrefix(request.contentType, "multipart/form-data") || !bytes.Contains(request.body, []byte("whisper-1")) || !bytes.Contains(request.body, []byte("note.wav")) { + t.Fatalf("transcription request = %q", request.body) + } + + _, speechDir, speechWork, speechHome := scriptSetup(t, modality.TextToSpeech, modality.AdapterAudioSpeech, "tts-1") + spoken := runScript(t, speechDir, speechWork, speechHome, []string{keyEnv}, "speak", "--text", "Build passed", "--out", "done.mp3") + if spoken.code != 0 || strings.TrimSpace(spoken.stdout) != filepath.Join(speechWork, "done.mp3") { + t.Fatalf("speak = %#v", spoken) + } + if data, _ := os.ReadFile(filepath.Join(speechWork, "done.mp3")); string(data) != "ID3 fake mp3 bytes" { + t.Fatalf("audio = %q", data) + } +} + +func TestScriptReadsTheFallbackFileOnlyWhenTheVariableIsUnset(t *testing.T) { + fake, dir, work, home := scriptSetup(t, modality.TextToSpeech, modality.AdapterAudioSpeech, "tts-1") + if err := os.MkdirAll(filepath.Join(home, ".bootagent", "skill-env"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(home, ".bootagent", "skill-env", "local.env"), []byte("BOOTAGENT_LOCAL_API_KEY='sk-from-\\'file'\n"), 0o600); err != nil { + t.Fatal(err) + } + if run := runScript(t, dir, work, home, nil, "speak", "--text", "hi"); run.code != 0 { + t.Fatalf("fallback run = %#v", run) + } + if request, _ := fake.last("/v1/audio/speech"); request.auth != "Bearer sk-from-'file" { + t.Fatalf("fallback auth = %q", request.auth) + } + if run := runScript(t, dir, work, home, []string{keyEnv}, "speak", "--text", "hi"); run.code != 0 { + t.Fatalf("env run = %#v", run) + } + if request, _ := fake.last("/v1/audio/speech"); request.auth != "Bearer "+scriptKey { + t.Fatalf("the variable did not win over the file: %q", request.auth) + } +} + +func TestScriptExitCodesAndRedaction(t *testing.T) { + fake, dir, work, home := scriptSetup(t, modality.Vision, modality.AdapterChatVision, "vision-model") + if err := os.WriteFile(filepath.Join(work, "a.png"), []byte("png"), 0o600); err != nil { + t.Fatal(err) + } + cases := []struct { + name string + status int + env []string + args []string + code int + }{ + {"no key", 0, nil, []string{"describe", "--image", "a.png"}, 3}, + {"missing image", 0, []string{keyEnv}, []string{"describe", "--image", "nope.png"}, 2}, + {"wrong subcommand", 0, []string{keyEnv}, []string{"generate", "--prompt", "x"}, 2}, + {"refused", http.StatusBadRequest, []string{keyEnv}, []string{"describe", "--image", "a.png"}, 4}, + {"key rejected", http.StatusUnauthorized, []string{keyEnv}, []string{"describe", "--image", "a.png"}, 5}, + {"busy", http.StatusServiceUnavailable, []string{keyEnv}, []string{"describe", "--image", "a.png"}, 6}, + } + for _, test := range cases { + fake.mu.Lock() + fake.status = test.status + fake.mu.Unlock() + run := runScript(t, dir, work, home, test.env, test.args...) + if run.code != test.code { + t.Errorf("%s: exit %d, want %d (%s)", test.name, run.code, test.code, run.stderr) + } + if strings.Contains(run.stdout+run.stderr, scriptKey) { + t.Errorf("%s: output carries the key: %s", test.name, run.stdout+run.stderr) + } + if test.status != 0 && !strings.Contains(run.stderr, "[redacted]") { + t.Errorf("%s: the provider's echo was not shown redacted: %s", test.name, run.stderr) + } + } + // check confirms setup without sending anything. + fake.mu.Lock() + before := len(fake.requests) + fake.mu.Unlock() + if run := runScript(t, dir, work, home, []string{keyEnv}, "check"); run.code != 0 || !strings.Contains(run.stdout, "ready") || strings.Contains(run.stdout, scriptKey) { + t.Fatalf("check = %#v", run) + } + fake.mu.Lock() + after := len(fake.requests) + fake.mu.Unlock() + if after != before { + t.Fatal("check sent a request") + } +} diff --git a/internal/skillgen/skillgen.go b/internal/skillgen/skillgen.go new file mode 100644 index 00000000..a5191a2f --- /dev/null +++ b/internal/skillgen/skillgen.go @@ -0,0 +1,289 @@ +// Package skillgen renders the Skill tree for one multimodal capability bound +// to one Provider and model: SKILL.md, config.json and scripts/run.mjs. +// +// The tree never holds the API key. config.json names the environment +// variable that carries it (ADR-011); the Spec has no field a key could be put +// in, so no caller can render one by mistake. +package skillgen + +import ( + "bytes" + "embed" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + "text/template" + + "github.com/MaimoryLab/BootAgent/internal/config" + "github.com/MaimoryLab/BootAgent/internal/modality" + "github.com/MaimoryLab/BootAgent/internal/provider" +) + +//go:embed templates +var templates embed.FS + +// ConfigSchemaVersion is config.json's version; run.mjs reads it as data. +const ConfigSchemaVersion = 1 + +// IDPrefix marks every generated Skill ID. +const IDPrefix = "bootagent-" + +// supported lists the capability and adapter pairs this version can render. +// run.mjs implements exactly these; anything else is refused instead of +// producing a Skill whose script cannot run it. +var supported = map[string]string{ + modality.Vision: modality.AdapterChatVision, + modality.ImageGeneration: modality.AdapterImages, + modality.SpeechToText: modality.AdapterAudioTranscriptions, + modality.TextToSpeech: modality.AdapterAudioSpeech, +} + +// Supports reports whether a capability over an adapter can be generated. +func Supports(capability, adapter string) bool { + return adapter != "" && supported[capability] == adapter +} + +// SupportedAdapters lists the adapters a Skill can be generated for. +func SupportedAdapters() []string { + result := make([]string, 0, len(supported)) + for _, capability := range modality.Capabilities { + if adapter, ok := supported[capability]; ok { + result = append(result, adapter) + } + } + return result +} + +// SkillID is the generated Skill's ID for a capability. +func SkillID(capability string) string { + return IDPrefix + capability +} + +// CapabilityFromSkillID reverses SkillID, empty for any other ID. +func CapabilityFromSkillID(id string) string { + capability, ok := strings.CutPrefix(id, IDPrefix) + if !ok { + return "" + } + for _, known := range modality.Capabilities { + if known == capability { + return capability + } + } + return "" +} + +// Spec is everything a Skill is generated from. It deliberately has no field +// for the key. +type Spec struct { + Capability string + Adapter string + ProviderID string + ProviderName string + Model string + // BaseURL is the Provider's OpenAI-compatible base, normalized to /v1. + BaseURL string + // NativeBaseURL is the host root for native routes; empty when unused. + NativeBaseURL string +} + +// Config is config.json. Nothing in it is secret. +type Config struct { + SchemaVersion int `json:"schema_version"` + Generator string `json:"generator"` + Capability string `json:"capability"` + Adapter string `json:"adapter"` + ProviderID string `json:"provider_id"` + ProviderName string `json:"provider_name"` + Model string `json:"model"` + BaseURL string `json:"base_url"` + NativeBaseURL string `json:"native_base_url,omitempty"` + // KeyEnv names the variable holding the key; KeyFile is the private + // fallback file read only when the variable is unset. "~" is the home + // directory, expanded by the script so the tree is the same on every + // machine. + KeyEnv string `json:"key_env"` + KeyFile string `json:"key_file"` + OutputDir string `json:"output_dir"` +} + +var providerIDPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,63}$`) + +// NewConfig validates a Spec and derives config.json from it. +func NewConfig(spec Spec) (Config, error) { + if !Supports(spec.Capability, spec.Adapter) { + return Config{}, fmt.Errorf("a Skill for %s over %s cannot be generated yet", spec.Capability, spec.Adapter) + } + if !providerIDPattern.MatchString(spec.ProviderID) { + return Config{}, errors.New("invalid Provider ID for a generated Skill") + } + model := strings.TrimSpace(spec.Model) + if model == "" || len(model) > 256 || strings.ContainsAny(model, "\r\n\x00") { + return Config{}, errors.New("invalid model for a generated Skill") + } + base, err := provider.ValidateBaseURL(strings.TrimSpace(spec.BaseURL)) + if err != nil { + return Config{}, err + } + name := strings.TrimSpace(spec.ProviderName) + if name == "" { + name = spec.ProviderID + } + if strings.ContainsAny(name, "\r\n\x00") { + return Config{}, errors.New("invalid Provider name for a generated Skill") + } + return Config{ + SchemaVersion: ConfigSchemaVersion, + Generator: "bootagent", + Capability: spec.Capability, + Adapter: spec.Adapter, + ProviderID: spec.ProviderID, + ProviderName: name, + Model: model, + BaseURL: provider.OpenAIBaseURL(base), + NativeBaseURL: strings.TrimRight(strings.TrimSpace(spec.NativeBaseURL), "/"), + KeyEnv: config.SkillEnvName(spec.ProviderID), + KeyFile: "~/.bootagent/skill-env/" + spec.ProviderID + ".env", + OutputDir: "bootagent-output", + }, nil +} + +type skillText struct { + SkillID string + Title string + Description string + Model string + ProviderName string + KeyEnv string + KeyFile string + CostRule string +} + +var titles = map[string]string{ + modality.Vision: "Image understanding", + modality.ImageGeneration: "Image generation", + modality.SpeechToText: "Speech to text", + modality.TextToSpeech: "Text to speech", +} + +var descriptions = map[string]string{ + modality.Vision: "Look at local image files (screenshots, photos, diagrams) and answer questions about them through %s on %s. Use when the user asks about an image and you cannot see images yourself.", + modality.ImageGeneration: "Generate images from a text description through %s on %s and save them as files. Use when the user asks you to create an image. Each run is billed: confirm with the user first.", + modality.SpeechToText: "Transcribe a local audio file to text through %s on %s. Use when the user asks for the text of a recording.", + modality.TextToSpeech: "Turn text into a spoken audio file through %s on %s. Use when the user asks for audio of some text. Each run is billed: confirm with the user first.", +} + +var costRules = map[string]string{ + modality.Vision: "Each run is a billed request to the user's Provider. Run it when the user asks about specific images; do not run it speculatively over many files without asking.", + modality.ImageGeneration: "Every run costs money. Before each run, tell the user what you are about to generate and how many images, and wait for them to agree.", + modality.SpeechToText: "Each run is billed by the length of the audio. Run it when the user asks for a transcript; ask first before transcribing long or many recordings.", + modality.TextToSpeech: "Every run costs money, billed by the length of the text. Before each run, tell the user what text you are about to speak and wait for them to agree.", +} + +// Render writes the Skill tree into dir, which must exist and be empty. The +// output depends only on the Spec, so the same Spec always hashes the same. +func Render(spec Spec, dir string) error { + cfg, err := NewConfig(spec) + if err != nil { + return err + } + files, err := Files(cfg) + if err != nil { + return err + } + for name, data := range files { + path := filepath.Join(dir, filepath.FromSlash(name)) + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return err + } + if err := os.WriteFile(path, data, 0o600); err != nil { + return err + } + } + return nil +} + +// Files returns the tree's contents by slash-separated path. +func Files(cfg Config) (map[string][]byte, error) { + if !Supports(cfg.Capability, cfg.Adapter) { + return nil, fmt.Errorf("a Skill for %s over %s cannot be generated yet", cfg.Capability, cfg.Adapter) + } + text := skillText{ + SkillID: SkillID(cfg.Capability), + Title: titles[cfg.Capability], + Description: fmt.Sprintf(descriptions[cfg.Capability], cfg.Model, cfg.ProviderName), + Model: cfg.Model, + ProviderName: cfg.ProviderName, + KeyEnv: cfg.KeyEnv, + KeyFile: cfg.KeyFile, + CostRule: costRules[cfg.Capability], + } + parsed, err := template.New("skill").Funcs(template.FuncMap{"yamlString": yamlString}).ParseFS(templates, "templates/common.md.tmpl", "templates/"+cfg.Capability+".md.tmpl") + if err != nil { + return nil, err + } + var skill bytes.Buffer + if err := parsed.ExecuteTemplate(&skill, cfg.Capability+".md.tmpl", text); err != nil { + return nil, err + } + configJSON, err := json.MarshalIndent(cfg, "", " ") + if err != nil { + return nil, err + } + script, err := templates.ReadFile("templates/run.mjs") + if err != nil { + return nil, err + } + return map[string][]byte{ + "SKILL.md": normalizeSkill(skill.Bytes()), + "config.json": append(configJSON, '\n'), + "scripts/run.mjs": script, + }, nil +} + +// ReadConfig reads config.json from a stored Skill tree. The key variable is +// re-derived from the Provider ID rather than trusted from the file, so a +// hand-edited tree cannot point BootAgent at a variable it does not own. +func ReadConfig(dir string) (Config, error) { + data, err := os.ReadFile(filepath.Join(dir, "config.json")) + if err != nil { + return Config{}, err + } + if len(data) > 64<<10 { + return Config{}, errors.New("generated Skill config is too large") + } + var cfg Config + if err := json.Unmarshal(data, &cfg); err != nil { + return Config{}, err + } + if cfg.SchemaVersion != ConfigSchemaVersion || cfg.Generator != "bootagent" || !providerIDPattern.MatchString(cfg.ProviderID) || strings.TrimSpace(cfg.Model) == "" { + return Config{}, errors.New("not a BootAgent generated Skill config") + } + cfg.KeyEnv = config.SkillEnvName(cfg.ProviderID) + return cfg, nil +} + +// yamlString renders a YAML double-quoted scalar. JSON string syntax is a +// subset of it, so encoding/json does the escaping. +func yamlString(value string) string { + encoded, _ := json.Marshal(value) + return string(encoded) +} + +// normalizeSkill collapses the blank lines template actions leave behind. +func normalizeSkill(data []byte) []byte { + lines := strings.Split(strings.ReplaceAll(string(data), "\r\n", "\n"), "\n") + out := make([]string, 0, len(lines)) + for _, line := range lines { + line = strings.TrimRight(line, " \t") + if line == "" && (len(out) == 0 || out[len(out)-1] == "") { + continue + } + out = append(out, line) + } + return []byte(strings.TrimRight(strings.Join(out, "\n"), "\n") + "\n") +} diff --git a/internal/skillgen/skillgen_test.go b/internal/skillgen/skillgen_test.go new file mode 100644 index 00000000..c160d55b --- /dev/null +++ b/internal/skillgen/skillgen_test.go @@ -0,0 +1,202 @@ +package skillgen + +import ( + "bytes" + "context" + "flag" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + + "github.com/MaimoryLab/BootAgent/internal/modality" + "github.com/MaimoryLab/BootAgent/internal/skill" +) + +var update = flag.Bool("update", false, "rewrite the golden files") + +var goldenSpecs = map[string]Spec{ + modality.Vision: { + Capability: modality.Vision, Adapter: modality.AdapterChatVision, + ProviderID: "novita", ProviderName: "Novita", Model: "qwen/qwen3-vl-8b", BaseURL: "https://api.novita.ai/openai", + }, + modality.ImageGeneration: { + Capability: modality.ImageGeneration, Adapter: modality.AdapterImages, + ProviderID: "my-gateway", ProviderName: `Gateway "Prod": EU`, Model: "gpt-image-1", BaseURL: "https://gw.example/v1/", + }, + modality.SpeechToText: { + Capability: modality.SpeechToText, Adapter: modality.AdapterAudioTranscriptions, + ProviderID: "ppio", ProviderName: "PPIO", Model: "whisper-1", BaseURL: "https://api.ppio.com/openai", + }, + modality.TextToSpeech: { + Capability: modality.TextToSpeech, Adapter: modality.AdapterAudioSpeech, + ProviderID: "local", ProviderName: "", Model: "tts-1", BaseURL: "http://127.0.0.1:8000", + }, +} + +func renderTo(t *testing.T, spec Spec) string { + t.Helper() + dir := t.TempDir() + if err := Render(spec, dir); err != nil { + t.Fatal(err) + } + return dir +} + +func TestGoldenFiles(t *testing.T) { + for capability, spec := range goldenSpecs { + t.Run(capability, func(t *testing.T) { + dir := renderTo(t, spec) + for _, name := range []string{"SKILL.md", "config.json"} { + got, err := os.ReadFile(filepath.Join(dir, name)) + if err != nil { + t.Fatal(err) + } + golden := filepath.Join("testdata", "golden", capability, name) + if *update { + if err := os.MkdirAll(filepath.Dir(golden), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(golden, got, 0o644); err != nil { + t.Fatal(err) + } + } + want, err := os.ReadFile(golden) + if err != nil { + t.Fatalf("%v (run with -update to create it)", err) + } + if !bytes.Equal(got, want) { + t.Errorf("%s differs from %s; run with -update after checking the change\n%s", name, golden, got) + } + } + script, _ := os.ReadFile(filepath.Join(dir, "scripts", "run.mjs")) + source, _ := os.ReadFile(filepath.Join("templates", "run.mjs")) + if !bytes.Equal(script, source) { + t.Error("scripts/run.mjs is not the embedded script") + } + }) + } +} + +// The design's promise: no key in any generated file. The Spec has no key +// field, so this guards against a future template interpolating something +// key-shaped from elsewhere. +var keyShaped = regexp.MustCompile(`sk-[A-Za-z0-9]|"api_key"|Bearer [A-Za-z0-9]`) + +func TestGeneratedTreesCarryNoKey(t *testing.T) { + for capability, spec := range goldenSpecs { + dir := renderTo(t, spec) + err := filepath.WalkDir(dir, func(path string, entry os.DirEntry, err error) error { + if err != nil || entry.IsDir() { + return err + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + if match := keyShaped.Find(data); match != nil && filepath.Base(path) != "run.mjs" { + t.Errorf("%s/%s contains %q", capability, filepath.Base(path), match) + } + return nil + }) + if err != nil { + t.Fatal(err) + } + } +} + +func TestRenderIsDeterministicAndReadableAsASkill(t *testing.T) { + spec := goldenSpecs[modality.ImageGeneration] + first, err := skill.HashTree(context.Background(), renderTo(t, spec)) + if err != nil { + t.Fatal(err) + } + dir := renderTo(t, spec) + second, err := skill.HashTree(context.Background(), dir) + if err != nil { + t.Fatal(err) + } + if first != second { + t.Fatalf("same Spec, different trees: %v vs %v", first, second) + } + // A provider name with quotes and a colon must not break the frontmatter. + name, description, diagnostic := skill.ReadMetadata(context.Background(), dir, "fallback") + if diagnostic != "" || name != "bootagent-image-generation" || !strings.Contains(description, `Gateway "Prod": EU`) { + t.Fatalf("metadata = %q, %q, %q", name, description, diagnostic) + } + changed := spec + changed.Model = "gpt-image-2" + third, _ := skill.HashTree(context.Background(), renderTo(t, changed)) + if third.Hash == first.Hash { + t.Fatal("a different model produced the same tree") + } +} + +func TestConfigNormalizesAndReadsBack(t *testing.T) { + dir := renderTo(t, goldenSpecs[modality.ImageGeneration]) + cfg, err := ReadConfig(dir) + if err != nil { + t.Fatal(err) + } + if cfg.BaseURL != "https://gw.example/v1" || cfg.KeyEnv != "BOOTAGENT_MY_GATEWAY_API_KEY" || cfg.KeyFile != "~/.bootagent/skill-env/my-gateway.env" { + t.Fatalf("config = %#v", cfg) + } + // A hand-edited key_env is not trusted: the name is re-derived. + path := filepath.Join(dir, "config.json") + data, _ := os.ReadFile(path) + if err := os.WriteFile(path, bytes.Replace(data, []byte("BOOTAGENT_MY_GATEWAY_API_KEY"), []byte("PATH"), 1), 0o600); err != nil { + t.Fatal(err) + } + cfg, err = ReadConfig(dir) + if err != nil || cfg.KeyEnv != "BOOTAGENT_MY_GATEWAY_API_KEY" { + t.Fatalf("tampered config = %#v, %v", cfg, err) + } + if err := os.WriteFile(path, []byte(`{"schema_version":1,"generator":"someone","provider_id":"x","model":"m"}`), 0o600); err != nil { + t.Fatal(err) + } + if _, err := ReadConfig(dir); err == nil { + t.Fatal("a foreign config was accepted") + } +} + +func TestNewConfigRefusesWhatItCannotGenerate(t *testing.T) { + base := goldenSpecs[modality.Vision] + cases := map[string]func(*Spec){ + "chat audio input": func(s *Spec) { s.Capability, s.Adapter = modality.SpeechToText, modality.AdapterChatAudioInput }, + "chat image output": func(s *Spec) { + s.Capability, s.Adapter = modality.ImageGeneration, modality.AdapterChatImageOutput + }, + "mismatched adapter": func(s *Spec) { s.Adapter = modality.AdapterImages }, + "video": func(s *Spec) { s.Capability, s.Adapter = modality.VideoUnderstanding, modality.AdapterChatVideo }, + "bad provider": func(s *Spec) { s.ProviderID = "Bad_ID" }, + "model with newline": func(s *Spec) { s.Model = "m\nrm -rf" }, + "empty model": func(s *Spec) { s.Model = " " }, + "bad base": func(s *Spec) { s.BaseURL = "ftp://x" }, + "base with userinfo": func(s *Spec) { s.BaseURL = "https://user:pass@x.test" }, + } + for name, mutate := range cases { + spec := base + mutate(&spec) + if _, err := NewConfig(spec); err == nil { + t.Errorf("%s accepted", name) + } + } +} + +func TestSkillIDRoundTrip(t *testing.T) { + for _, capability := range modality.Capabilities { + id := SkillID(capability) + if err := skill.ValidateID(id); err != nil { + t.Fatalf("%s is not a valid Skill ID: %v", id, err) + } + if CapabilityFromSkillID(id) != capability { + t.Fatalf("round trip of %s", id) + } + } + for _, id := range []string{"review", "bootagent-", "bootagent-smell"} { + if CapabilityFromSkillID(id) != "" { + t.Fatalf("%s read as generated", id) + } + } +} diff --git a/internal/skillgen/templates/common.md.tmpl b/internal/skillgen/templates/common.md.tmpl new file mode 100644 index 00000000..c49ab814 --- /dev/null +++ b/internal/skillgen/templates/common.md.tmpl @@ -0,0 +1,29 @@ +{{define "header"}}--- +name: {{.SkillID}} +description: {{.Description | yamlString}} +--- + +# {{.Title}} + +Generated by BootAgent. It calls `{{.Model}}` on the user's {{.ProviderName}} account. BootAgent rewrites this Skill when the model or Provider changes, so do not edit it. +{{end}} +{{define "footer"}} +## Rules + +- {{.CostRule}} +- Never print, echo or log environment variables, `{{.KeyFile}}`, or anything that could reveal the API key. The script reads the key itself; you never need it. +- Only send files the user asked about. They go to the user's own Provider and nowhere else. + +## Exit codes + +| Code | Meaning | What to tell the user | +| --- | --- | --- | +| 0 | Success | Report the result. | +| 2 | Bad arguments or input file | Fix the command, or ask the user for the right file. | +| 3 | Setup problem: Node.js older than 18, or no API key found | Ask the user to restart the Agent so it picks up `{{.KeyEnv}}`, or to reinstall this Skill from BootAgent. | +| 4 | The Provider refused the request | Show the Provider's message. The model may not support this input; the user can choose another model in BootAgent. | +| 5 | The Provider rejected the API key | Ask the user to check the {{.ProviderName}} key in BootAgent. | +| 6 | The Provider was unreachable, busy or timed out | Suggest trying again later. | + +`node /scripts/run.mjs check` confirms the key is available without sending a request. +{{end}} diff --git a/internal/skillgen/templates/image-generation.md.tmpl b/internal/skillgen/templates/image-generation.md.tmpl new file mode 100644 index 00000000..f2f3bb20 --- /dev/null +++ b/internal/skillgen/templates/image-generation.md.tmpl @@ -0,0 +1,21 @@ +{{template "header" .}} +## When to use + +When the user asks you to create an image: an illustration, an icon draft, a mock-up, a diagram-style picture. + +## Command + +Run the script in this Skill's directory (the directory containing this `SKILL.md`): + +```bash +node /scripts/run.mjs generate --prompt "" [--size ] [--n ] [--out ] +``` + +- `--prompt`: a full description of the image to create. +- `--size`: for example `1024x1024`. Leave it out unless the user asks; valid sizes depend on the model. +- `--n`: how many images, 1 to 4. Default 1. +- `--out`: directory for the files. Default `bootagent-output` in the current directory. +- `--timeout`: seconds to wait. Default 300. + +The saved file paths are printed to stdout, one per line. Existing files are never overwritten. +{{template "footer" .}} diff --git a/internal/skillgen/templates/run.mjs b/internal/skillgen/templates/run.mjs new file mode 100644 index 00000000..0c6396a5 --- /dev/null +++ b/internal/skillgen/templates/run.mjs @@ -0,0 +1,369 @@ +#!/usr/bin/env node +// Generated by BootAgent. Do not edit: BootAgent rewrites this Skill when its +// model or Provider changes. +// +// Calls one multimodal model on the user's own Provider. Node 18 or later, no +// dependencies. The API key comes from the environment variable named in +// config.json, or from BootAgent's private fallback file; it is never printed. + +import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs"; +import { homedir } from "node:os"; +import { basename, dirname, extname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const EXIT = { ok: 0, internal: 1, usage: 2, setup: 3, refused: 4, key: 5, unavailable: 6 }; +const MiB = 1024 * 1024; +const LIMITS = { image: 20 * MiB, audio: 25 * MiB, download: 50 * MiB, speech: 50 * MiB }; + +class Failure extends Error { + constructor(code, message) { + super(message); + this.code = code; + } +} + +let secret = ""; + +// The key never reaches output, even when a provider echoes the request back. +function redact(text) { + let value = String(text ?? ""); + if (secret) value = value.split(secret).join("[redacted]"); + return value.replace(/Bearer\s+[^\s"']+/g, "Bearer [redacted]"); +} + +function fail(code, message) { + throw new Failure(code, message); +} + +const here = dirname(fileURLToPath(import.meta.url)); + +function loadConfig() { + const path = join(here, "..", "config.json"); + let config; + try { + config = JSON.parse(readFileSync(path, "utf8")); + } catch { + fail(EXIT.setup, "config.json next to this Skill is missing or invalid. Reinstall the Skill from BootAgent."); + } + for (const field of ["capability", "adapter", "model", "base_url", "key_env"]) { + if (typeof config[field] !== "string" || !config[field]) fail(EXIT.setup, `config.json has no ${field}. Reinstall the Skill from BootAgent.`); + } + if (!/^https?:\/\//.test(config.base_url)) fail(EXIT.setup, "config.json has an invalid base_url. Reinstall the Skill from BootAgent."); + return config; +} + +function expandHome(path) { + if (path === "~") return homedir(); + if (path.startsWith("~/") || path.startsWith("~\\")) return join(homedir(), path.slice(2)); + return path; +} + +// Same quoting BootAgent writes: bare, or single-quoted with \' and \\. +function parseEnvFile(text, name) { + for (const raw of text.split(/\r?\n/)) { + const line = raw.trim(); + if (!line || line.startsWith("#")) continue; + const eq = line.indexOf("="); + if (eq < 0 || line.slice(0, eq).trim() !== name) continue; + let value = line.slice(eq + 1).trim(); + if (value.length >= 2 && value.startsWith("'") && value.endsWith("'")) { + value = value.slice(1, -1).replace(/\\(['\\])/g, "$1"); + } else if (value.length >= 2 && value.startsWith('"') && value.endsWith('"')) { + value = value.slice(1, -1).replace(/\\(["\\])/g, "$1"); + } + return value; + } + return ""; +} + +function loadKey(config) { + let key = process.env[config.key_env] ?? ""; + if (!key && typeof config.key_file === "string" && config.key_file) { + const path = expandHome(config.key_file); + if (existsSync(path)) { + try { + key = parseEnvFile(readFileSync(path, "utf8"), config.key_env); + } catch { + key = ""; + } + } + } + key = key.trim(); + if (!key) { + fail(EXIT.setup, `No API key found in ${config.key_env}. Restart the Agent so it picks up the variable BootAgent wrote, or reinstall the Skill from BootAgent.`); + } + secret = key; + return key; +} + +function parseArgs(argv) { + const options = { _: [] }; + for (let index = 0; index < argv.length; index++) { + const arg = argv[index]; + if (!arg.startsWith("--")) { + options._.push(arg); + continue; + } + const name = arg.slice(2); + if (name === "help") { + options.help = true; + continue; + } + const value = argv[index + 1]; + if (value === undefined || value.startsWith("--")) fail(EXIT.usage, `--${name} needs a value.`); + index++; + if (name === "image") (options.image ??= []).push(value); + else options[name] = value; + } + return options; +} + +function positiveInt(value, name, fallback, max) { + if (value === undefined) return fallback; + const number = Number(value); + if (!Number.isInteger(number) || number < 1 || number > max) fail(EXIT.usage, `--${name} must be a whole number from 1 to ${max}.`); + return number; +} + +function readInput(path, limit, kind) { + const full = resolve(path); + let info; + try { + info = statSync(full); + } catch { + fail(EXIT.usage, `${kind} file not found: ${path}`); + } + if (!info.isFile()) fail(EXIT.usage, `${kind} path is not a file: ${path}`); + if (info.size > limit) fail(EXIT.usage, `${kind} file is larger than ${Math.round(limit / MiB)} MB: ${path}`); + return readFileSync(full); +} + +const IMAGE_TYPES = { ".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".webp": "image/webp", ".gif": "image/gif" }; + +function stamp() { + return new Date().toISOString().replace(/[-:]/g, "").replace(/\..+$/, "").replace("T", "-"); +} + +function freePath(path) { + if (!existsSync(path)) return path; + const extension = extname(path); + const stem = path.slice(0, path.length - extension.length); + for (let counter = 2; ; counter++) { + const candidate = `${stem}-${counter}${extension}`; + if (!existsSync(candidate)) return candidate; + } +} + +function writeOutput(path, data) { + const target = freePath(resolve(path)); + mkdirSync(dirname(target), { recursive: true }); + writeFileSync(target, data); + return target; +} + +function detail(text) { + let message = text; + try { + const parsed = JSON.parse(text); + const error = parsed?.error; + message = (typeof error === "string" && error) || error?.message || parsed?.message || parsed?.detail || text; + } catch { + // Not JSON; keep the text. + } + message = redact(String(message)).replace(/\s+/g, " ").trim(); + if (/^ 400 ? `${message.slice(0, 400)}…` : message; +} + +async function call(config, key, path, init, timeoutMs) { + const url = config.base_url.replace(/\/+$/, "") + path; + let response; + try { + response = await fetch(url, { + ...init, + headers: { ...(init.headers ?? {}), Authorization: `Bearer ${key}` }, + signal: AbortSignal.timeout(timeoutMs), + }); + } catch (error) { + if (error?.name === "TimeoutError" || error?.name === "AbortError") fail(EXIT.unavailable, `The provider did not answer within ${Math.round(timeoutMs / 1000)} s.`); + fail(EXIT.unavailable, `Cannot reach the provider: ${redact(error?.cause?.code ?? error?.message ?? error)}`); + } + if (response.ok) return response; + const said = detail(await response.text().catch(() => "")); + const suffix = said ? `: ${said}` : "."; + if (response.status === 401 || response.status === 403) fail(EXIT.key, `The provider rejected the API key (HTTP ${response.status})${suffix}`); + if (response.status === 408 || response.status === 429 || response.status >= 500) fail(EXIT.unavailable, `The provider is busy or failing (HTTP ${response.status})${suffix}`); + fail(EXIT.refused, `The provider refused the request (HTTP ${response.status})${suffix}`); +} + +async function json(response) { + try { + return await response.json(); + } catch { + fail(EXIT.unavailable, "The provider returned a response that is not JSON."); + } +} + +// Result URLs are pre-signed and may be on another host, so the key is not +// sent with them. +async function download(url, timeoutMs) { + let response; + try { + response = await fetch(url, { signal: AbortSignal.timeout(timeoutMs) }); + } catch { + fail(EXIT.unavailable, "Cannot download the result from the provider."); + } + if (!response.ok) fail(EXIT.unavailable, `Cannot download the result (HTTP ${response.status}).`); + const data = Buffer.from(await response.arrayBuffer()); + if (data.length > LIMITS.download) fail(EXIT.unavailable, "The result is larger than expected and was not saved."); + return data; +} + +function imageExtension(data) { + if (data.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) return ".png"; + if (data[0] === 0xff && data[1] === 0xd8) return ".jpg"; + if (data.subarray(0, 4).toString("latin1") === "RIFF" && data.subarray(8, 12).toString("latin1") === "WEBP") return ".webp"; + if (data.subarray(0, 3).toString("latin1") === "GIF") return ".gif"; + return ".png"; +} + +async function describe(config, key, options, timeoutMs) { + const images = options.image ?? []; + if (!images.length) fail(EXIT.usage, "Give at least one --image."); + if (images.length > 8) fail(EXIT.usage, "Give at most 8 images."); + const content = [{ type: "text", text: options.prompt || "Describe this image." }]; + for (const path of images) { + const type = IMAGE_TYPES[extname(path).toLowerCase()]; + if (!type) fail(EXIT.usage, `Unsupported image type: ${path} (use PNG, JPEG, WebP or GIF).`); + const data = readInput(path, LIMITS.image, "Image"); + content.push({ type: "image_url", image_url: { url: `data:${type};base64,${data.toString("base64")}` } }); + } + const response = await call(config, key, "/chat/completions", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + model: config.model, + messages: [{ role: "user", content }], + max_tokens: positiveInt(options["max-tokens"], "max-tokens", 1024, 32768), + }), + }, timeoutMs); + const body = await json(response); + const text = body?.choices?.[0]?.message?.content; + const answer = Array.isArray(text) ? text.map((part) => part?.text ?? "").join("") : text; + if (typeof answer !== "string") fail(EXIT.unavailable, "The provider returned no text."); + process.stdout.write(answer.trim() + "\n"); +} + +async function generate(config, key, options, timeoutMs) { + if (!options.prompt) fail(EXIT.usage, "Give a --prompt."); + const count = positiveInt(options.n, "n", 1, 4); + const request = { model: config.model, prompt: options.prompt, n: count }; + if (options.size) request.size = options.size; + const response = await call(config, key, "/images/generations", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(request), + }, timeoutMs); + const body = await json(response); + const items = Array.isArray(body?.data) ? body.data : []; + if (!items.length) fail(EXIT.unavailable, "The provider returned no image."); + const directory = resolve(options.out || config.output_dir || "bootagent-output"); + const base = `image-${stamp()}`; + const saved = []; + for (const [index, item] of items.entries()) { + let data; + if (typeof item?.b64_json === "string") data = Buffer.from(item.b64_json, "base64"); + else if (typeof item?.url === "string") data = await download(item.url, timeoutMs); + else continue; + const name = items.length > 1 ? `${base}-${index + 1}` : base; + saved.push(writeOutput(join(directory, name + imageExtension(data)), data)); + } + if (!saved.length) fail(EXIT.unavailable, "The provider returned no image data."); + process.stdout.write(saved.join("\n") + "\n"); +} + +async function transcribe(config, key, options, timeoutMs) { + if (!options.audio) fail(EXIT.usage, "Give an --audio file."); + const data = readInput(options.audio, LIMITS.audio, "Audio"); + const form = new FormData(); + form.append("model", config.model); + form.append("file", new Blob([data]), basename(options.audio)); + if (options.language) form.append("language", options.language); + if (options.prompt) form.append("prompt", options.prompt); + const response = await call(config, key, "/audio/transcriptions", { method: "POST", body: form }, timeoutMs); + const body = await json(response); + if (typeof body?.text !== "string") fail(EXIT.unavailable, "The provider returned no transcript."); + process.stdout.write(body.text.trim() + "\n"); +} + +const SPEECH_FORMATS = ["mp3", "wav", "opus", "aac", "flac", "pcm"]; + +async function speak(config, key, options, timeoutMs) { + if (!options.text) fail(EXIT.usage, "Give the --text to speak."); + if (options.text.length > 4096) fail(EXIT.usage, "--text is longer than 4096 characters; split it."); + const format = (options.format || "mp3").toLowerCase(); + if (!SPEECH_FORMATS.includes(format)) fail(EXIT.usage, `--format must be one of ${SPEECH_FORMATS.join(", ")}.`); + const response = await call(config, key, "/audio/speech", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ model: config.model, input: options.text, voice: options.voice || "alloy", response_format: format }), + }, timeoutMs); + const data = Buffer.from(await response.arrayBuffer()); + if (!data.length) fail(EXIT.unavailable, "The provider returned no audio."); + if (data.length > LIMITS.speech) fail(EXIT.unavailable, "The audio is larger than expected and was not saved."); + const target = options.out && extname(options.out) + ? options.out + : join(options.out || config.output_dir || "bootagent-output", `speech-${stamp()}.${format}`); + process.stdout.write(writeOutput(target, data) + "\n"); +} + +const COMMANDS = { + vision: { name: "describe", run: describe, timeout: 120 }, + "image-generation": { name: "generate", run: generate, timeout: 300 }, + "speech-to-text": { name: "transcribe", run: transcribe, timeout: 300 }, + "text-to-speech": { name: "speak", run: speak, timeout: 120 }, +}; + +const ADAPTERS = { + vision: "openai-chat-vision", + "image-generation": "openai-images", + "speech-to-text": "openai-audio-transcriptions", + "text-to-speech": "openai-audio-speech", +}; + +async function main() { + const [major] = process.versions.node.split(".").map(Number); + if (!(major >= 18) || typeof fetch !== "function" || typeof FormData !== "function") { + fail(EXIT.setup, `Node.js 18 or later is required; this is ${process.versions.node}.`); + } + const config = loadConfig(); + const command = COMMANDS[config.capability]; + if (!command || ADAPTERS[config.capability] !== config.adapter) { + fail(EXIT.setup, `This version of the script cannot run ${config.capability} over ${config.adapter}. Reinstall the Skill from BootAgent.`); + } + const [subcommand, ...rest] = process.argv.slice(2); + if (!subcommand || subcommand === "--help" || subcommand === "help") { + process.stdout.write(`Usage: node scripts/run.mjs ${command.name} [options]\n node scripts/run.mjs check\nSee SKILL.md for the options and exit codes.\n`); + return; + } + if (subcommand === "check") { + loadKey(config); + process.stdout.write(`ready: ${config.capability} with ${config.model}\n`); + return; + } + if (subcommand !== command.name) fail(EXIT.usage, `This Skill runs "${command.name}", not "${subcommand}".`); + const options = parseArgs(rest); + if (options._.length) fail(EXIT.usage, `Unexpected argument: ${options._[0]}`); + const timeout = positiveInt(options.timeout, "timeout", command.timeout, 3600) * 1000; + const key = loadKey(config); + await command.run(config, key, options, timeout); +} + +main().then( + () => process.exit(EXIT.ok), + (error) => { + const code = error instanceof Failure ? error.code : EXIT.internal; + process.stderr.write(redact(error instanceof Failure ? error.message : `Unexpected error: ${error?.message ?? error}`) + "\n"); + process.exit(code); + }, +); diff --git a/internal/skillgen/templates/speech-to-text.md.tmpl b/internal/skillgen/templates/speech-to-text.md.tmpl new file mode 100644 index 00000000..036ac2c4 --- /dev/null +++ b/internal/skillgen/templates/speech-to-text.md.tmpl @@ -0,0 +1,20 @@ +{{template "header" .}} +## When to use + +When the user asks for the text of an audio recording: transcribing a voice note, a meeting recording or an interview. + +## Command + +Run the script in this Skill's directory (the directory containing this `SKILL.md`): + +```bash +node /scripts/run.mjs transcribe --audio [--language ] [--prompt ""] +``` + +- `--audio`: a local audio file (mp3, wav, m4a, webm and similar), at most 25 MB. +- `--language`: the spoken language as an ISO 639-1 code, for example `en` or `zh`. Leave it out to let the model detect it. +- `--prompt`: names or terms that appear in the recording, to help spelling. +- `--timeout`: seconds to wait. Default 300. + +The transcript is printed to stdout. +{{template "footer" .}} diff --git a/internal/skillgen/templates/text-to-speech.md.tmpl b/internal/skillgen/templates/text-to-speech.md.tmpl new file mode 100644 index 00000000..18fefaed --- /dev/null +++ b/internal/skillgen/templates/text-to-speech.md.tmpl @@ -0,0 +1,21 @@ +{{template "header" .}} +## When to use + +When the user asks you to turn text into spoken audio: reading out a summary, producing a voice-over draft. + +## Command + +Run the script in this Skill's directory (the directory containing this `SKILL.md`): + +```bash +node /scripts/run.mjs speak --text "" [--voice ] [--format mp3|wav|opus|aac|flac|pcm] [--out ] +``` + +- `--text`: what to say, at most 4096 characters. Split longer text into several runs. +- `--voice`: the voice name. Default `alloy`; other models may name their voices differently, and the Provider's error says so. +- `--format`: audio format. Default `mp3`. +- `--out`: a file path, or a directory. Default `bootagent-output` in the current directory. +- `--timeout`: seconds to wait. Default 120. + +The saved file path is printed to stdout. Existing files are never overwritten. +{{template "footer" .}} diff --git a/internal/skillgen/templates/vision.md.tmpl b/internal/skillgen/templates/vision.md.tmpl new file mode 100644 index 00000000..3dd4dd87 --- /dev/null +++ b/internal/skillgen/templates/vision.md.tmpl @@ -0,0 +1,20 @@ +{{template "header" .}} +## When to use + +When the user asks about the contents of an image file: reading a screenshot, finding an error message in a UI, comparing two images, describing a diagram or photo. Use it whenever your own model cannot see images. + +## Command + +Run the script in this Skill's directory (the directory containing this `SKILL.md`): + +```bash +node /scripts/run.mjs describe --image [--image ...] [--prompt ""] [--max-tokens ] +``` + +- `--image`: a local PNG, JPEG, WebP or GIF file, at most 20 MB. Repeat for up to 8 images. +- `--prompt`: what to ask about the images. Put the user's actual question here. Default: "Describe this image." +- `--max-tokens`: the longest answer to allow. Default 1024. +- `--timeout`: seconds to wait. Default 120. + +The answer is printed to stdout as plain text. +{{template "footer" .}} diff --git a/internal/skillgen/testdata/golden/image-generation/SKILL.md b/internal/skillgen/testdata/golden/image-generation/SKILL.md new file mode 100644 index 00000000..9373c7bf --- /dev/null +++ b/internal/skillgen/testdata/golden/image-generation/SKILL.md @@ -0,0 +1,47 @@ +--- +name: bootagent-image-generation +description: "Generate images from a text description through gpt-image-1 on Gateway \"Prod\": EU and save them as files. Use when the user asks you to create an image. Each run is billed: confirm with the user first." +--- + +# Image generation + +Generated by BootAgent. It calls `gpt-image-1` on the user's Gateway "Prod": EU account. BootAgent rewrites this Skill when the model or Provider changes, so do not edit it. + +## When to use + +When the user asks you to create an image: an illustration, an icon draft, a mock-up, a diagram-style picture. + +## Command + +Run the script in this Skill's directory (the directory containing this `SKILL.md`): + +```bash +node /scripts/run.mjs generate --prompt "" [--size ] [--n ] [--out ] +``` + +- `--prompt`: a full description of the image to create. +- `--size`: for example `1024x1024`. Leave it out unless the user asks; valid sizes depend on the model. +- `--n`: how many images, 1 to 4. Default 1. +- `--out`: directory for the files. Default `bootagent-output` in the current directory. +- `--timeout`: seconds to wait. Default 300. + +The saved file paths are printed to stdout, one per line. Existing files are never overwritten. + +## Rules + +- Every run costs money. Before each run, tell the user what you are about to generate and how many images, and wait for them to agree. +- Never print, echo or log environment variables, `~/.bootagent/skill-env/my-gateway.env`, or anything that could reveal the API key. The script reads the key itself; you never need it. +- Only send files the user asked about. They go to the user's own Provider and nowhere else. + +## Exit codes + +| Code | Meaning | What to tell the user | +| --- | --- | --- | +| 0 | Success | Report the result. | +| 2 | Bad arguments or input file | Fix the command, or ask the user for the right file. | +| 3 | Setup problem: Node.js older than 18, or no API key found | Ask the user to restart the Agent so it picks up `BOOTAGENT_MY_GATEWAY_API_KEY`, or to reinstall this Skill from BootAgent. | +| 4 | The Provider refused the request | Show the Provider's message. The model may not support this input; the user can choose another model in BootAgent. | +| 5 | The Provider rejected the API key | Ask the user to check the Gateway "Prod": EU key in BootAgent. | +| 6 | The Provider was unreachable, busy or timed out | Suggest trying again later. | + +`node /scripts/run.mjs check` confirms the key is available without sending a request. diff --git a/internal/skillgen/testdata/golden/image-generation/config.json b/internal/skillgen/testdata/golden/image-generation/config.json new file mode 100644 index 00000000..1bd98ed2 --- /dev/null +++ b/internal/skillgen/testdata/golden/image-generation/config.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "generator": "bootagent", + "capability": "image-generation", + "adapter": "openai-images", + "provider_id": "my-gateway", + "provider_name": "Gateway \"Prod\": EU", + "model": "gpt-image-1", + "base_url": "https://gw.example/v1", + "key_env": "BOOTAGENT_MY_GATEWAY_API_KEY", + "key_file": "~/.bootagent/skill-env/my-gateway.env", + "output_dir": "bootagent-output" +} diff --git a/internal/skillgen/testdata/golden/speech-to-text/SKILL.md b/internal/skillgen/testdata/golden/speech-to-text/SKILL.md new file mode 100644 index 00000000..8e75c6f8 --- /dev/null +++ b/internal/skillgen/testdata/golden/speech-to-text/SKILL.md @@ -0,0 +1,46 @@ +--- +name: bootagent-speech-to-text +description: "Transcribe a local audio file to text through whisper-1 on PPIO. Use when the user asks for the text of a recording." +--- + +# Speech to text + +Generated by BootAgent. It calls `whisper-1` on the user's PPIO account. BootAgent rewrites this Skill when the model or Provider changes, so do not edit it. + +## When to use + +When the user asks for the text of an audio recording: transcribing a voice note, a meeting recording or an interview. + +## Command + +Run the script in this Skill's directory (the directory containing this `SKILL.md`): + +```bash +node /scripts/run.mjs transcribe --audio [--language ] [--prompt ""] +``` + +- `--audio`: a local audio file (mp3, wav, m4a, webm and similar), at most 25 MB. +- `--language`: the spoken language as an ISO 639-1 code, for example `en` or `zh`. Leave it out to let the model detect it. +- `--prompt`: names or terms that appear in the recording, to help spelling. +- `--timeout`: seconds to wait. Default 300. + +The transcript is printed to stdout. + +## Rules + +- Each run is billed by the length of the audio. Run it when the user asks for a transcript; ask first before transcribing long or many recordings. +- Never print, echo or log environment variables, `~/.bootagent/skill-env/ppio.env`, or anything that could reveal the API key. The script reads the key itself; you never need it. +- Only send files the user asked about. They go to the user's own Provider and nowhere else. + +## Exit codes + +| Code | Meaning | What to tell the user | +| --- | --- | --- | +| 0 | Success | Report the result. | +| 2 | Bad arguments or input file | Fix the command, or ask the user for the right file. | +| 3 | Setup problem: Node.js older than 18, or no API key found | Ask the user to restart the Agent so it picks up `BOOTAGENT_PPIO_API_KEY`, or to reinstall this Skill from BootAgent. | +| 4 | The Provider refused the request | Show the Provider's message. The model may not support this input; the user can choose another model in BootAgent. | +| 5 | The Provider rejected the API key | Ask the user to check the PPIO key in BootAgent. | +| 6 | The Provider was unreachable, busy or timed out | Suggest trying again later. | + +`node /scripts/run.mjs check` confirms the key is available without sending a request. diff --git a/internal/skillgen/testdata/golden/speech-to-text/config.json b/internal/skillgen/testdata/golden/speech-to-text/config.json new file mode 100644 index 00000000..db21a42a --- /dev/null +++ b/internal/skillgen/testdata/golden/speech-to-text/config.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "generator": "bootagent", + "capability": "speech-to-text", + "adapter": "openai-audio-transcriptions", + "provider_id": "ppio", + "provider_name": "PPIO", + "model": "whisper-1", + "base_url": "https://api.ppio.com/openai/v1", + "key_env": "BOOTAGENT_PPIO_API_KEY", + "key_file": "~/.bootagent/skill-env/ppio.env", + "output_dir": "bootagent-output" +} diff --git a/internal/skillgen/testdata/golden/text-to-speech/SKILL.md b/internal/skillgen/testdata/golden/text-to-speech/SKILL.md new file mode 100644 index 00000000..15325f6d --- /dev/null +++ b/internal/skillgen/testdata/golden/text-to-speech/SKILL.md @@ -0,0 +1,47 @@ +--- +name: bootagent-text-to-speech +description: "Turn text into a spoken audio file through tts-1 on local. Use when the user asks for audio of some text. Each run is billed: confirm with the user first." +--- + +# Text to speech + +Generated by BootAgent. It calls `tts-1` on the user's local account. BootAgent rewrites this Skill when the model or Provider changes, so do not edit it. + +## When to use + +When the user asks you to turn text into spoken audio: reading out a summary, producing a voice-over draft. + +## Command + +Run the script in this Skill's directory (the directory containing this `SKILL.md`): + +```bash +node /scripts/run.mjs speak --text "" [--voice ] [--format mp3|wav|opus|aac|flac|pcm] [--out ] +``` + +- `--text`: what to say, at most 4096 characters. Split longer text into several runs. +- `--voice`: the voice name. Default `alloy`; other models may name their voices differently, and the Provider's error says so. +- `--format`: audio format. Default `mp3`. +- `--out`: a file path, or a directory. Default `bootagent-output` in the current directory. +- `--timeout`: seconds to wait. Default 120. + +The saved file path is printed to stdout. Existing files are never overwritten. + +## Rules + +- Every run costs money, billed by the length of the text. Before each run, tell the user what text you are about to speak and wait for them to agree. +- Never print, echo or log environment variables, `~/.bootagent/skill-env/local.env`, or anything that could reveal the API key. The script reads the key itself; you never need it. +- Only send files the user asked about. They go to the user's own Provider and nowhere else. + +## Exit codes + +| Code | Meaning | What to tell the user | +| --- | --- | --- | +| 0 | Success | Report the result. | +| 2 | Bad arguments or input file | Fix the command, or ask the user for the right file. | +| 3 | Setup problem: Node.js older than 18, or no API key found | Ask the user to restart the Agent so it picks up `BOOTAGENT_LOCAL_API_KEY`, or to reinstall this Skill from BootAgent. | +| 4 | The Provider refused the request | Show the Provider's message. The model may not support this input; the user can choose another model in BootAgent. | +| 5 | The Provider rejected the API key | Ask the user to check the local key in BootAgent. | +| 6 | The Provider was unreachable, busy or timed out | Suggest trying again later. | + +`node /scripts/run.mjs check` confirms the key is available without sending a request. diff --git a/internal/skillgen/testdata/golden/text-to-speech/config.json b/internal/skillgen/testdata/golden/text-to-speech/config.json new file mode 100644 index 00000000..dcdb14d0 --- /dev/null +++ b/internal/skillgen/testdata/golden/text-to-speech/config.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "generator": "bootagent", + "capability": "text-to-speech", + "adapter": "openai-audio-speech", + "provider_id": "local", + "provider_name": "local", + "model": "tts-1", + "base_url": "http://127.0.0.1:8000/v1", + "key_env": "BOOTAGENT_LOCAL_API_KEY", + "key_file": "~/.bootagent/skill-env/local.env", + "output_dir": "bootagent-output" +} diff --git a/internal/skillgen/testdata/golden/vision/SKILL.md b/internal/skillgen/testdata/golden/vision/SKILL.md new file mode 100644 index 00000000..ebc27b2e --- /dev/null +++ b/internal/skillgen/testdata/golden/vision/SKILL.md @@ -0,0 +1,46 @@ +--- +name: bootagent-vision +description: "Look at local image files (screenshots, photos, diagrams) and answer questions about them through qwen/qwen3-vl-8b on Novita. Use when the user asks about an image and you cannot see images yourself." +--- + +# Image understanding + +Generated by BootAgent. It calls `qwen/qwen3-vl-8b` on the user's Novita account. BootAgent rewrites this Skill when the model or Provider changes, so do not edit it. + +## When to use + +When the user asks about the contents of an image file: reading a screenshot, finding an error message in a UI, comparing two images, describing a diagram or photo. Use it whenever your own model cannot see images. + +## Command + +Run the script in this Skill's directory (the directory containing this `SKILL.md`): + +```bash +node /scripts/run.mjs describe --image [--image ...] [--prompt ""] [--max-tokens ] +``` + +- `--image`: a local PNG, JPEG, WebP or GIF file, at most 20 MB. Repeat for up to 8 images. +- `--prompt`: what to ask about the images. Put the user's actual question here. Default: "Describe this image." +- `--max-tokens`: the longest answer to allow. Default 1024. +- `--timeout`: seconds to wait. Default 120. + +The answer is printed to stdout as plain text. + +## Rules + +- Each run is a billed request to the user's Provider. Run it when the user asks about specific images; do not run it speculatively over many files without asking. +- Never print, echo or log environment variables, `~/.bootagent/skill-env/novita.env`, or anything that could reveal the API key. The script reads the key itself; you never need it. +- Only send files the user asked about. They go to the user's own Provider and nowhere else. + +## Exit codes + +| Code | Meaning | What to tell the user | +| --- | --- | --- | +| 0 | Success | Report the result. | +| 2 | Bad arguments or input file | Fix the command, or ask the user for the right file. | +| 3 | Setup problem: Node.js older than 18, or no API key found | Ask the user to restart the Agent so it picks up `BOOTAGENT_NOVITA_API_KEY`, or to reinstall this Skill from BootAgent. | +| 4 | The Provider refused the request | Show the Provider's message. The model may not support this input; the user can choose another model in BootAgent. | +| 5 | The Provider rejected the API key | Ask the user to check the Novita key in BootAgent. | +| 6 | The Provider was unreachable, busy or timed out | Suggest trying again later. | + +`node /scripts/run.mjs check` confirms the key is available without sending a request. diff --git a/internal/skillgen/testdata/golden/vision/config.json b/internal/skillgen/testdata/golden/vision/config.json new file mode 100644 index 00000000..a0901bba --- /dev/null +++ b/internal/skillgen/testdata/golden/vision/config.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "generator": "bootagent", + "capability": "vision", + "adapter": "openai-chat-vision", + "provider_id": "novita", + "provider_name": "Novita", + "model": "qwen/qwen3-vl-8b", + "base_url": "https://api.novita.ai/openai/v1", + "key_env": "BOOTAGENT_NOVITA_API_KEY", + "key_file": "~/.bootagent/skill-env/novita.env", + "output_dir": "bootagent-output" +} From 0c1d90404ea035bbbd8de5314f0020137d4a41b0 Mon Sep 17 00:00:00 2001 From: yujiezhang-ops Date: Fri, 9 Oct 2026 17:06:34 +0800 Subject: [PATCH 07/18] feat: add native speech and async video generation Skills Add the routes Novita, PPIO and JieKou.AI document for one shared native API family: synchronous MiniMax Speech (novita-audio) and async submit, poll and download (novita-async). run.mjs gains a native speak and a video command that paces polling by the provider's eta, streams the result to disk without the key, and exits 7 with the task ID when a run outlasts its timeout, so video-result can collect it without paying again. The catalog now lists only models whose request was read from vendor docs: MiniMax Speech on all three, GPT Image and two text-to-video models on JieKou.AI, and ming-image on Novita. A catalog entry outranks a declaration for the capabilities it names. PPIO and Novita no longer document video generation, so it is marked unsupported for both and the ppio-video adapter stays reserved. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 6 +- README_ZH.md | 6 +- docs/multimodal-skills-design.md | 94 ++++++-- .../BootAgent/internal/app/models.ts | 5 + .../BootAgent/internal/modality/models.ts | 5 + frontend/src/backend/wails.ts | 1 + .../src/components/MultimodalSection.test.tsx | 32 ++- frontend/src/components/MultimodalSection.tsx | 10 +- frontend/src/components/multimodalPrompts.ts | 4 + frontend/src/i18n.tsx | 4 + frontend/src/types/api.ts | 5 +- internal/app/multimodal.go | 19 +- internal/app/multimodal_install.go | 5 +- internal/app/multimodal_install_test.go | 92 +++++++ internal/catalog/modalities.go | 16 +- internal/modality/capability.go | 38 ++- internal/modality/resolve.go | 73 +++++- internal/modality/resolve_test.go | 78 +++++- internal/provider/client.go | 85 +++++++ internal/provider/client_test.go | 42 ++++ internal/skillgen/native_script_test.go | 224 ++++++++++++++++++ internal/skillgen/skillgen.go | 72 +++++- internal/skillgen/skillgen_test.go | 27 +++ internal/skillgen/templates/common.md.tmpl | 3 + internal/skillgen/templates/run.mjs | 203 ++++++++++++++-- .../skillgen/templates/text-to-speech.md.tmpl | 13 +- .../templates/video-generation.md.tmpl | 40 ++++ .../golden/text-to-speech-native/SKILL.md | 48 ++++ .../golden/text-to-speech-native/config.json | 14 ++ .../testdata/golden/video-generation/SKILL.md | 55 +++++ .../golden/video-generation/config.json | 19 ++ manifests/modalities.lock.json | 161 +++++++++++-- 32 files changed, 1395 insertions(+), 104 deletions(-) create mode 100644 internal/skillgen/native_script_test.go create mode 100644 internal/skillgen/templates/video-generation.md.tmpl create mode 100644 internal/skillgen/testdata/golden/text-to-speech-native/SKILL.md create mode 100644 internal/skillgen/testdata/golden/text-to-speech-native/config.json create mode 100644 internal/skillgen/testdata/golden/video-generation/SKILL.md create mode 100644 internal/skillgen/testdata/golden/video-generation/config.json diff --git a/README.md b/README.md index 5b15d7a2..bd2f1588 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ Launch BootAgent - Keep a local Skill library, scan Skills already present on supported Agents, and choose which Agents receive each Skill. - See which models on your keyed Providers can understand images, video or audio, or generate images, and where each finding comes from (declared by the Provider, inferred from the model name, or probed). Listing models is free; a probe is a real request and is sent only when you click it, and an image-generation probe asks first. -- Install a multimodal Skill (image understanding, image generation, speech-to-text, text-to-speech) into Claude Code or Codex, with example prompts to copy. Its script needs Node.js 18 or later. BootAgent writes the Provider key into the Agent's own configuration as `BOOTAGENT__API_KEY`, so the Agent and its model can read it; the install confirmation says so. Uninstalling the last Skill that uses a Provider removes the variable. +- Install a multimodal Skill (image understanding, image generation, speech-to-text, text-to-speech, video generation) into Claude Code or Codex, with example prompts to copy. Built-in Providers add the models their docs name: MiniMax Speech on Novita, PPIO and JieKou.AI, GPT Image and async text-to-video on JieKou.AI. Video is never probed, because a probe would be a full billed generation. Its script needs Node.js 18 or later. BootAgent writes the Provider key into the Agent's own configuration as `BOOTAGENT__API_KEY`, so the Agent and its model can read it; the install confirmation says so. Uninstalling the last Skill that uses a Provider removes the variable. - Discover MCP servers from initialized Claude Code, Codex, OpenCode, Kilo CLI, and Hermes installations. - Store MCP servers in a local user-level Registry, select sync targets explicitly, and apply changes only when you are ready. @@ -135,7 +135,9 @@ BootAgent does not redistribute Agent packages and does not bundle Node.js, Git, | Symptom | First check | | --- | --- | | Agent is not listed | Refresh Environment overview, check that the command is on `PATH`, or use the Agent's documented manual install path. | -| Provider connection check fails | Confirm the endpoint and model match the Agent's protocol; an endpoint that exposes only another API is rejected before writing configuration. | +| Provider connection check fails | Confirm the endpoint and model match the Agent's protocol. Use Check model for a saved Profile; saving or applying a Profile does not automatically send a model request. | +| Codex reports `does not support endpoint: responses` | Codex is configured to use Responses. Select a model that supports it, or use the API protocol adapter with a working Chat Completions Profile and apply its generated Codex Profile. A model appearing in `/models` does not prove Responses support. | +| Windows detects a desktop app but its version is unavailable | BootAgent retries the registered package's metadata through Windows PackageManager. If the metadata remains inaccessible, launch is still available; this warning is separate from model API compatibility. | | Marketplace looks incomplete | Check the source status and refresh. A bundled snapshot remains available when an online source is unavailable. | | Import reports a conflict | Review the preview, choose skip or overwrite, and apply only after checking the affected resource names. | | BootAgent cannot check for an update | In Settings, switch between GitHub and the domestic Gitee mirror, then retry. Existing installation files are not replaced by a failed check. | diff --git a/README_ZH.md b/README_ZH.md index aad6f479..52f87913 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -62,7 +62,7 @@ BootAgent 是一个本地桌面工作台，用来统一管理 AI 编程 Agent。 - 在本机维护 Skill 管理库，扫描支持的 Agent 已有的 Skill，并选择每个 Skill 要同步到哪些 Agent。 - 查看已保存 API Key 的 Provider 中哪些模型能理解图片、视频或音频，或能生成图片，以及每项结论的来源（Provider 声明、按模型名推断或实测）。列出模型不收费；实测是一次真实请求，只在你点击时发出，图片生成实测会先确认。 -- 把多模态 Skill（图片理解、图片生成、语音识别、语音合成）安装到 Claude Code 或 Codex，并提供可复制的示例提示词。Skill 脚本需要 Node.js 18 或更高版本。BootAgent 会把 Provider 的 Key 以 `BOOTAGENT__API_KEY` 写进 Agent 自己的配置，因此 Agent 及其模型能读到它，安装确认里会说明。卸载最后一个用到该 Provider 的 Skill 时，这个变量会被移除。 +- 把多模态 Skill（图片理解、图片生成、语音识别、语音合成、视频生成）安装到 Claude Code 或 Codex，并提供可复制的示例提示词。内置 Provider 会补充其文档中列出的模型：Novita、PPIO 和 JieKou.AI 的 MiniMax Speech，JieKou.AI 的 GPT Image 与异步文生视频。视频生成从不实测，因为一次实测就是一次完整的计费生成。Skill 脚本需要 Node.js 18 或更高版本。BootAgent 会把 Provider 的 Key 以 `BOOTAGENT__API_KEY` 写进 Agent 自己的配置，因此 Agent 及其模型能读到它，安装确认里会说明。卸载最后一个用到该 Provider 的 Skill 时，这个变量会被移除。 - 从已初始化的 Claude Code、Codex、OpenCode、Kilo CLI 和 Hermes 中发现 MCP 服务器。 - 使用本机用户级 Registry 保存 MCP 服务器，明确选择同步目标，并在确认后再应用改动。 @@ -133,7 +133,9 @@ BootAgent 不重新分发 Agent 包，也不捆绑 Node.js、Git、WebView 或 A | 现象 | 先检查什么 | | --- | --- | | 找不到 Agent | 刷新“环境总览”，确认命令在 `PATH` 中，或按 Agent 官方指引手动安装后再次检测。 | -| Provider 连接检查失败 | 确认端点和模型与 Agent 使用的协议一致；只支持另一种 API 的端点会在写入前被拒绝。 | +| Provider 连接检查失败 | 确认端点和模型与 Agent 使用的协议一致。对已保存的 Profile 使用“检测模型”；保存或应用 Profile 不会自动发送模型请求。 | +| Codex 报错 `does not support endpoint: responses` | Codex 的配置使用 Responses。选择支持它的模型，或在 API 协议适配中选择可用的 Chat Completions Profile，再应用生成的 Codex Profile。模型出现在 `/models` 中并不证明它支持 Responses。 | +| Windows 检测到桌面应用但版本不可用 | BootAgent 会通过 Windows PackageManager 重试读取已注册应用包的元数据。如果仍无法读取，仍可尝试启动；这个提示与模型 API 兼容性无关。 | | 工具市场内容不完整 | 查看来源状态并点击刷新；在线来源不可用时仍会显示内置目录。 | | 导入提示冲突 | 查看导入预览，选择跳过或覆盖，并确认受影响的资源名称。 | | BootAgent 检查更新失败 | 在“设置”中切换 GitHub 与国内 Gitee 镜像后重试；检查失败不会替换现有安装文件。 | diff --git a/docs/multimodal-skills-design.md b/docs/multimodal-skills-design.md index 0fc91366..b03b09d3 100644 --- a/docs/multimodal-skills-design.md +++ b/docs/multimodal-skills-design.md @@ -1,6 +1,6 @@ # Multimodal Skills: Design -Status: In progress. Phase 1 (detection) and Phase 2 (generation and install for Claude Code and Codex) are implemented. Branch `codex/multimodal-skills`. +Status: In progress. Phases 1 to 3 are implemented: detection; generation and install for Claude Code and Codex; native speech and async video on the routes the vendors document. Branch `codex/multimodal-skills`. ## Goal @@ -35,22 +35,54 @@ and choosing between several Providers for the same capability. ## Provider landscape -Endpoints differ by capability far more than chat does. Only the rows marked -verified were confirmed from the Provider's own documentation; the rest are -inferences that must be checked before the matching adapter is written. +Endpoints differ by capability far more than chat does. Every route below was +read from the vendor's own documentation on 2026-10-09; the request path comes +from the page's API playground, since the Markdown exports omit it. | Provider | Image understanding | Image / video generation | Audio | | --- | --- | --- | --- | -| DeepSeek | `deepseek-flash` takes images on Chat Completions (verified) | None | None | -| Moonshot | `kimi-k3`, `kimi-k2.6` and the `kimi-k2.7-code` models take image and video input on Chat Completions (verified) | None | None | -| Novita | Vision chat models on the OpenAI route | Async: `POST /v3/async/` returns a `task_id`; `GET /v3/async/task-result` returns `images[]`, `videos[]` or `audios[]` with URLs that expire, 3600 s in the example (verified). Some image models also serve `/openai/v1/images/generations`. | Native routes, not OpenAI `/audio/*`: for example `/v4beta/txt2speech` and the synchronous GLM TTS and ASR endpoints (verified) | -| PPIO | Vision chat models on the OpenAI route | `POST /v3/video/create` for unified video generation (verified); task polling assumed to follow Novita's `/v3/async/task-result` with the same `TASK_STATUS_*` values (unverified) | Assumed to mirror Novita (unverified) | -| JieKou.AI | Gateway that reports text, vision, audio and video support (unverified, no endpoint pages found) | Unknown | Unknown | +| DeepSeek | `deepseek-flash` takes images on Chat Completions | None | None | +| Moonshot | `kimi-k3`, `kimi-k2.6` and the `kimi-k2.7-code` models take image and video input on Chat Completions | None | None | +| Novita | Vision chat models on the OpenAI route | `ming-image-0.1-design` on `POST /openai/v1/images/generations`. No video generation model is documented any more. | MiniMax Speech 2.8 Turbo and HD: synchronous `POST /v3/`, asynchronous `POST /v3/async/`. Fish Audio on `POST /v4beta/txt2speech`. No speech-to-text route is documented. | +| PPIO | Vision chat models on the OpenAI route | None documented. The unified video route (`/v3/video/create`) that was documented when this design began now returns 404 and is in neither the sitemap nor the docs index. | MiniMax Speech 2.8 Turbo and HD on the same paths as Novita | +| JieKou.AI | Vision chat models on the OpenAI route | GPT Image (`gpt-image-2`, `gpt-image-2-oai`, `gpt-image-2.5-flare-oai`, `gpt-image-2.5-sunburst-oai`) on `POST /openai/v1/images/generations`. Async video, for example `POST /v3/async/wan2.7-t2v` and `POST /v3/async/kling-v3.0-4k-t2v`, returning a `task_id`. | MiniMax Speech 2.8 Turbo and HD on the same paths as Novita | + +All three gateways share one native API family. `GET /v3/async/task-result?task_id=` +answers with `task.status` (`TASK_STATUS_QUEUED`, `TASK_STATUS_PROCESSING`, +`TASK_STATUS_SUCCEED`, `TASK_STATUS_FAILED`), `task.reason`, `task.eta`, and +`images[]`, `videos[]` or `audios[]` whose `*_url` expires after `*_url_ttl` +seconds (3600 by default; the TTL is an integer for images and a string for video +and audio). The synchronous MiniMax speech route returns the audio as hex in +`data.audio` and reports refusals in `base_resp.status_code` under HTTP 200. The catalog's `base_url` is the OpenAI-compatible route (for example `https://api.novita.ai/openai`). Native async and audio routes sit at the host root, so the modality catalog carries its own native base per Provider instead -of deriving it from `base_url`. +of deriving it from `base_url`: `https://api.novita.ai`, `https://api.ppio.com` +and `https://api.highwayapi.ai`. + +### What Phase 3 found and how it was handled + +- Because the native API family is shared, the adapters are named after Novita, + where it was first documented, and serve all three: `novita-audio` for the + synchronous speech route, `novita-async` for submit, poll and download. +- `ppio-video` is reserved and not implemented: PPIO no longer documents a + video route. The catalog marks video generation unsupported for PPIO and + Novita, and the UI says so for each. +- JieKou.AI's generation endpoints, which the first draft could not find, are now + documented, so its async video and GPT Image models are in the catalog. +- Speech-to-text has no native route on any of the three; it stays on the + OpenAI-compatible route with a hand-entered model. +- JieKou.AI's GPT Image models are on the images route, not chat output, so they + are catalog entries on `openai-images`. Gemini image output and chat audio + input are still undocumented anywhere (JieKou.AI's Gemini page covers text and + tools only), so `openai-chat-image-output` and `openai-chat-audio-input` stay + uninstallable with no probe. +- The catalog lists only models whose request body was read: two video models, + four GPT Image models, `ming-image-0.1-design` and the two MiniMax Speech + models. A catalog entry may carry `options` (the request fields beyond the + prompt it documents, from a closed vocabulary) and `recommended` (preselected + in the UI, usually the cheaper of two). ## Architecture @@ -98,9 +130,16 @@ listing, so they were not sampled. For each model ID from `ListModels`, the first source that answers wins: 1. A probe verdict in the capability cache. -2. The model's declared `input_modalities` / `output_modalities`, when present. -3. An ID rule from the table below. -4. Unknown: not offered. +2. A catalog entry for the model, for the capabilities it names. +3. The model's declared `input_modalities` / `output_modalities`, when present. +4. An ID rule from the table below. +5. Unknown: not offered. + +The catalog sits above the declaration, which the first draft did not have: a +catalog entry names a verified route, while a declaration only says a model +outputs images, not how a gateway returns them. `ming-image-0.1-design` is +declared with image output on Novita and is reached on the images route; the +declaration alone would have put it on the unverified chat route. Async generation and audio models never appear in `/models`, so they come only from the catalog's per-Provider generation entries, which also name the adapter. @@ -216,6 +255,7 @@ refused / key / transient / network classification from | Speech-to-text | 0.5 s of silent 16 kHz WAV to `/audio/transcriptions` | Minimal | | Text-to-speech | The text `ok` to `/audio/speech`, voice `alloy` | Minimal | | Image generation | One image from `/images/generations`, no size sent | One image; the button says so and a dialog asks first, and the backend refuses the probe without that confirmation | +| Text-to-speech (native) | The text `ok` to `POST /v3/` with a documented system voice; the verdict is read from `base_resp.status_code` too | Minimal | | Video generation | Not probed | A probe is a full billed generation that cannot be cancelled reliably, so this capability is catalog-only | | Video understanding | Native video models: catalog-only. Frame sampling: inherits the image-understanding verdict | None | @@ -310,12 +350,21 @@ or a colon cannot break the frontmatter. | Image understanding | `describe --image … [--prompt] [--max-tokens]` | The answer on stdout | | Image generation | `generate --prompt [--size] [--n 1-4] [--out ]` | Saved file paths on stdout | | Speech-to-text | `transcribe --audio [--language] [--prompt]` | The transcript on stdout | -| Text-to-speech | `speak --text [--voice] [--format] [--out]` | The saved file path on stdout | +| Text-to-speech | `speak --text [--voice] [--format] [--out]` (native route: also `--speed`) | The saved file path on stdout | +| Video generation | `video --prompt [options the model documents] [--out]`, and `video-result --task ` | The saved file path on stdout; `task: ` on stderr | Every Skill also has `check`, which confirms the key is available without sending a request. Exit codes: 0 success, 2 bad arguments or input, 3 setup (Node older than 18, no key), 4 refused by the Provider, 5 key rejected, 6 -unreachable, busy or timed out. Outputs default to `bootagent-output/` in the +unreachable, busy or timed out, 7 (video only) submitted but not finished in time. + +Video polls the task with the Provider's `eta` as the pace, clamped to 1 to 15 +s, until 10 minutes have passed by default. A busy or unreachable poll is +retried; a refused or failed task is final, since retrying spends money. A +timeout prints the task ID so `video-result` can collect it later without +submitting again, and `SKILL.md` tells the Agent never to retry on its own. +Results are streamed to disk as they download, without the key, and a partial +file is removed on failure. Outputs default to `bootagent-output/` in the working directory and never overwrite an existing file. Result URLs are downloaded without the key, because they may be on another host. @@ -467,13 +516,16 @@ those Agents' instruction files. 2. Generator, install and env delivery for Claude Code and Codex: image understanding, image generation, speech-to-text and text-to-speech over the OpenAI-compatible adapters. -3. Novita and PPIO async adapters: image and video generation, native audio. +3. Native speech and async video on the routes Novita, PPIO and JieKou.AI + document; catalog image models on the images route. PPIO video is not + implemented (no documented route). 4. Video understanding; OpenCode, Hermes and the guide for Agents without Skills. ## Open questions -- PPIO's task-result path and status values, and every JieKou.AI endpoint, need - confirming against their documentation or a real call. +- Gemini image output and chat audio input need a documented request and + response format before their adapters can be written. A native speech-to-text + route, and PPIO and Novita video generation, need documenting by the vendors. - How OpenCode and Hermes pass configured environment variables to their shell tools. - Hermes is installed by its official script, so Node may be absent. If so, should @@ -486,7 +538,7 @@ those Agents' instruction files. - [DeepSeek: V4-Flash-Vision release](https://api-docs.deepseek.com/news/news260821/), [vision guide](https://api-docs.deepseek.com/guides/vision/), [updates](https://api-docs.deepseek.com/updates) - [Kimi: vision models](https://platform.kimi.ai/docs/guide/use-kimi-vision-model), [API overview](https://platform.kimi.ai/docs/overview) -- [Novita: task result](https://docs.novita.ai/api-reference/model-apis-task-result), [webhook](https://novita.ai/docs/api-reference/model-apis-webhook), [Wan 2.7 text-to-video](https://novita.ai/docs/de/api-reference/model-apis-wan2.7-t2v), [Fish Audio TTS](https://docs.novita.ai/api-reference/model-apis-fish-audio-text-to-speech), [GLM TTS and ASR](https://blogs.novita.ai/glm-tts-asr-voice-clone-api-quick-start/) -- [PPIO: unified video generation](https://ppio.com/docs/models/reference-unified-video-generation) -- [JieKou.AI docs](https://docs.jiekou.ai) +- [Novita: task result](https://docs.novita.ai/api-reference/model-apis-task-result), [MiniMax Speech 2.8 Turbo](https://docs.novita.ai/api-reference/model-apis-minimax-speech-2.8-turbo), [MiniMax Speech 2.8 Turbo async](https://docs.novita.ai/api-reference/model-apis-minimax-speech-2.8-turbo-async), [Ming Image](https://docs.novita.ai/api-reference/model-apis-ming-image-txt2img), [Fish Audio TTS](https://docs.novita.ai/api-reference/model-apis-fish-audio-text-to-speech) +- [PPIO: task result](https://ppio.com/docs/models/reference-get-async-task-result), [MiniMax Speech 2.8 Turbo](https://ppio.com/docs/models/reference-minimax-speech-2.8-turbo) +- [JieKou.AI: task result](https://docs.jiekou.ai/docs/models/reference-get-async-task-result), [Wan 2.7 text-to-video](https://docs.jiekou.ai/docs/models/reference-wan2.7-t2v), [Kling v3.0 4K text-to-video](https://docs.jiekou.ai/docs/models/reference-kling-v3.0-4k-t2v), [GPT Image](https://docs.jiekou.ai/docs/models/reference-gpt-image), [Gemini](https://docs.jiekou.ai/docs/providers/gemini), [MiniMax Speech 2.8 Turbo](https://docs.jiekou.ai/docs/models/reference-minimax-speech-2.8-turbo) - [Codex: configuration reference](https://developers.openai.com/codex/config-reference) diff --git a/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/app/models.ts b/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/app/models.ts index d1274234..b6ee9fd7 100644 --- a/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/app/models.ts +++ b/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/app/models.ts @@ -454,6 +454,11 @@ export interface MultimodalProvider { */ "keyEnv": string; + /** + * Unsupported lists capabilities this Provider documents no API for. + */ + "unsupported": string[] | null; + /** * Listed is false when /models could not be read. Message and ErrorCode * then say why; catalog generation models are still listed, since they do diff --git a/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/modality/models.ts b/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/modality/models.ts index d9a8713a..0b256066 100644 --- a/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/modality/models.ts +++ b/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/modality/models.ts @@ -27,6 +27,11 @@ export interface Capability { * Probe is the live check available for this capability; see ProbeKind. */ "probe": string; + + /** + * Recommended marks the catalog's preferred model for this capability. + */ + "recommended"?: boolean; } /** diff --git a/frontend/src/backend/wails.ts b/frontend/src/backend/wails.ts index edce6903..af0d04e9 100644 --- a/frontend/src/backend/wails.ts +++ b/frontend/src/backend/wails.ts @@ -358,6 +358,7 @@ export const wailsApi = { call(() => MultimodalService.Detect()).then((detection) => ({ providers: (detection?.providers ?? []).map((entry) => ({ ...entry, + unsupported: entry.unsupported ?? [], models: (entry.models ?? []).map((model) => ({ ...model, capabilities: model.capabilities ?? [] })), })), installed: (detection?.installed ?? []).map((entry) => ({ ...entry, agents: entry.agents ?? [] })), diff --git a/frontend/src/components/MultimodalSection.test.tsx b/frontend/src/components/MultimodalSection.test.tsx index fc725de5..ca639c20 100644 --- a/frontend/src/components/MultimodalSection.test.tsx +++ b/frontend/src/components/MultimodalSection.test.tsx @@ -32,6 +32,7 @@ const gateway: MultimodalProvider = { id: "novita", name: "Novita", keyEnv: "BOOTAGENT_NOVITA_API_KEY", + unsupported: [], listed: true, models: [ { @@ -83,6 +84,14 @@ describe("groupByCapability", () => { }); }); +describe("recommended models", () => { + it("preselects the catalog's recommended model among equally usable ones", () => { + const speech = (id: string, recommended: boolean) => ({ id, capabilities: [{ id: "text-to-speech" as const, source: "catalog" as const, adapter: "novita-audio", supported: true, installable: true, probe: "minimal" as const, recommended }] }); + const groups = groupByCapability({ ...gateway, models: [speech("minimax-speech-2.8-hd", false), speech("minimax-speech-2.8-turbo", true)] }); + expect(groups[0].options[0].model).toBe("minimax-speech-2.8-turbo"); + }); +}); + describe("installLabel", () => { it("never calls a guess installable, and says what would settle it", () => { const [guess, video] = gateway.models[0].capabilities; @@ -125,7 +134,7 @@ describe("groupInstalled", () => { describe("example prompts", () => { it("has bilingual prompts for every capability that can be installed", () => { - for (const capability of ["vision", "image-generation", "speech-to-text", "text-to-speech"] as const) { + for (const capability of ["vision", "image-generation", "speech-to-text", "text-to-speech", "video-generation"] as const) { const prompts = EXAMPLE_PROMPTS[capability] ?? []; expect(prompts.length).toBeGreaterThanOrEqual(2); for (const prompt of prompts) { @@ -238,6 +247,25 @@ describe("MultimodalSection", () => { expect(screen.getAllByRole("button", { name: /^实测$/ })).toHaveLength(1); }); + it("says which capabilities a Provider documents no API for", async () => { + detectMultimodal.mockResolvedValue(detection({ providers: [{ ...gateway, unsupported: ["video-generation"] }] })); + renderSection(); + expect(await screen.findByText("视频生成：该模型服务目前没有可核实的公开接口，暂不支持。")).toBeTruthy(); + }); + + it("offers install without a probe for a catalog video model", async () => { + detectMultimodal.mockResolvedValue(detection({ + generatable: [...GENERATABLE, "novita-async"], + providers: [{ ...gateway, models: [{ id: "wan2.7-t2v", capabilities: [{ id: "video-generation", source: "catalog", adapter: "novita-async", supported: true, installable: true, probe: "", recommended: true }] }] }], + })); + renderSection(); + expect(await screen.findByLabelText("视频生成")).toBeTruthy(); + expect(screen.getByText("内置目录")).toBeTruthy(); + const row = screen.getByLabelText("视频生成").closest("li") as HTMLElement; + expect(within(row).queryByRole("button", { name: /^实测$/ })).toBeNull(); + expect(within(row).getByRole("button", { name: /^安装$/ })).toBeTruthy(); + }); + it("points to the Providers page when no key is saved", async () => { detectMultimodal.mockResolvedValue(detection({ providers: [] })); renderSection(); @@ -245,7 +273,7 @@ describe("MultimodalSection", () => { }); it("reports a failed listing without hiding the Provider", async () => { - detectMultimodal.mockResolvedValue(detection({ providers: [{ id: "deepseek", name: "DeepSeek", keyEnv: "BOOTAGENT_DEEPSEEK_API_KEY", listed: false, message: "API key was rejected (401).", errorCode: "API_KEY_REJECTED", models: [] }] })); + detectMultimodal.mockResolvedValue(detection({ providers: [{ id: "deepseek", name: "DeepSeek", keyEnv: "BOOTAGENT_DEEPSEEK_API_KEY", unsupported: [], listed: false, message: "API key was rejected (401).", errorCode: "API_KEY_REJECTED", models: [] }] })); renderSection(); expect(await screen.findByText(/无法读取模型列表/)).toBeTruthy(); }); diff --git a/frontend/src/components/MultimodalSection.tsx b/frontend/src/components/MultimodalSection.tsx index 2f2dea70..2e809e87 100644 --- a/frontend/src/components/MultimodalSection.tsx +++ b/frontend/src/components/MultimodalSection.tsx @@ -54,8 +54,9 @@ export interface CapabilityGroup { /** * Regroups one Provider's models by capability. Within a capability, models - * that can be installed come first, then by strength of evidence, then by ID, - * so the first option is the sensible default. + * that can be installed come first, then the catalog's recommended one, then + * by strength of evidence, then by ID, so the first option is the sensible + * default. */ export function groupByCapability(entry: MultimodalProvider): CapabilityGroup[] { const groups = new Map(); @@ -71,6 +72,8 @@ export function groupByCapability(entry: MultimodalProvider): CapabilityGroup[] options: (groups.get(id) ?? []).sort((left, right) => { const usable = Number(right.capability.installable) - Number(left.capability.installable); if (usable) return usable; + const recommended = Number(Boolean(right.capability.recommended)) - Number(Boolean(left.capability.recommended)); + if (recommended) return recommended; const evidence = SOURCE_RANK[left.capability.source] - SOURCE_RANK[right.capability.source]; return evidence || left.model.localeCompare(right.model); }), @@ -290,6 +293,9 @@ export function MultimodalSection({ agentLabels = {}, onSkillsChanged }: Multimo {entry.name || entry.id} {!entry.listed ? {t("无法读取模型列表：{message}", { message: entry.message || entry.errorCode || "" })} : null} + {entry.unsupported.length ? ( +

{t("{capabilities}：该模型服务目前没有可核实的公开接口，暂不支持。", { capabilities: entry.unsupported.map((id) => t(CAPABILITY_LABELS[id])).join(separator) })}

+ ) : null} {!groups.length ? (

{entry.listed ? t("没有找到多模态模型。") : t("可以稍后重新检测。")}

) : ( diff --git a/frontend/src/components/multimodalPrompts.ts b/frontend/src/components/multimodalPrompts.ts index bddddefa..1c0bfb1d 100644 --- a/frontend/src/components/multimodalPrompts.ts +++ b/frontend/src/components/multimodalPrompts.ts @@ -25,6 +25,10 @@ export const EXAMPLE_PROMPTS: Partial & { capabilities: MultimodalCapability[] }; -export type MultimodalProvider = Omit & { models: MultimodalModel[] }; +export type MultimodalProvider = Omit & { + models: MultimodalModel[]; + unsupported: MultimodalCapabilityId[]; +}; export type InstalledMultimodal = Omit & { capability: MultimodalCapabilityId; agents: string[]; diff --git a/internal/app/multimodal.go b/internal/app/multimodal.go index 5d3b7eb0..23734823 100644 --- a/internal/app/multimodal.go +++ b/internal/app/multimodal.go @@ -42,6 +42,8 @@ type MultimodalProvider struct { // KeyEnv is the variable an install writes this Provider's key into, so // the install confirmation can name it. KeyEnv string `json:"keyEnv"` + // Unsupported lists capabilities this Provider documents no API for. + Unsupported []string `json:"unsupported"` // Listed is false when /models could not be read. Message and ErrorCode // then say why; catalog generation models are still listed, since they do // not come from /models. @@ -139,7 +141,10 @@ func (u *UseCases) DetectMultimodal(ctx context.Context) (MultimodalDetection, e } func (u *UseCases) detectProvider(ctx context.Context, rules modality.Rules, target provider.Entry) MultimodalProvider { - result := MultimodalProvider{ID: target.ID, Name: target.Name, KeyEnv: configWriter.SkillEnvName(target.ID), Models: []modality.Model{}} + result := MultimodalProvider{ID: target.ID, Name: target.Name, KeyEnv: configWriter.SkillEnvName(target.ID), Unsupported: rules.Unsupported(target.ID), Models: []modality.Model{}} + if result.Unsupported == nil { + result.Unsupported = []string{} + } listing, err := u.provider.ListModels(ctx, "custom", target.APIKey, target.BaseURL) var entries []provider.ModelEntry switch { @@ -201,8 +206,18 @@ func (u *UseCases) ProbeMultimodal(ctx context.Context, providerID, model, capab check.Status, check.Reason = ModelCheckUnverified, "no_key" return check, nil } + base := target.BaseURL var probe func(context.Context, string, string, string) (provider.ProbeResult, error) switch adapter { + case modality.AdapterNovitaAudio: + rules, err := modality.Load() + if err != nil { + return nil, err + } + if base = rules.NativeBaseURL(target.ID); base == "" { + return nil, oneerrors.New(oneerrors.InvalidRequest, "This Provider has no native speech route") + } + probe = u.provider.ProbeNativeSpeech case modality.AdapterChatVision: probe = u.provider.ProbeVision case modality.AdapterAudioTranscriptions: @@ -214,7 +229,7 @@ func (u *UseCases) ProbeMultimodal(ctx context.Context, providerID, model, capab default: return nil, oneerrors.New(oneerrors.InvalidRequest, "No live check is available for "+adapter) } - result, err := probe(ctx, target.APIKey, model, target.BaseURL) + result, err := probe(ctx, target.APIKey, model, base) if ctx.Err() != nil { return nil, contextError(ctx, "Multimodal probe was cancelled") } diff --git a/internal/app/multimodal_install.go b/internal/app/multimodal_install.go index 7dd3bcfe..20eb4315 100644 --- a/internal/app/multimodal_install.go +++ b/internal/app/multimodal_install.go @@ -123,6 +123,9 @@ func (u *UseCases) InstallMultimodal(ctx context.Context, request MultimodalInst Capability: capability, Adapter: found.Adapter, ProviderID: target.ID, ProviderName: target.Name, Model: model, BaseURL: target.BaseURL, NativeBaseURL: rules.NativeBaseURL(target.ID), } + if generation, ok := rules.Generation(target.ID, model); ok && generation.Adapter == found.Adapter { + spec.Options = generation.Options + } u.writeMu.Lock() defer u.writeMu.Unlock() @@ -499,7 +502,7 @@ func (u *UseCases) refreshGeneratedSkillsLocked(ctx context.Context, target prov spec := skillgen.Spec{ Capability: variant.config.Capability, Adapter: variant.config.Adapter, ProviderID: target.ID, ProviderName: target.Name, Model: variant.config.Model, - BaseURL: target.BaseURL, NativeBaseURL: variant.config.NativeBaseURL, + BaseURL: target.BaseURL, NativeBaseURL: variant.config.NativeBaseURL, Options: variant.config.Options, } published, err := u.publishGeneratedLocked(ctx, spec, variant.targets) if err != nil { diff --git a/internal/app/multimodal_install_test.go b/internal/app/multimodal_install_test.go index b54ba9c0..89dd06ff 100644 --- a/internal/app/multimodal_install_test.go +++ b/internal/app/multimodal_install_test.go @@ -384,3 +384,95 @@ func TestImportedLookalikeGetsNoKey(t *testing.T) { t.Fatalf("an imported tree was listed as installed: %#v", installed) } } + +// jiekouCore keys the built-in JieKou.AI Provider, whose catalog lists async +// video, native speech and GPT Image models. +func jiekouCore(t *testing.T, gateway *multimodalGateway) (*UseCases, string) { + t.Helper() + core, home := multimodalInstallCore(t, gateway) + if _, err := core.SaveProvider(context.Background(), provider.Entry{ID: "jiekou", Name: "JieKou.AI", BaseURL: "https://api.highwayapi.ai/openai", APIKey: multimodalKey}, false, false); err != nil { + t.Fatal(err) + } + return core, home +} + +func TestCatalogVideoInstallsWithItsOptionsAndNoProbe(t *testing.T) { + gateway := &multimodalGateway{listing: `{"data":[]}`} + core, home := jiekouCore(t, gateway) + detection, err := core.DetectMultimodal(context.Background()) + if err != nil { + t.Fatal(err) + } + var jiekou MultimodalProvider + for _, entry := range detection.Providers { + if entry.ID == "jiekou" { + jiekou = entry + } + } + video, ok := modality.Find(jiekou.Models, "kling-v3.0-4k-t2v", modality.VideoGeneration) + if !ok || !video.Installable || video.Probe != modality.ProbeNone { + t.Fatalf("catalog video = %#v, %v", video, ok) + } + // Video generation is never probed: a probe would be a full billed run. + if _, err := core.ProbeMultimodal(context.Background(), "jiekou", "kling-v3.0-4k-t2v", modality.VideoGeneration, modality.AdapterNovitaAsync, true); err == nil { + t.Fatal("a video generation probe was accepted") + } + result, err := core.InstallMultimodal(context.Background(), MultimodalInstallRequest{Provider: "jiekou", Model: "kling-v3.0-4k-t2v", Capability: modality.VideoGeneration, Agents: []string{"codex"}}) + if err != nil || !result.Results[0].Installed || !result.Results[0].EnvWritten { + t.Fatalf("install = %#v", result) + } + if posts := gateway.billed(); len(posts) != 0 { + t.Fatalf("install sent billed requests: %v", posts) + } + cfg, err := skillgen.ReadConfig(filepath.Join(home, ".codex", "skills", "bootagent-video-generation")) + if err != nil { + t.Fatal(err) + } + if cfg.NativeBaseURL != "https://api.highwayapi.ai" || strings.Join(cfg.Options, ",") != "duration,aspect_ratio,negative_prompt,sound" || cfg.KeyEnv != "BOOTAGENT_JIEKOU_API_KEY" { + t.Fatalf("config = %#v", cfg) + } + if codexSet(t, home)["BOOTAGENT_JIEKOU_API_KEY"] != multimodalKey { + t.Fatal("the key variable was not written") + } +} + +func TestNativeSpeechProbeUsesTheNativeRoute(t *testing.T) { + gateway := &multimodalGateway{listing: `{"data":[]}`} + core, _ := jiekouCore(t, gateway) + check, err := core.ProbeMultimodal(context.Background(), "jiekou", "minimax-speech-2.8-turbo", modality.TextToSpeech, modality.AdapterNovitaAudio, false) + if err != nil || check.Status != ModelCheckAvailable { + t.Fatalf("probe = %#v, %v", check, err) + } + posts := gateway.billed() + if len(posts) != 1 || !strings.HasPrefix(posts[0], "POST /v3/minimax-speech-2.8-turbo") { + t.Fatalf("probe went to %v", posts) + } + // A Provider without a native route cannot take this probe. + if _, err := core.ProbeMultimodal(context.Background(), "gw", "minimax-speech-2.8-turbo", modality.TextToSpeech, modality.AdapterNovitaAudio, false); err == nil { + t.Fatal("a native probe ran on a Provider with no native route") + } +} + +func TestDetectionMarksUndocumentedCapabilities(t *testing.T) { + gateway := &multimodalGateway{listing: `{"data":[]}`} + core, _ := multimodalInstallCore(t, gateway) + if _, err := core.SaveProvider(context.Background(), provider.Entry{ID: "ppio", Name: "PPIO", BaseURL: "https://api.ppio.com/openai", APIKey: multimodalKey}, false, false); err != nil { + t.Fatal(err) + } + detection, err := core.DetectMultimodal(context.Background()) + if err != nil { + t.Fatal(err) + } + for _, entry := range detection.Providers { + switch entry.ID { + case "ppio": + if strings.Join(entry.Unsupported, ",") != modality.VideoGeneration { + t.Fatalf("ppio unsupported = %v", entry.Unsupported) + } + case "gw": + if len(entry.Unsupported) != 0 { + t.Fatalf("a custom Provider was marked: %v", entry.Unsupported) + } + } + } +} diff --git a/internal/catalog/modalities.go b/internal/catalog/modalities.go index e31c1023..fa9de81f 100644 --- a/internal/catalog/modalities.go +++ b/internal/catalog/modalities.go @@ -44,6 +44,10 @@ type ModalityProvider struct { // (https://api.novita.ai/openai), which is not where those routes live. NativeBaseURL string `json:"native_base_url"` Generation []ModalityGenerationModel `json:"generation"` + // Unsupported names capabilities this Provider is known not to offer + // through any documented API, so the UI can say so instead of showing + // nothing. + Unsupported []string `json:"unsupported,omitempty"` } // ModalityGenerationModel is a model only the catalog knows about: the async @@ -52,6 +56,13 @@ type ModalityGenerationModel struct { ID string `json:"id"` Capabilities []string `json:"capabilities"` Adapter string `json:"adapter"` + // Options are the request fields the model documents beyond the prompt, + // from the adapter's option vocabulary. The generated script refuses any + // other. + Options []string `json:"options,omitempty"` + // Recommended marks the model preselected for its capability on this + // Provider, usually the cheaper of two that do the same job. + Recommended bool `json:"recommended,omitempty"` } func LoadEmbeddedModalities() (ModalityManifest, error) { @@ -97,7 +108,7 @@ func ParseModalities(data []byte) (ModalityManifest, error) { if _, ok := ProviderByID(providerID); !ok { return ModalityManifest{}, invalidModalities(fmt.Sprintf("Modality provider %q is not a built-in Provider", providerID)) } - if !httpsURL(entry.NativeBaseURL) || strings.HasSuffix(entry.NativeBaseURL, "/") { + if entry.NativeBaseURL != "" && (!httpsURL(entry.NativeBaseURL) || strings.HasSuffix(entry.NativeBaseURL, "/")) { return ModalityManifest{}, invalidModalities(fmt.Sprintf("Modality provider %q has an invalid native_base_url", providerID)) } seen := map[string]bool{} @@ -130,9 +141,10 @@ func cloneModalityManifest(source ModalityManifest) ModalityManifest { models := make([]ModalityGenerationModel, len(entry.Generation)) for index, model := range entry.Generation { model.Capabilities = append([]string(nil), model.Capabilities...) + model.Options = append([]string(nil), model.Options...) models[index] = model } - result.Providers[id] = ModalityProvider{NativeBaseURL: entry.NativeBaseURL, Generation: models} + result.Providers[id] = ModalityProvider{NativeBaseURL: entry.NativeBaseURL, Generation: models, Unsupported: append([]string(nil), entry.Unsupported...)} } return result } diff --git a/internal/modality/capability.go b/internal/modality/capability.go index 61ffdb42..b797a8fa 100644 --- a/internal/modality/capability.go +++ b/internal/modality/capability.go @@ -39,12 +39,43 @@ const ( AdapterImages = "openai-images" AdapterAudioTranscriptions = "openai-audio-transcriptions" AdapterAudioSpeech = "openai-audio-speech" - // Native routes at the Provider's host root, named by the catalog. + // Native routes at the Provider's host root, named by the catalog. Novita, + // PPIO and JieKou.AI document the same native API family, so the adapters + // are named after Novita, where it was first documented, and serve all + // three: novita-async submits to /v3/async/ and polls + // /v3/async/task-result; novita-audio posts to /v3/ and gets the + // audio back in the response. AdapterNovitaAsync = "novita-async" AdapterNovitaAudio = "novita-audio" - AdapterPPIOVideo = "ppio-video" + // AdapterPPIOVideo is reserved. PPIO's unified video route + // (/v3/video/create) was documented when this design began and is not any + // more; no catalog entry may name it until it can be verified again. + AdapterPPIOVideo = "ppio-video" ) +// nativeAdapters reach the Provider's host root rather than its OpenAI base, +// so a Skill using one needs the catalog's native_base_url. +var nativeAdapters = map[string]bool{AdapterNovitaAsync: true, AdapterNovitaAudio: true, AdapterPPIOVideo: true} + +// IsNative reports whether an adapter needs the Provider's native base URL. +func IsNative(adapter string) bool { + return nativeAdapters[adapter] +} + +// Generation options are the request fields beyond the prompt that an async +// generation model may document. The vocabulary is closed: the generated +// script maps each to one command-line flag and refuses anything else. +const ( + OptionDuration = "duration" + OptionSize = "size" + OptionAspectRatio = "aspect_ratio" + OptionNegativePrompt = "negative_prompt" + OptionSound = "sound" +) + +// Options is the closed option vocabulary. +var Options = []string{OptionDuration, OptionSize, OptionAspectRatio, OptionNegativePrompt, OptionSound} + // Adapters is the closed vocabulary the catalog may name. var Adapters = []string{ AdapterChatVision, AdapterChatVideo, AdapterChatImageOutput, AdapterChatAudioInput, @@ -99,6 +130,7 @@ var probeKind = map[string]string{ AdapterAudioTranscriptions: ProbeMinimal, AdapterAudioSpeech: ProbeMinimal, AdapterImages: ProbeConfirm, + AdapterNovitaAudio: ProbeMinimal, } // ProbeKind reports how an adapter can be probed. @@ -117,7 +149,7 @@ var adapterCapability = map[string]string{ AdapterAudioTranscriptions: SpeechToText, AdapterAudioSpeech: TextToSpeech, AdapterNovitaAsync: "", - AdapterNovitaAudio: "", + AdapterNovitaAudio: TextToSpeech, AdapterPPIOVideo: VideoGeneration, } diff --git a/internal/modality/resolve.go b/internal/modality/resolve.go index b943e652..41c7dc9e 100644 --- a/internal/modality/resolve.go +++ b/internal/modality/resolve.go @@ -27,6 +27,8 @@ type Capability struct { Installable bool `json:"installable"` // Probe is the live check available for this capability; see ProbeKind. Probe string `json:"probe"` + // Recommended marks the catalog's preferred model for this capability. + Recommended bool `json:"recommended,omitempty"` } // Model is one model with every capability found for it, in Capabilities order. @@ -87,16 +89,50 @@ func NewRules(manifest catalog.ModalityManifest) (Rules, error) { if !slices.Contains(Adapters, model.Adapter) { return Rules{}, invalid(fmt.Sprintf("Modality provider %q model %q names unknown adapter %q", providerID, model.ID, model.Adapter)) } + if model.Adapter == AdapterPPIOVideo { + return Rules{}, invalid(fmt.Sprintf("Modality provider %q model %q names the reserved adapter %q", providerID, model.ID, model.Adapter)) + } + if IsNative(model.Adapter) && entry.NativeBaseURL == "" { + return Rules{}, invalid(fmt.Sprintf("Modality provider %q model %q needs a native_base_url", providerID, model.ID)) + } for _, capability := range model.Capabilities { if !slices.Contains(Capabilities, capability) { return Rules{}, invalid(fmt.Sprintf("Modality provider %q model %q names unknown capability %q", providerID, model.ID, capability)) } + if !Serves(model.Adapter, capability) { + return Rules{}, invalid(fmt.Sprintf("Modality provider %q model %q: %s does not serve %s", providerID, model.ID, model.Adapter, capability)) + } + } + for _, option := range model.Options { + if !slices.Contains(Options, option) { + return Rules{}, invalid(fmt.Sprintf("Modality provider %q model %q names unknown option %q", providerID, model.ID, option)) + } + } + } + for _, capability := range entry.Unsupported { + if !slices.Contains(Capabilities, capability) { + return Rules{}, invalid(fmt.Sprintf("Modality provider %q marks unknown capability %q unsupported", providerID, capability)) } } } return result, nil } +// Unsupported lists the capabilities a Provider is known not to offer. +func (r Rules) Unsupported(providerID string) []string { + return slices.Clone(r.providers[providerID].Unsupported) +} + +// Generation returns the catalog entry for one Provider's model, if any. +func (r Rules) Generation(providerID, modelID string) (catalog.ModalityGenerationModel, bool) { + for _, model := range r.providers[providerID].Generation { + if model.ID == modelID { + return model, true + } + } + return catalog.ModalityGenerationModel{}, false +} + // NativeBaseURL is the host root for a Provider's native routes, empty when // the catalog names none. func (r Rules) NativeBaseURL(providerID string) string { @@ -160,8 +196,13 @@ func DeclaredCapabilities(entry provider.ModelEntry) []string { // // The declared metadata answers for the whole model: a model that declares // modalities is not given more by an ID rule, because the Provider's own word -// about a model it serves outranks a guess from its name. Generation models -// the catalog lists for the Provider are appended with source catalog. +// about a model it serves outranks a guess from its name. +// +// A catalog entry for a listed model outranks both for the capabilities it +// names: the catalog names a verified route, where a declaration only says +// the model produces, say, images, not how a gateway returns them. Catalog +// models no listing has (async generation, native audio) are appended with +// source catalog. func (r Rules) Resolve(providerID string, entries []provider.ModelEntry, verdicts map[string]map[string]provider.ModalityVerdict) []Model { models := make([]Model, 0, len(entries)) seen := map[string]bool{} @@ -176,12 +217,17 @@ func (r Rules) Resolve(providerID string, entries []provider.ModelEntry, verdict source, claimed = SourceDeclared, DeclaredCapabilities(entry) } probed := verdicts[id] + generation, cataloged := r.Generation(providerID, id) var capabilities []Capability for _, capabilityID := range Capabilities { if verdict, ok := probed[capabilityID]; ok { capabilities = append(capabilities, probedCapability(capabilityID, verdict)) continue } + if cataloged && slices.Contains(generation.Capabilities, capabilityID) { + capabilities = append(capabilities, catalogCapability(capabilityID, generation)) + continue + } if !slices.Contains(claimed, capabilityID) { continue } @@ -214,14 +260,7 @@ func (r Rules) Resolve(providerID string, entries []provider.ModelEntry, verdict capabilities = append(capabilities, probedCapability(capabilityID, verdict)) continue } - capabilities = append(capabilities, Capability{ - ID: capabilityID, - Source: SourceCatalog, - Adapter: generation.Adapter, - Supported: true, - Installable: true, - Probe: ProbeKind(generation.Adapter), - }) + capabilities = append(capabilities, catalogCapability(capabilityID, generation)) } models = append(models, Model{ID: generation.ID, Capabilities: capabilities}) } @@ -260,6 +299,20 @@ func Find(models []Model, modelID, capability string) (Capability, bool) { return Capability{}, false } +// catalogCapability is a capability on a verified route the catalog names. It +// installs without a probe; the probe, where one exists, is still offered. +func catalogCapability(capabilityID string, generation catalog.ModalityGenerationModel) Capability { + return Capability{ + ID: capabilityID, + Source: SourceCatalog, + Adapter: generation.Adapter, + Supported: true, + Installable: true, + Probe: ProbeKind(generation.Adapter), + Recommended: generation.Recommended, + } +} + func probedCapability(capabilityID string, verdict provider.ModalityVerdict) Capability { adapter := verdict.Adapter if adapter == "" { diff --git a/internal/modality/resolve_test.go b/internal/modality/resolve_test.go index aa239f90..49bb70c8 100644 --- a/internal/modality/resolve_test.go +++ b/internal/modality/resolve_test.go @@ -119,7 +119,9 @@ func TestResolveReplaysSnapshotsSafely(t *testing.T) { if probeOnly[capability.Adapter] && capability.Installable { t.Errorf("%s: %s %s installable on unverified adapter %s without a probe", name, model.ID, capability.ID, capability.Adapter) } - if capability.ID == TextToSpeech || capability.ID == VideoGeneration { + // Speech and video generation have no chat route; only the + // catalog, which names a native route, may offer them. + if (capability.ID == TextToSpeech || capability.ID == VideoGeneration) && capability.Source != SourceCatalog { t.Errorf("%s: %s offered %s from a chat listing", name, model.ID, capability.ID) } } @@ -291,3 +293,77 @@ func TestAdapterVocabulary(t *testing.T) { t.Fatal("a capability with no OpenAI-compatible route offered manual entry") } } + +// A catalog entry names a verified route. For a listed model it outranks the +// declaration, which only says the model outputs images, not how a gateway +// returns them. +func TestCatalogOutranksDeclarationForListedModels(t *testing.T) { + rules := embeddedRules(t) + entries := loadSnapshot(t, "novita") + models := rules.Resolve("novita", entries, nil) + ming, ok := Find(models, "ming-image-0.1-design", ImageGeneration) + if !ok || ming.Source != SourceCatalog || ming.Adapter != AdapterImages || !ming.Installable || !ming.Recommended { + t.Fatalf("ming image = %#v, %v", ming, ok) + } + // Its other declared capabilities still come from the declaration. + if vision, ok := Find(models, "ming-image-0.1-design", Vision); !ok || vision.Source != SourceDeclared { + t.Fatalf("ming vision = %#v, %v", vision, ok) + } + // The layer model shares the name prefix but is edit-only: the catalog + // does not list it, so it stays on the unverified chat route. + if layer, ok := Find(models, "ming-image-0.1-design-layer", ImageGeneration); ok && layer.Installable { + t.Fatalf("edit-only model installable: %#v", layer) + } + // A probe verdict still outranks the catalog. + probed := rules.Resolve("novita", entries, map[string]map[string]provider.ModalityVerdict{ + "ming-image-0.1-design": {ImageGeneration: {Supported: false, Source: SourceProbe, Adapter: AdapterImages}}, + }) + if found, _ := Find(probed, "ming-image-0.1-design", ImageGeneration); found.Source != SourceProbe || found.Installable { + t.Fatalf("probe did not outrank catalog: %#v", found) + } +} + +func TestEmbeddedCatalogOffersVerifiedNativeRoutes(t *testing.T) { + rules := embeddedRules(t) + for _, providerID := range []string{"novita", "ppio", "jiekou"} { + models := rules.Resolve(providerID, nil, nil) + speech, ok := Find(models, "minimax-speech-2.8-turbo", TextToSpeech) + if !ok || speech.Adapter != AdapterNovitaAudio || !speech.Installable || speech.Probe != ProbeMinimal { + t.Errorf("%s speech = %#v, %v", providerID, speech, ok) + } + if rules.NativeBaseURL(providerID) == "" { + t.Errorf("%s has no native base", providerID) + } + } + video, ok := Find(rules.Resolve("jiekou", nil, nil), "wan2.7-t2v", VideoGeneration) + if !ok || video.Adapter != AdapterNovitaAsync || !video.Installable || video.Probe != ProbeNone { + t.Fatalf("jiekou video = %#v, %v", video, ok) + } + generation, _ := rules.Generation("jiekou", "kling-v3.0-4k-t2v") + if strings.Join(generation.Options, ",") != "duration,aspect_ratio,negative_prompt,sound" { + t.Fatalf("kling options = %v", generation.Options) + } + for _, providerID := range []string{"novita", "ppio"} { + if !slices.Contains(rules.Unsupported(providerID), VideoGeneration) { + t.Errorf("%s video generation should be marked unsupported", providerID) + } + } +} + +func TestNewRulesRejectsBrokenGenerationEntries(t *testing.T) { + with := func(provider catalog.ModalityProvider) catalog.ModalityManifest { + return catalog.ModalityManifest{SchemaVersion: 1, IDRules: []catalog.ModalityIDRule{{Pattern: "^x", Capabilities: []string{Vision}}}, Providers: map[string]catalog.ModalityProvider{"novita": provider}} + } + cases := map[string]catalog.ModalityProvider{ + "native without base": {Generation: []catalog.ModalityGenerationModel{{ID: "m", Capabilities: []string{TextToSpeech}, Adapter: AdapterNovitaAudio}}}, + "reserved adapter": {NativeBaseURL: "https://api.novita.ai", Generation: []catalog.ModalityGenerationModel{{ID: "m", Capabilities: []string{VideoGeneration}, Adapter: AdapterPPIOVideo}}}, + "adapter mismatch": {NativeBaseURL: "https://api.novita.ai", Generation: []catalog.ModalityGenerationModel{{ID: "m", Capabilities: []string{Vision}, Adapter: AdapterNovitaAudio}}}, + "unknown option": {NativeBaseURL: "https://api.novita.ai", Generation: []catalog.ModalityGenerationModel{{ID: "m", Capabilities: []string{VideoGeneration}, Adapter: AdapterNovitaAsync, Options: []string{"fps"}}}}, + "unknown unsupported": {Unsupported: []string{"smell"}}, + } + for name, entry := range cases { + if _, err := NewRules(with(entry)); err == nil { + t.Errorf("%s accepted", name) + } + } +} diff --git a/internal/provider/client.go b/internal/provider/client.go index 8043f282..c3343e3f 100644 --- a/internal/provider/client.go +++ b/internal/provider/client.go @@ -11,6 +11,7 @@ import ( "mime/multipart" "net" "net/http" + "regexp" "strings" "time" @@ -222,6 +223,90 @@ func (c *Client) postJSONProbe(ctx context.Context, apiKey, model, customBase, p return c.sendProbe(request, ProtocolOpenAI, apiKey, model, timeout), nil } +// nativeModelPattern bounds a model ID used as a path segment on a native +// route, so a hand-edited ID cannot add path or query parts. +var nativeModelPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$`) + +// NativeSpeechVoice is the system voice the native speech probe asks for. +// It is one of the system voices Novita, PPIO and JieKou.AI document for +// MiniMax Speech 2.8. +const NativeSpeechVoice = "English_Graceful_Lady" + +// ProbeNativeSpeech asks a native MiniMax-style speech route +// (POST /v3/) to say "ok". These routes answer HTTP 200 +// for some refusals and put the verdict in base_resp.status_code, so the body +// is read on success too and mapped onto the HTTP classification: 1004 is a +// rejected key, 1002 and 1039 are rate limits, 1000 and 1001 are the +// provider's own failures, anything else is a refusal. +func (c *Client) ProbeNativeSpeech(ctx context.Context, apiKey, model, nativeBase string) (ProbeResult, error) { + if apiKey == "" { + return ProbeResult{}, oneerrors.New(oneerrors.InvalidRequest, "API key is required") + } + if !nativeModelPattern.MatchString(model) { + return ProbeResult{}, oneerrors.New(oneerrors.InvalidRequest, "A valid model is required") + } + base, err := ValidateBaseURL(strings.TrimSpace(nativeBase)) + if err != nil { + return ProbeResult{}, err + } + payload, err := json.Marshal(map[string]any{ + "text": "ok", + "voice_setting": map[string]string{"voice_id": NativeSpeechVoice}, + "audio_setting": map[string]string{"format": "mp3"}, + }) + if err != nil { + return ProbeResult{}, probeBuildError(err) + } + request, err := http.NewRequestWithContext(ctx, http.MethodPost, base+"/v3/"+model, bytes.NewReader(payload)) + if err != nil { + return ProbeResult{}, oneerrors.New(oneerrors.InvalidRequest, "Provider endpoint is invalid", oneerrors.WithCause(err)) + } + request.Header.Set("Authorization", "Bearer "+apiKey) + request.Header.Set("Content-Type", "application/json") + response, cancelRequest, err := c.doWithin(request, audioProbeTimeout) + defer cancelRequest() + if err != nil || response == nil { + if response != nil && response.Body != nil { + _ = response.Body.Close() + } + if err == nil { + err = errors.New("Provider transport returned no response") + } + result := transportResult(err) + result.Protocol = new(ProtocolOpenAI) + return result, nil + } + defer response.Body.Close() + body, _, _ := c.readBody(response.Body) + if response.StatusCode < 200 || response.StatusCode >= 300 { + result := classifyHTTPProbe(response.StatusCode, string(body), ProtocolOpenAI, model) + result.Detail = upstreamDetail(body, apiKey) + return result, nil + } + var parsed struct { + BaseResp *struct { + StatusCode int `json:"status_code"` + StatusMsg string `json:"status_msg"` + } `json:"base_resp"` + } + _ = json.Unmarshal(body, &parsed) + if parsed.BaseResp == nil || parsed.BaseResp.StatusCode == 0 { + return ProbeResult{OK: true, Reachable: true, Status: response.StatusCode, Message: "Speech route accepted the request.", Protocol: new(ProtocolOpenAI)}, nil + } + status := http.StatusBadRequest + switch parsed.BaseResp.StatusCode { + case 1004: + status = http.StatusUnauthorized + case 1002, 1039: + status = http.StatusTooManyRequests + case 1000, 1001: + status = http.StatusServiceUnavailable + } + result := classifyHTTPProbe(status, "", ProtocolOpenAI, model) + result.Detail = upstreamDetail([]byte(parsed.BaseResp.StatusMsg), apiKey) + return result, nil +} + func probeTarget(apiKey, model, customBase string) (string, error) { if apiKey == "" { return "", oneerrors.New(oneerrors.InvalidRequest, "API key is required") diff --git a/internal/provider/client_test.go b/internal/provider/client_test.go index edb464fc..bb91d37c 100644 --- a/internal/provider/client_test.go +++ b/internal/provider/client_test.go @@ -6,6 +6,7 @@ import ( "encoding/base64" "encoding/json" "errors" + "fmt" "image/png" "io" "net/http" @@ -480,3 +481,44 @@ func TestSlowProbesOutlastTheChatTimeout(t *testing.T) { _, _ = client.ProbeImageGeneration(context.Background(), "k", "m", "https://proxy.test") _, _ = client.ProbeSpeech(context.Background(), "k", "m", "https://proxy.test") } + +func TestProbeNativeSpeechReadsTheVerdictInTheBody(t *testing.T) { + var seen *http.Request + var body map[string]any + answer := `{"data":{"audio":"4944"},"base_resp":{"status_code":0,"status_msg":"success"}}` + client := NewClient(fakeDoer(func(request *http.Request) (*http.Response, error) { + seen = request + _ = json.NewDecoder(request.Body).Decode(&body) + return fakeResponse(http.StatusOK, answer), nil + })) + result, err := client.ProbeNativeSpeech(context.Background(), "sk-native", "minimax-speech-2.8-turbo", "https://api.novita.ai") + if err != nil || !result.OK { + t.Fatalf("result = %#v, %v", result, err) + } + if seen.URL.String() != "https://api.novita.ai/v3/minimax-speech-2.8-turbo" || body["text"] != "ok" { + t.Fatalf("url = %v, body = %#v", seen.URL, body) + } + if voice := body["voice_setting"].(map[string]any)["voice_id"]; voice != NativeSpeechVoice { + t.Fatalf("voice = %v", voice) + } + for code, want := range map[int]string{1004: oneerrors.APIKeyRejected, 1002: oneerrors.ProviderUnreachable, 2013: oneerrors.ProviderUnreachable} { + answer = fmt.Sprintf(`{"base_resp":{"status_code":%d,"status_msg":"refused for this voice"}}`, code) + result, err := client.ProbeNativeSpeech(context.Background(), "sk-native", "minimax-speech-2.8-turbo", "https://api.novita.ai") + if err != nil || result.OK || result.ErrorCode == nil || *result.ErrorCode != want { + t.Fatalf("code %d: result = %#v, %v", code, result, err) + } + if result.Detail != "refused for this voice" { + t.Fatalf("detail = %q", result.Detail) + } + } + answer = `{"base_resp":{"status_code":1002,"status_msg":"rate limit"}}` + // The app reads 408, 429 and 5xx as transient from the status alone. + if result, _ := client.ProbeNativeSpeech(context.Background(), "sk-native", "m", "https://api.novita.ai"); result.Status != http.StatusTooManyRequests || !result.Reachable { + t.Fatalf("rate limit should map to 429: %#v", result) + } + for _, model := range []string{"", "../x", "m?x=1", "a/b"} { + if _, err := client.ProbeNativeSpeech(context.Background(), "sk-native", model, "https://api.novita.ai"); err == nil { + t.Fatalf("model %q accepted as a path segment", model) + } + } +} diff --git a/internal/skillgen/native_script_test.go b/internal/skillgen/native_script_test.go new file mode 100644 index 00000000..1c111cb6 --- /dev/null +++ b/internal/skillgen/native_script_test.go @@ -0,0 +1,224 @@ +package skillgen + +import ( + "encoding/json" + "encoding/pem" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/MaimoryLab/BootAgent/internal/modality" +) + +// nativeProvider serves the MiniMax-style speech route and the async task +// routes Novita, PPIO and JieKou.AI document, over TLS: native routes must be +// https, and Node trusts the test certificate through NODE_EXTRA_CA_CERTS. +type nativeProvider struct { + mu sync.Mutex + polls int + outcome string // "succeed", "fail" or "pending" + baseResp int + bodies map[string]map[string]any + auth map[string]string +} + +func (f *nativeProvider) serve(server **httptest.Server) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var body map[string]any + data, _ := io.ReadAll(r.Body) + _ = json.Unmarshal(data, &body) + f.mu.Lock() + f.bodies[r.URL.Path] = body + f.auth[r.URL.Path] = r.Header.Get("Authorization") + outcome, baseResp := f.outcome, f.baseResp + f.mu.Unlock() + switch { + case r.URL.Path == "/v3/minimax-speech-2.8-turbo": + if baseResp != 0 { + _, _ = w.Write([]byte(`{"base_resp":{"status_code":` + itoa(baseResp) + `,"status_msg":"refused for ` + r.Header.Get("Authorization") + `"}}`)) + return + } + _, _ = w.Write([]byte(`{"data":{"audio":"494433","status":2},"base_resp":{"status_code":0,"status_msg":"success"}}`)) + case r.URL.Path == "/v3/async/wan2.7-t2v": + _, _ = w.Write([]byte(`{"task_id":"task-123"}`)) + case r.URL.Path == "/v3/async/task-result": + f.mu.Lock() + f.polls++ + polls := f.polls + f.mu.Unlock() + if r.URL.Query().Get("task_id") != "task-123" { + http.Error(w, `{"error":"unknown task"}`, http.StatusNotFound) + return + } + switch { + case outcome == "pending" || polls < 2: + _, _ = w.Write([]byte(`{"task":{"task_id":"task-123","status":"TASK_STATUS_PROCESSING","eta":1}}`)) + case outcome == "fail": + _, _ = w.Write([]byte(`{"task":{"task_id":"task-123","status":"TASK_STATUS_FAILED","reason":"content policy"}}`)) + default: + _, _ = w.Write([]byte(`{"task":{"task_id":"task-123","status":"TASK_STATUS_SUCCEED"},"videos":[{"video_url":"` + (*server).URL + `/files/clip.mp4","video_url_ttl":"3600","video_type":"mp4"}]}`)) + } + case r.URL.Path == "/files/clip.mp4": + _, _ = w.Write([]byte("fake mp4 bytes")) + default: + http.NotFound(w, r) + } + } +} + +func (f *nativeProvider) body(path string) (map[string]any, bool) { + f.mu.Lock() + defer f.mu.Unlock() + body, ok := f.bodies[path] + return body, ok +} + +func (f *nativeProvider) authFor(path string) string { + f.mu.Lock() + defer f.mu.Unlock() + return f.auth[path] +} + +func (f *nativeProvider) requests() int { + f.mu.Lock() + defer f.mu.Unlock() + return len(f.bodies) +} + +func (f *nativeProvider) pollCount() int { + f.mu.Lock() + defer f.mu.Unlock() + return f.polls +} + +func (f *nativeProvider) set(change func(*nativeProvider)) { + f.mu.Lock() + defer f.mu.Unlock() + change(f) +} + +func itoa(value int) string { + encoded, _ := json.Marshal(value) + return string(encoded) +} + +func nativeSetup(t *testing.T, spec Spec) (*nativeProvider, string, string, string, []string) { + t.Helper() + fake := &nativeProvider{outcome: "succeed", bodies: map[string]map[string]any{}, auth: map[string]string{}} + var server *httptest.Server + server = httptest.NewTLSServer(fake.serve(&server)) + t.Cleanup(server.Close) + ca := filepath.Join(t.TempDir(), "ca.pem") + if err := os.WriteFile(ca, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw}), 0o600); err != nil { + t.Fatal(err) + } + spec.BaseURL, spec.NativeBaseURL = server.URL, server.URL + dir := renderTo(t, spec) + return fake, dir, resolvedTempDir(t), resolvedTempDir(t), []string{keyEnv, "NODE_EXTRA_CA_CERTS=" + ca} +} + +var nativeSpeechSpec = Spec{Capability: modality.TextToSpeech, Adapter: modality.AdapterNovitaAudio, ProviderID: "local", ProviderName: "Local", Model: "minimax-speech-2.8-turbo"} + +var videoSpec = Spec{Capability: modality.VideoGeneration, Adapter: modality.AdapterNovitaAsync, ProviderID: "local", ProviderName: "Local", Model: "wan2.7-t2v", Options: []string{"duration", "size", "negative_prompt"}} + +func TestScriptSpeaksOnTheNativeRoute(t *testing.T) { + fake, dir, work, home, env := nativeSetup(t, nativeSpeechSpec) + run := runScript(t, dir, work, home, env, "speak", "--text", "Build passed", "--voice", "Chinese (Mandarin)_Lyrical_Voice", "--speed", "1.2", "--out", "done.mp3") + if run.code != 0 || strings.TrimSpace(run.stdout) != filepath.Join(work, "done.mp3") { + t.Fatalf("run = %#v", run) + } + if data, _ := os.ReadFile(filepath.Join(work, "done.mp3")); string(data) != "ID3" { + t.Fatalf("hex audio was not decoded: %q", data) + } + body, _ := fake.body("/v3/minimax-speech-2.8-turbo") + voice := body["voice_setting"].(map[string]any) + if body["text"] != "Build passed" || voice["voice_id"] != "Chinese (Mandarin)_Lyrical_Voice" || voice["speed"] != 1.2 || body["output_format"] != "hex" { + t.Fatalf("body = %#v", body) + } + if fake.authFor("/v3/minimax-speech-2.8-turbo") != "Bearer "+scriptKey { + t.Fatal("native route did not get the key") + } + // A refusal in base_resp under HTTP 200 is still a refusal, and the + // echoed key is scrubbed. + for code, exit := range map[int]int{1004: 5, 1002: 6, 2013: 4} { + fake.set(func(f *nativeProvider) { f.baseResp = code }) + run = runScript(t, dir, work, home, env, "speak", "--text", "hi") + if run.code != exit || strings.Contains(run.stderr, scriptKey) { + t.Errorf("base_resp %d: exit %d, stderr %q", code, run.code, run.stderr) + } + } + if run := runScript(t, dir, work, home, env, "speak", "--text", "hi", "--format", "opus"); run.code != 2 { + t.Fatalf("an OpenAI-only format was accepted: %#v", run) + } +} + +func TestScriptGeneratesVideoAndDownloadsWithoutTheKey(t *testing.T) { + fake, dir, work, home, env := nativeSetup(t, videoSpec) + run := runScript(t, dir, work, home, env, "video", "--prompt", "a paper plane over a city", "--duration", "5", "--size", "1280*720") + if run.code != 0 || !strings.Contains(run.stderr, "task: task-123") { + t.Fatalf("run = %#v", run) + } + saved := strings.TrimSpace(run.stdout) + if !strings.HasPrefix(saved, filepath.Join(work, "bootagent-output")) || !strings.HasSuffix(saved, ".mp4") { + t.Fatalf("saved = %q", saved) + } + if data, _ := os.ReadFile(saved); string(data) != "fake mp4 bytes" { + t.Fatalf("video = %q", data) + } + if matches, _ := filepath.Glob(filepath.Join(work, "bootagent-output", "*.part")); len(matches) != 0 { + t.Fatalf("partial files left: %v", matches) + } + body, _ := fake.body("/v3/async/wan2.7-t2v") + if body["prompt"] != "a paper plane over a city" || body["duration"] != float64(5) || body["size"] != "1280*720" { + t.Fatalf("submit body = %#v", body) + } + if polls := fake.pollCount(); polls < 2 { + t.Fatalf("the task was not polled until done: %d polls", polls) + } + if fake.authFor("/files/clip.mp4") != "" { + t.Fatal("the result URL was fetched with the key") + } + // An option the model does not document is refused before anything is + // submitted. + before := fake.requests() + if run := runScript(t, dir, work, home, env, "video", "--prompt", "x", "--aspect-ratio", "16:9"); run.code != 2 { + t.Fatalf("an undocumented option was accepted: %#v", run) + } + if fake.requests() != before { + t.Fatal("a refused command still sent a request") + } +} + +func TestScriptReportsFailedAndPendingVideoTasks(t *testing.T) { + fake, dir, work, home, env := nativeSetup(t, videoSpec) + fake.set(func(f *nativeProvider) { f.outcome, f.polls = "fail", 1 }) + run := runScript(t, dir, work, home, env, "video", "--prompt", "x") + if run.code != 4 || !strings.Contains(run.stderr, "content policy") { + t.Fatalf("failed task = %#v", run) + } + fake.set(func(f *nativeProvider) { f.outcome = "pending" }) + run = runScript(t, dir, work, home, env, "video", "--prompt", "x", "--timeout", "2") + if run.code != 7 || !strings.Contains(run.stderr, "video-result --task task-123") { + t.Fatalf("pending task = %#v", run) + } + // Fetching later does not submit again. + fake.set(func(f *nativeProvider) { + f.outcome = "succeed" + delete(f.bodies, "/v3/async/wan2.7-t2v") + }) + run = runScript(t, dir, work, home, env, "video-result", "--task", "task-123") + if run.code != 0 || !strings.HasSuffix(strings.TrimSpace(run.stdout), ".mp4") { + t.Fatalf("video-result = %#v", run) + } + if _, resubmitted := fake.body("/v3/async/wan2.7-t2v"); resubmitted { + t.Fatal("video-result submitted a new task") + } + if run := runScript(t, dir, work, home, env, "video-result", "--task", "../etc"); run.code != 2 { + t.Fatalf("a malformed task id was accepted: %#v", run) + } +} diff --git a/internal/skillgen/skillgen.go b/internal/skillgen/skillgen.go index a5191a2f..1444ca71 100644 --- a/internal/skillgen/skillgen.go +++ b/internal/skillgen/skillgen.go @@ -12,9 +12,11 @@ import ( "encoding/json" "errors" "fmt" + "net/url" "os" "path/filepath" "regexp" + "slices" "strings" "text/template" @@ -35,24 +37,27 @@ const IDPrefix = "bootagent-" // supported lists the capability and adapter pairs this version can render. // run.mjs implements exactly these; anything else is refused instead of // producing a Skill whose script cannot run it. -var supported = map[string]string{ - modality.Vision: modality.AdapterChatVision, - modality.ImageGeneration: modality.AdapterImages, - modality.SpeechToText: modality.AdapterAudioTranscriptions, - modality.TextToSpeech: modality.AdapterAudioSpeech, +var supported = map[string][]string{ + modality.Vision: {modality.AdapterChatVision}, + modality.ImageGeneration: {modality.AdapterImages}, + modality.SpeechToText: {modality.AdapterAudioTranscriptions}, + modality.TextToSpeech: {modality.AdapterAudioSpeech, modality.AdapterNovitaAudio}, + modality.VideoGeneration: {modality.AdapterNovitaAsync}, } // Supports reports whether a capability over an adapter can be generated. func Supports(capability, adapter string) bool { - return adapter != "" && supported[capability] == adapter + return adapter != "" && slices.Contains(supported[capability], adapter) } // SupportedAdapters lists the adapters a Skill can be generated for. func SupportedAdapters() []string { - result := make([]string, 0, len(supported)) + var result []string for _, capability := range modality.Capabilities { - if adapter, ok := supported[capability]; ok { - result = append(result, adapter) + for _, adapter := range supported[capability] { + if !slices.Contains(result, adapter) { + result = append(result, adapter) + } } } return result @@ -87,8 +92,12 @@ type Spec struct { Model string // BaseURL is the Provider's OpenAI-compatible base, normalized to /v1. BaseURL string - // NativeBaseURL is the host root for native routes; empty when unused. + // NativeBaseURL is the host root for native routes; required by a native + // adapter, ignored otherwise. NativeBaseURL string + // Options are the request fields beyond the prompt the model documents, + // from the catalog. Only async generation uses them. + Options []string } // Config is config.json. Nothing in it is secret. @@ -102,6 +111,9 @@ type Config struct { Model string `json:"model"` BaseURL string `json:"base_url"` NativeBaseURL string `json:"native_base_url,omitempty"` + // Options lists the request fields the model accepts beyond the prompt; + // the script refuses a flag for any other. + Options []string `json:"options,omitempty"` // KeyEnv names the variable holding the key; KeyFile is the private // fallback file read only when the variable is unset. "~" is the home // directory, expanded by the script so the tree is the same on every @@ -111,7 +123,10 @@ type Config struct { OutputDir string `json:"output_dir"` } -var providerIDPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,63}$`) +var ( + providerIDPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,63}$`) + nativeModelPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$`) +) // NewConfig validates a Spec and derives config.json from it. func NewConfig(spec Spec) (Config, error) { @@ -136,6 +151,29 @@ func NewConfig(spec Spec) (Config, error) { if strings.ContainsAny(name, "\r\n\x00") { return Config{}, errors.New("invalid Provider name for a generated Skill") } + native := "" + if modality.IsNative(spec.Adapter) { + // The model ID becomes a path segment on a native route. + if !nativeModelPattern.MatchString(model) { + return Config{}, errors.New("invalid model for a native route") + } + parsed, err := url.Parse(strings.TrimSpace(spec.NativeBaseURL)) + if err != nil || parsed.Scheme != "https" || parsed.Host == "" || parsed.User != nil || (parsed.Path != "" && parsed.Path != "/") || parsed.RawQuery != "" { + return Config{}, errors.New("a native route needs the Provider's https host root") + } + native = strings.TrimRight(spec.NativeBaseURL, "/") + } + var options []string + if spec.Adapter == modality.AdapterNovitaAsync { + for _, option := range spec.Options { + if !slices.Contains(modality.Options, option) { + return Config{}, fmt.Errorf("unknown generation option %q", option) + } + if !slices.Contains(options, option) { + options = append(options, option) + } + } + } return Config{ SchemaVersion: ConfigSchemaVersion, Generator: "bootagent", @@ -145,7 +183,8 @@ func NewConfig(spec Spec) (Config, error) { ProviderName: name, Model: model, BaseURL: provider.OpenAIBaseURL(base), - NativeBaseURL: strings.TrimRight(strings.TrimSpace(spec.NativeBaseURL), "/"), + NativeBaseURL: native, + Options: options, KeyEnv: config.SkillEnvName(spec.ProviderID), KeyFile: "~/.bootagent/skill-env/" + spec.ProviderID + ".env", OutputDir: "bootagent-output", @@ -161,6 +200,8 @@ type skillText struct { KeyEnv string KeyFile string CostRule string + Adapter string + Options []string } var titles = map[string]string{ @@ -168,6 +209,7 @@ var titles = map[string]string{ modality.ImageGeneration: "Image generation", modality.SpeechToText: "Speech to text", modality.TextToSpeech: "Text to speech", + modality.VideoGeneration: "Video generation", } var descriptions = map[string]string{ @@ -175,6 +217,7 @@ var descriptions = map[string]string{ modality.ImageGeneration: "Generate images from a text description through %s on %s and save them as files. Use when the user asks you to create an image. Each run is billed: confirm with the user first.", modality.SpeechToText: "Transcribe a local audio file to text through %s on %s. Use when the user asks for the text of a recording.", modality.TextToSpeech: "Turn text into a spoken audio file through %s on %s. Use when the user asks for audio of some text. Each run is billed: confirm with the user first.", + modality.VideoGeneration: "Generate a short video from a text description through %s on %s and save it as a file. Use when the user asks you to create a video. Each run is billed by the second and cannot be cancelled: confirm with the user first.", } var costRules = map[string]string{ @@ -182,6 +225,7 @@ var costRules = map[string]string{ modality.ImageGeneration: "Every run costs money. Before each run, tell the user what you are about to generate and how many images, and wait for them to agree.", modality.SpeechToText: "Each run is billed by the length of the audio. Run it when the user asks for a transcript; ask first before transcribing long or many recordings.", modality.TextToSpeech: "Every run costs money, billed by the length of the text. Before each run, tell the user what text you are about to speak and wait for them to agree.", + modality.VideoGeneration: "Every run costs money, billed by the second of video, and a submitted run cannot be cancelled. Before each run, tell the user the prompt, the duration and the resolution or aspect ratio you will use, and wait for them to agree. Never retry a failed or timed-out run on your own: a timed-out run may still finish and be billed.", } // Render writes the Skill tree into dir, which must exist and be empty. The @@ -221,8 +265,10 @@ func Files(cfg Config) (map[string][]byte, error) { KeyEnv: cfg.KeyEnv, KeyFile: cfg.KeyFile, CostRule: costRules[cfg.Capability], + Adapter: cfg.Adapter, + Options: cfg.Options, } - parsed, err := template.New("skill").Funcs(template.FuncMap{"yamlString": yamlString}).ParseFS(templates, "templates/common.md.tmpl", "templates/"+cfg.Capability+".md.tmpl") + parsed, err := template.New("skill").Funcs(template.FuncMap{"yamlString": yamlString, "has": slices.Contains[[]string, string]}).ParseFS(templates, "templates/common.md.tmpl", "templates/"+cfg.Capability+".md.tmpl") if err != nil { return nil, err } diff --git a/internal/skillgen/skillgen_test.go b/internal/skillgen/skillgen_test.go index c160d55b..662e1571 100644 --- a/internal/skillgen/skillgen_test.go +++ b/internal/skillgen/skillgen_test.go @@ -33,6 +33,15 @@ var goldenSpecs = map[string]Spec{ Capability: modality.TextToSpeech, Adapter: modality.AdapterAudioSpeech, ProviderID: "local", ProviderName: "", Model: "tts-1", BaseURL: "http://127.0.0.1:8000", }, + "text-to-speech-native": { + Capability: modality.TextToSpeech, Adapter: modality.AdapterNovitaAudio, + ProviderID: "ppio", ProviderName: "PPIO", Model: "minimax-speech-2.8-turbo", BaseURL: "https://api.ppio.com/openai", NativeBaseURL: "https://api.ppio.com", + }, + modality.VideoGeneration: { + Capability: modality.VideoGeneration, Adapter: modality.AdapterNovitaAsync, + ProviderID: "jiekou", ProviderName: "JieKou.AI", Model: "wan2.7-t2v", BaseURL: "https://api.highwayapi.ai/openai", NativeBaseURL: "https://api.highwayapi.ai", + Options: []string{"duration", "size", "negative_prompt"}, + }, } func renderTo(t *testing.T, spec Spec) string { @@ -173,6 +182,24 @@ func TestNewConfigRefusesWhatItCannotGenerate(t *testing.T) { "model with newline": func(s *Spec) { s.Model = "m\nrm -rf" }, "empty model": func(s *Spec) { s.Model = " " }, "bad base": func(s *Spec) { s.BaseURL = "ftp://x" }, + "native without base": func(s *Spec) { + s.Capability, s.Adapter, s.NativeBaseURL = modality.TextToSpeech, modality.AdapterNovitaAudio, "" + }, + "native over http": func(s *Spec) { + s.Capability, s.Adapter, s.NativeBaseURL = modality.TextToSpeech, modality.AdapterNovitaAudio, "http://api.novita.ai" + }, + "native base with a path": func(s *Spec) { + s.Capability, s.Adapter, s.NativeBaseURL = modality.TextToSpeech, modality.AdapterNovitaAudio, "https://api.novita.ai/openai" + }, + "native model with a slash": func(s *Spec) { + s.Capability, s.Adapter, s.NativeBaseURL, s.Model = modality.TextToSpeech, modality.AdapterNovitaAudio, "https://api.novita.ai", "a/../b" + }, + "unknown video option": func(s *Spec) { + s.Capability, s.Adapter, s.NativeBaseURL, s.Options = modality.VideoGeneration, modality.AdapterNovitaAsync, "https://api.novita.ai", []string{"fps"} + }, + "reserved ppio video": func(s *Spec) { + s.Capability, s.Adapter, s.NativeBaseURL = modality.VideoGeneration, modality.AdapterPPIOVideo, "https://api.ppio.com" + }, "base with userinfo": func(s *Spec) { s.BaseURL = "https://user:pass@x.test" }, } for name, mutate := range cases { diff --git a/internal/skillgen/templates/common.md.tmpl b/internal/skillgen/templates/common.md.tmpl index c49ab814..5794a172 100644 --- a/internal/skillgen/templates/common.md.tmpl +++ b/internal/skillgen/templates/common.md.tmpl @@ -24,6 +24,9 @@ Generated by BootAgent. It calls `{{.Model}}` on the user's {{.ProviderName}} ac | 4 | The Provider refused the request | Show the Provider's message. The model may not support this input; the user can choose another model in BootAgent. | | 5 | The Provider rejected the API key | Ask the user to check the {{.ProviderName}} key in BootAgent. | | 6 | The Provider was unreachable, busy or timed out | Suggest trying again later. | +{{- if eq .Adapter "novita-async"}} +| 7 | The task was submitted but had not finished in time | Tell the user it may still finish and be billed; offer to fetch it with `video-result` instead of submitting again. | +{{- end}} `node /scripts/run.mjs check` confirms the key is available without sending a request. {{end}} diff --git a/internal/skillgen/templates/run.mjs b/internal/skillgen/templates/run.mjs index 0c6396a5..2b19cb94 100644 --- a/internal/skillgen/templates/run.mjs +++ b/internal/skillgen/templates/run.mjs @@ -6,14 +6,14 @@ // dependencies. The API key comes from the environment variable named in // config.json, or from BootAgent's private fallback file; it is never printed. -import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs"; +import { createWriteStream, existsSync, mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs"; import { homedir } from "node:os"; import { basename, dirname, extname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; -const EXIT = { ok: 0, internal: 1, usage: 2, setup: 3, refused: 4, key: 5, unavailable: 6 }; +const EXIT = { ok: 0, internal: 1, usage: 2, setup: 3, refused: 4, key: 5, unavailable: 6, pending: 7 }; const MiB = 1024 * 1024; -const LIMITS = { image: 20 * MiB, audio: 25 * MiB, download: 50 * MiB, speech: 50 * MiB }; +const LIMITS = { image: 20 * MiB, audio: 25 * MiB, download: 50 * MiB, speech: 50 * MiB, video: 2048 * MiB }; class Failure extends Error { constructor(code, message) { @@ -49,6 +49,9 @@ function loadConfig() { if (typeof config[field] !== "string" || !config[field]) fail(EXIT.setup, `config.json has no ${field}. Reinstall the Skill from BootAgent.`); } if (!/^https?:\/\//.test(config.base_url)) fail(EXIT.setup, "config.json has an invalid base_url. Reinstall the Skill from BootAgent."); + if (NATIVE.has(config.adapter) && !/^https:\/\/[^/]+$/.test(config.native_base_url ?? "")) { + fail(EXIT.setup, "config.json has no native_base_url. Reinstall the Skill from BootAgent."); + } return config; } @@ -175,8 +178,11 @@ function detail(text) { return message.length > 400 ? `${message.slice(0, 400)}…` : message; } -async function call(config, key, path, init, timeoutMs) { - const url = config.base_url.replace(/\/+$/, "") + path; +const NATIVE = new Set(["novita-audio", "novita-async"]); + +// base picks the OpenAI-compatible route or the Provider's host root. +async function call(config, key, path, init, timeoutMs, base = config.base_url) { + const url = base.replace(/\/+$/, "") + path; let response; try { response = await fetch(url, { @@ -317,18 +323,172 @@ async function speak(config, key, options, timeoutMs) { process.stdout.write(writeOutput(target, data) + "\n"); } -const COMMANDS = { - vision: { name: "describe", run: describe, timeout: 120 }, - "image-generation": { name: "generate", run: generate, timeout: 300 }, - "speech-to-text": { name: "transcribe", run: transcribe, timeout: 300 }, - "text-to-speech": { name: "speak", run: speak, timeout: 120 }, -}; +// MiniMax-style routes answer HTTP 200 for some refusals and carry the +// verdict in base_resp.status_code. +function checkBaseResp(body) { + const code = body?.base_resp?.status_code; + if (code === undefined || code === 0) return; + const said = detail(JSON.stringify({ message: body?.base_resp?.status_msg ?? "" })); + const suffix = said ? `: ${said}` : "."; + if (code === 1004) fail(EXIT.key, `The provider rejected the API key (${code})${suffix}`); + if (code === 1002 || code === 1039) fail(EXIT.unavailable, `The provider is rate limiting (${code})${suffix}`); + if (code === 1000 || code === 1001) fail(EXIT.unavailable, `The provider failed (${code})${suffix}`); + fail(EXIT.refused, `The provider refused the request (${code})${suffix}`); +} + +const NATIVE_SPEECH_FORMATS = ["mp3", "wav", "flac", "pcm"]; + +async function speakNative(config, key, options, timeoutMs) { + if (!options.text) fail(EXIT.usage, "Give the --text to speak."); + if (options.text.length > 9999) fail(EXIT.usage, "--text is longer than 9999 characters; split it."); + const format = (options.format || "mp3").toLowerCase(); + if (!NATIVE_SPEECH_FORMATS.includes(format)) fail(EXIT.usage, `--format must be one of ${NATIVE_SPEECH_FORMATS.join(", ")}.`); + const voice = { voice_id: options.voice || "English_Graceful_Lady" }; + if (options.speed !== undefined) { + const speed = Number(options.speed); + if (!(speed >= 0.5 && speed <= 2)) fail(EXIT.usage, "--speed must be from 0.5 to 2."); + voice.speed = speed; + } + const response = await call(config, key, `/v3/${encodeURIComponent(config.model)}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ text: options.text, voice_setting: voice, audio_setting: { format }, output_format: "hex" }), + }, timeoutMs, config.native_base_url); + const body = await json(response); + checkBaseResp(body); + const hex = body?.data?.audio; + if (typeof hex !== "string" || !hex || !/^[0-9a-fA-F]+$/.test(hex) || hex.length % 2) fail(EXIT.unavailable, "The provider returned no audio."); + const data = Buffer.from(hex, "hex"); + if (data.length > LIMITS.speech) fail(EXIT.unavailable, "The audio is larger than expected and was not saved."); + const target = options.out && extname(options.out) + ? options.out + : join(options.out || config.output_dir || "bootagent-output", `speech-${stamp()}.${format}`); + process.stdout.write(writeOutput(target, data) + "\n"); +} + +const VIDEO_FLAGS = { duration: "duration", size: "size", "aspect-ratio": "aspect_ratio", "negative-prompt": "negative_prompt", sound: "sound" }; + +async function generateVideo(config, key, options, timeoutMs) { + if (!options.prompt) fail(EXIT.usage, "Give a --prompt."); + const allowed = new Set(config.options ?? []); + const request = { prompt: options.prompt }; + for (const [flag, field] of Object.entries(VIDEO_FLAGS)) { + if (options[flag] === undefined) continue; + if (!allowed.has(field)) fail(EXIT.usage, `${config.model} does not take --${flag}.`); + if (field === "duration") request.duration = positiveInt(options[flag], flag, 5, 60); + else if (field === "sound") { + if (options[flag] !== "true" && options[flag] !== "false") fail(EXIT.usage, "--sound must be true or false."); + request.sound = options[flag] === "true"; + } else request[field] = options[flag]; + } + for (const flag of Object.keys(options)) { + if (!["_", "prompt", "out", "timeout", ...Object.keys(VIDEO_FLAGS)].includes(flag)) fail(EXIT.usage, `Unknown option --${flag}.`); + } + const deadline = Date.now() + timeoutMs; + const response = await call(config, key, `/v3/async/${encodeURIComponent(config.model)}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(request), + }, Math.min(timeoutMs, 60000), config.native_base_url); + const body = await json(response); + const task = body?.task_id; + if (typeof task !== "string" || !task) fail(EXIT.unavailable, "The provider returned no task id."); + process.stderr.write(`task: ${task}\n`); + await finishVideo(config, key, task, options, deadline); +} + +async function videoResult(config, key, options, timeoutMs) { + if (!options.task || !/^[A-Za-z0-9._-]{1,128}$/.test(options.task)) fail(EXIT.usage, "Give the --task id the video command printed."); + await finishVideo(config, key, options.task, options, Date.now() + timeoutMs); +} + +const sleep = (ms) => new Promise((done) => setTimeout(done, ms)); + +// Polls the task until it settles or the deadline passes. A busy or briefly +// unreachable provider is polled again; a refusal or a failed task is final, +// because retrying a generation spends money. +async function finishVideo(config, key, task, options, deadline) { + let delay = 5000; + for (;;) { + let body; + try { + const response = await call(config, key, `/v3/async/task-result?task_id=${encodeURIComponent(task)}`, { method: "GET" }, 30000, config.native_base_url); + body = await json(response); + } catch (error) { + if (!(error instanceof Failure) || error.code !== EXIT.unavailable) throw error; + } + const status = body?.task?.status; + if (status === "TASK_STATUS_SUCCEED") { + const videos = Array.isArray(body?.videos) ? body.videos.filter((item) => typeof item?.video_url === "string") : []; + if (!videos.length) fail(EXIT.unavailable, `Task ${task} finished without a video.`); + const directory = resolve(options.out || config.output_dir || "bootagent-output"); + const saved = []; + for (const [index, item] of videos.entries()) { + const extension = item.video_type === "gif" ? "gif" : "mp4"; + const name = `video-${stamp()}${videos.length > 1 ? `-${index + 1}` : ""}.${extension}`; + saved.push(await downloadTo(item.video_url, join(directory, name), Math.max(deadline - Date.now(), 120000))); + } + process.stdout.write(saved.join("\n") + "\n"); + return; + } + if (status === "TASK_STATUS_FAILED") { + const reason = redact(String(body?.task?.reason ?? "")).slice(0, 400); + fail(EXIT.refused, `Task ${task} failed${reason ? `: ${reason}` : "."}`); + } + // The provider's eta paces the next poll when it gives one; otherwise + // back off from 5 s to 15 s. + const eta = Number(body?.task?.eta); + const wait = Number.isFinite(eta) && eta > 0 ? Math.min(Math.max(eta * 1000, 1000), 15000) : delay; + if (Date.now() + wait > deadline) { + fail(EXIT.pending, `Task ${task} has not finished yet. It may still finish and be billed; fetch it later with: video-result --task ${task}`); + } + await sleep(wait); + delay = Math.min(delay + 5000, 15000); + } +} + +// Streams a result to disk, so a large video is never held in memory, and +// removes the partial file on failure. The URL is pre-signed and may be on +// another host, so the key is not sent. +async function downloadTo(url, path, timeoutMs) { + let response; + try { + response = await fetch(url, { signal: AbortSignal.timeout(timeoutMs) }); + } catch { + fail(EXIT.unavailable, "Cannot download the result from the provider."); + } + if (!response.ok || !response.body) fail(EXIT.unavailable, `Cannot download the result (HTTP ${response.status}).`); + const target = freePath(resolve(path)); + mkdirSync(dirname(target), { recursive: true }); + const partial = `${target}.part`; + const out = createWriteStream(partial); + let size = 0; + try { + for await (const chunk of response.body) { + size += chunk.length; + if (size > LIMITS.video) throw new Failure(EXIT.unavailable, "The result is larger than expected and was not saved."); + if (!out.write(chunk)) await new Promise((done) => out.once("drain", done)); + } + await new Promise((done, failed) => out.end((error) => (error ? failed(error) : done()))); + renameSync(partial, target); + } catch (error) { + out.destroy(); + rmSync(partial, { force: true }); + if (error instanceof Failure) throw error; + fail(EXIT.unavailable, "The download was interrupted."); + } + return target; +} -const ADAPTERS = { - vision: "openai-chat-vision", - "image-generation": "openai-images", - "speech-to-text": "openai-audio-transcriptions", - "text-to-speech": "openai-audio-speech", +// Each capability and adapter this script implements: the subcommand, its +// handler and default timeout in seconds, plus any extra subcommands. +const COMMANDS = { + "vision/openai-chat-vision": { name: "describe", run: describe, timeout: 120 }, + "image-generation/openai-images": { name: "generate", run: generate, timeout: 300 }, + "speech-to-text/openai-audio-transcriptions": { name: "transcribe", run: transcribe, timeout: 300 }, + "text-to-speech/openai-audio-speech": { name: "speak", run: speak, timeout: 120 }, + "text-to-speech/novita-audio": { name: "speak", run: speakNative, timeout: 120 }, + "video-generation/novita-async": { name: "video", run: generateVideo, timeout: 600, extra: { "video-result": videoResult } }, }; async function main() { @@ -337,8 +497,8 @@ async function main() { fail(EXIT.setup, `Node.js 18 or later is required; this is ${process.versions.node}.`); } const config = loadConfig(); - const command = COMMANDS[config.capability]; - if (!command || ADAPTERS[config.capability] !== config.adapter) { + const command = COMMANDS[`${config.capability}/${config.adapter}`]; + if (!command) { fail(EXIT.setup, `This version of the script cannot run ${config.capability} over ${config.adapter}. Reinstall the Skill from BootAgent.`); } const [subcommand, ...rest] = process.argv.slice(2); @@ -351,12 +511,13 @@ async function main() { process.stdout.write(`ready: ${config.capability} with ${config.model}\n`); return; } - if (subcommand !== command.name) fail(EXIT.usage, `This Skill runs "${command.name}", not "${subcommand}".`); + const run = subcommand === command.name ? command.run : command.extra?.[subcommand]; + if (!run) fail(EXIT.usage, `This Skill runs "${command.name}", not "${subcommand}".`); const options = parseArgs(rest); if (options._.length) fail(EXIT.usage, `Unexpected argument: ${options._[0]}`); - const timeout = positiveInt(options.timeout, "timeout", command.timeout, 3600) * 1000; + const timeout = positiveInt(options.timeout, "timeout", command.timeout, 7200) * 1000; const key = loadKey(config); - await command.run(config, key, options, timeout); + await run(config, key, options, timeout); } main().then( diff --git a/internal/skillgen/templates/text-to-speech.md.tmpl b/internal/skillgen/templates/text-to-speech.md.tmpl index 18fefaed..91846c7c 100644 --- a/internal/skillgen/templates/text-to-speech.md.tmpl +++ b/internal/skillgen/templates/text-to-speech.md.tmpl @@ -6,7 +6,18 @@ When the user asks you to turn text into spoken audio: reading out a summary, pr ## Command Run the script in this Skill's directory (the directory containing this `SKILL.md`): +{{if eq .Adapter "novita-audio"}} +```bash +node /scripts/run.mjs speak --text "" [--voice ] [--format mp3|wav|flac|pcm] [--speed <0.5-2>] [--out ] +``` +- `--text`: what to say, at most 9999 characters. Split longer text into several runs. +- `--voice`: a MiniMax system voice ID. Default `English_Graceful_Lady`; others include `English_Insightful_Speaker`, `English_Persuasive_Man`, `Chinese (Mandarin)_Lyrical_Voice` and `Japanese_Whisper_Belle`. Quote IDs that contain spaces. +- `--format`: audio format. Default `mp3`. +- `--speed`: speaking speed from 0.5 to 2. Default 1. +- `--out`: a file path, or a directory. Default `bootagent-output` in the current directory. +- `--timeout`: seconds to wait. Default 120. +{{else}} ```bash node /scripts/run.mjs speak --text "" [--voice ] [--format mp3|wav|opus|aac|flac|pcm] [--out ] ``` @@ -16,6 +27,6 @@ node /scripts/run.mjs speak --text "" [--voice /scripts/run.mjs video --prompt ""{{if has .Options "duration"}} [--duration ]{{end}}{{if has .Options "size"}} [--size ]{{end}}{{if has .Options "aspect_ratio"}} [--aspect-ratio 16:9|9:16|1:1]{{end}}{{if has .Options "negative_prompt"}} [--negative-prompt ""]{{end}}{{if has .Options "sound"}} [--sound true|false]{{end}} [--out ] +``` + +- `--prompt`: a full description of the video: subject, motion, camera, mood. +{{- if has .Options "duration"}} +- `--duration`: length in seconds. Billed by the second; ask the user before going above the default of 5. +{{- end}} +{{- if has .Options "size"}} +- `--size`: resolution as width*height, for example `1280*720` or `1920*1080`. Larger costs more. +{{- end}} +{{- if has .Options "aspect_ratio"}} +- `--aspect-ratio`: `16:9`, `9:16` or `1:1`. +{{- end}} +{{- if has .Options "negative_prompt"}} +- `--negative-prompt`: what must not appear. +{{- end}} +{{- if has .Options "sound"}} +- `--sound`: `true` to generate audio with the video. +{{- end}} +- `--out`: directory for the file. Default `bootagent-output` in the current directory. +- `--timeout`: seconds to wait for the result. Default 600. + +The script submits the task, prints `task: ` to stderr, waits for it, downloads the video before the Provider's link expires, and prints the saved file path to stdout. If it times out (exit code 7), the task may still finish: fetch it later with + +```bash +node /scripts/run.mjs video-result --task [--out ] +``` + +instead of submitting the same prompt again. +{{template "footer" .}} diff --git a/internal/skillgen/testdata/golden/text-to-speech-native/SKILL.md b/internal/skillgen/testdata/golden/text-to-speech-native/SKILL.md new file mode 100644 index 00000000..25911097 --- /dev/null +++ b/internal/skillgen/testdata/golden/text-to-speech-native/SKILL.md @@ -0,0 +1,48 @@ +--- +name: bootagent-text-to-speech +description: "Turn text into a spoken audio file through minimax-speech-2.8-turbo on PPIO. Use when the user asks for audio of some text. Each run is billed: confirm with the user first." +--- + +# Text to speech + +Generated by BootAgent. It calls `minimax-speech-2.8-turbo` on the user's PPIO account. BootAgent rewrites this Skill when the model or Provider changes, so do not edit it. + +## When to use + +When the user asks you to turn text into spoken audio: reading out a summary, producing a voice-over draft. + +## Command + +Run the script in this Skill's directory (the directory containing this `SKILL.md`): + +```bash +node /scripts/run.mjs speak --text "" [--voice ] [--format mp3|wav|flac|pcm] [--speed <0.5-2>] [--out ] +``` + +- `--text`: what to say, at most 9999 characters. Split longer text into several runs. +- `--voice`: a MiniMax system voice ID. Default `English_Graceful_Lady`; others include `English_Insightful_Speaker`, `English_Persuasive_Man`, `Chinese (Mandarin)_Lyrical_Voice` and `Japanese_Whisper_Belle`. Quote IDs that contain spaces. +- `--format`: audio format. Default `mp3`. +- `--speed`: speaking speed from 0.5 to 2. Default 1. +- `--out`: a file path, or a directory. Default `bootagent-output` in the current directory. +- `--timeout`: seconds to wait. Default 120. + +The saved file path is printed to stdout. Existing files are never overwritten. + +## Rules + +- Every run costs money, billed by the length of the text. Before each run, tell the user what text you are about to speak and wait for them to agree. +- Never print, echo or log environment variables, `~/.bootagent/skill-env/ppio.env`, or anything that could reveal the API key. The script reads the key itself; you never need it. +- Only send files the user asked about. They go to the user's own Provider and nowhere else. + +## Exit codes + +| Code | Meaning | What to tell the user | +| --- | --- | --- | +| 0 | Success | Report the result. | +| 2 | Bad arguments or input file | Fix the command, or ask the user for the right file. | +| 3 | Setup problem: Node.js older than 18, or no API key found | Ask the user to restart the Agent so it picks up `BOOTAGENT_PPIO_API_KEY`, or to reinstall this Skill from BootAgent. | +| 4 | The Provider refused the request | Show the Provider's message. The model may not support this input; the user can choose another model in BootAgent. | +| 5 | The Provider rejected the API key | Ask the user to check the PPIO key in BootAgent. | +| 6 | The Provider was unreachable, busy or timed out | Suggest trying again later. | + +`node /scripts/run.mjs check` confirms the key is available without sending a request. diff --git a/internal/skillgen/testdata/golden/text-to-speech-native/config.json b/internal/skillgen/testdata/golden/text-to-speech-native/config.json new file mode 100644 index 00000000..8b566c79 --- /dev/null +++ b/internal/skillgen/testdata/golden/text-to-speech-native/config.json @@ -0,0 +1,14 @@ +{ + "schema_version": 1, + "generator": "bootagent", + "capability": "text-to-speech", + "adapter": "novita-audio", + "provider_id": "ppio", + "provider_name": "PPIO", + "model": "minimax-speech-2.8-turbo", + "base_url": "https://api.ppio.com/openai/v1", + "native_base_url": "https://api.ppio.com", + "key_env": "BOOTAGENT_PPIO_API_KEY", + "key_file": "~/.bootagent/skill-env/ppio.env", + "output_dir": "bootagent-output" +} diff --git a/internal/skillgen/testdata/golden/video-generation/SKILL.md b/internal/skillgen/testdata/golden/video-generation/SKILL.md new file mode 100644 index 00000000..045ec326 --- /dev/null +++ b/internal/skillgen/testdata/golden/video-generation/SKILL.md @@ -0,0 +1,55 @@ +--- +name: bootagent-video-generation +description: "Generate a short video from a text description through wan2.7-t2v on JieKou.AI and save it as a file. Use when the user asks you to create a video. Each run is billed by the second and cannot be cancelled: confirm with the user first." +--- + +# Video generation + +Generated by BootAgent. It calls `wan2.7-t2v` on the user's JieKou.AI account. BootAgent rewrites this Skill when the model or Provider changes, so do not edit it. + +## When to use + +When the user asks you to create a short video clip from a description. Video is the most expensive thing this Skill can do; read the rules below first. + +## Command + +Run the script in this Skill's directory (the directory containing this `SKILL.md`): + +```bash +node /scripts/run.mjs video --prompt "" [--duration ] [--size ] [--negative-prompt ""] [--out ] +``` + +- `--prompt`: a full description of the video: subject, motion, camera, mood. +- `--duration`: length in seconds. Billed by the second; ask the user before going above the default of 5. +- `--size`: resolution as width*height, for example `1280*720` or `1920*1080`. Larger costs more. +- `--negative-prompt`: what must not appear. +- `--out`: directory for the file. Default `bootagent-output` in the current directory. +- `--timeout`: seconds to wait for the result. Default 600. + +The script submits the task, prints `task: ` to stderr, waits for it, downloads the video before the Provider's link expires, and prints the saved file path to stdout. If it times out (exit code 7), the task may still finish: fetch it later with + +```bash +node /scripts/run.mjs video-result --task [--out ] +``` + +instead of submitting the same prompt again. + +## Rules + +- Every run costs money, billed by the second of video, and a submitted run cannot be cancelled. Before each run, tell the user the prompt, the duration and the resolution or aspect ratio you will use, and wait for them to agree. Never retry a failed or timed-out run on your own: a timed-out run may still finish and be billed. +- Never print, echo or log environment variables, `~/.bootagent/skill-env/jiekou.env`, or anything that could reveal the API key. The script reads the key itself; you never need it. +- Only send files the user asked about. They go to the user's own Provider and nowhere else. + +## Exit codes + +| Code | Meaning | What to tell the user | +| --- | --- | --- | +| 0 | Success | Report the result. | +| 2 | Bad arguments or input file | Fix the command, or ask the user for the right file. | +| 3 | Setup problem: Node.js older than 18, or no API key found | Ask the user to restart the Agent so it picks up `BOOTAGENT_JIEKOU_API_KEY`, or to reinstall this Skill from BootAgent. | +| 4 | The Provider refused the request | Show the Provider's message. The model may not support this input; the user can choose another model in BootAgent. | +| 5 | The Provider rejected the API key | Ask the user to check the JieKou.AI key in BootAgent. | +| 6 | The Provider was unreachable, busy or timed out | Suggest trying again later. | +| 7 | The task was submitted but had not finished in time | Tell the user it may still finish and be billed; offer to fetch it with `video-result` instead of submitting again. | + +`node /scripts/run.mjs check` confirms the key is available without sending a request. diff --git a/internal/skillgen/testdata/golden/video-generation/config.json b/internal/skillgen/testdata/golden/video-generation/config.json new file mode 100644 index 00000000..db262b54 --- /dev/null +++ b/internal/skillgen/testdata/golden/video-generation/config.json @@ -0,0 +1,19 @@ +{ + "schema_version": 1, + "generator": "bootagent", + "capability": "video-generation", + "adapter": "novita-async", + "provider_id": "jiekou", + "provider_name": "JieKou.AI", + "model": "wan2.7-t2v", + "base_url": "https://api.highwayapi.ai/openai/v1", + "native_base_url": "https://api.highwayapi.ai", + "options": [ + "duration", + "size", + "negative_prompt" + ], + "key_env": "BOOTAGENT_JIEKOU_API_KEY", + "key_file": "~/.bootagent/skill-env/jiekou.env", + "output_dir": "bootagent-output" +} diff --git a/manifests/modalities.lock.json b/manifests/modalities.lock.json index a5a9fd19..ef4c41af 100644 --- a/manifests/modalities.lock.json +++ b/manifests/modalities.lock.json @@ -1,33 +1,154 @@ { "schema_version": 1, "id_rules": [ - { "pattern": "^gpt-image-", "capabilities": ["image-generation"] }, - { "pattern": "^gemini-.*-image", "capabilities": ["vision", "image-generation"] }, - { "pattern": "^ming-image-", "capabilities": ["image-generation"] }, - { "pattern": "(^|-)omni(-|$)", "exclude": "captioner", "capabilities": ["vision", "speech-to-text"] }, - { "pattern": "^gemini-", "exclude": "-image", "capabilities": ["vision", "video-understanding", "speech-to-text"] }, - { "pattern": "^claude-", "capabilities": ["vision"] }, - { "pattern": "^(gpt-4o|gpt-4\\.1|gpt-5|gpt-6|o3)", "exclude": "^gpt-oss", "capabilities": ["vision"] }, - { "pattern": "^grok-(4|code)", "capabilities": ["vision"] }, - { "pattern": "^kimi-k(2\\.[5-9]|3)", "capabilities": ["vision", "video-understanding"] }, - { "pattern": "^deepseek-(v4\\.[1-9]-flash|v4-flash-vision)", "capabilities": ["vision"] }, - { "pattern": "(^|-)ocr(-|$)", "capabilities": ["vision"] }, - { "pattern": "(^|-)vl(-|$)", "capabilities": ["vision"] }, - { "pattern": "^glm-[\\d.]+v", "capabilities": ["vision"] }, - { "pattern": "^gemma-[34]-", "capabilities": ["vision"] }, - { "pattern": "^llama-4-", "capabilities": ["vision"] }, - { "pattern": "^minimax-m3", "capabilities": ["vision", "video-understanding"] }, - { "pattern": "^qwen3\\.(5|6|8)-(\\d+b|flash|max|plus)", "capabilities": ["vision", "video-understanding"] }, - { "pattern": "^mimo-v2\\.(5$|6-)", "capabilities": ["vision", "video-understanding"] } + {"pattern": "^gpt-image-", "capabilities": ["image-generation"]}, + {"pattern": "^gemini-.*-image", "capabilities": ["vision", "image-generation"]}, + {"pattern": "^ming-image-", "capabilities": ["image-generation"]}, + {"pattern": "(^|-)omni(-|$)", "exclude": "captioner", "capabilities": ["vision", "speech-to-text"]}, + {"pattern": "^gemini-", "exclude": "-image", "capabilities": ["vision", "video-understanding", "speech-to-text"]}, + {"pattern": "^claude-", "capabilities": ["vision"]}, + {"pattern": "^(gpt-4o|gpt-4\\.1|gpt-5|gpt-6|o3)", "exclude": "^gpt-oss", "capabilities": ["vision"]}, + {"pattern": "^grok-(4|code)", "capabilities": ["vision"]}, + {"pattern": "^kimi-k(2\\.[5-9]|3)", "capabilities": ["vision", "video-understanding"]}, + {"pattern": "^deepseek-(v4\\.[1-9]-flash|v4-flash-vision)", "capabilities": ["vision"]}, + {"pattern": "(^|-)ocr(-|$)", "capabilities": ["vision"]}, + {"pattern": "(^|-)vl(-|$)", "capabilities": ["vision"]}, + {"pattern": "^glm-[\\d.]+v", "capabilities": ["vision"]}, + {"pattern": "^gemma-[34]-", "capabilities": ["vision"]}, + {"pattern": "^llama-4-", "capabilities": ["vision"]}, + {"pattern": "^minimax-m3", "capabilities": ["vision", "video-understanding"]}, + {"pattern": "^qwen3\\.(5|6|8)-(\\d+b|flash|max|plus)", "capabilities": ["vision", "video-understanding"]}, + {"pattern": "^mimo-v2\\.(5$|6-)", "capabilities": ["vision", "video-understanding"]} ], "providers": { "novita": { "native_base_url": "https://api.novita.ai", - "generation": [] + "generation": [ + { + "id": "ming-image-0.1-design", + "capabilities": [ + "image-generation" + ], + "adapter": "openai-images", + "recommended": true + }, + { + "id": "minimax-speech-2.8-turbo", + "capabilities": [ + "text-to-speech" + ], + "adapter": "novita-audio", + "recommended": true + }, + { + "id": "minimax-speech-2.8-hd", + "capabilities": [ + "text-to-speech" + ], + "adapter": "novita-audio" + } + ], + "unsupported": [ + "video-generation" + ] }, "ppio": { "native_base_url": "https://api.ppio.com", - "generation": [] + "generation": [ + { + "id": "minimax-speech-2.8-turbo", + "capabilities": [ + "text-to-speech" + ], + "adapter": "novita-audio", + "recommended": true + }, + { + "id": "minimax-speech-2.8-hd", + "capabilities": [ + "text-to-speech" + ], + "adapter": "novita-audio" + } + ], + "unsupported": [ + "video-generation" + ] + }, + "jiekou": { + "native_base_url": "https://api.highwayapi.ai", + "generation": [ + { + "id": "gpt-image-2.5-flare-oai", + "capabilities": [ + "image-generation" + ], + "adapter": "openai-images", + "recommended": true + }, + { + "id": "gpt-image-2.5-sunburst-oai", + "capabilities": [ + "image-generation" + ], + "adapter": "openai-images" + }, + { + "id": "gpt-image-2", + "capabilities": [ + "image-generation" + ], + "adapter": "openai-images" + }, + { + "id": "gpt-image-2-oai", + "capabilities": [ + "image-generation" + ], + "adapter": "openai-images" + }, + { + "id": "wan2.7-t2v", + "capabilities": [ + "video-generation" + ], + "adapter": "novita-async", + "options": [ + "duration", + "size", + "negative_prompt" + ], + "recommended": true + }, + { + "id": "kling-v3.0-4k-t2v", + "capabilities": [ + "video-generation" + ], + "adapter": "novita-async", + "options": [ + "duration", + "aspect_ratio", + "negative_prompt", + "sound" + ] + }, + { + "id": "minimax-speech-2.8-turbo", + "capabilities": [ + "text-to-speech" + ], + "adapter": "novita-audio", + "recommended": true + }, + { + "id": "minimax-speech-2.8-hd", + "capabilities": [ + "text-to-speech" + ], + "adapter": "novita-audio" + } + ] } } } From 80c16b0d2393659d7d96ac59b495bf0fea3fdf16 Mon Sep 17 00:00:00 2001 From: yujiezhang-ops Date: Fri, 9 Oct 2026 17:34:20 +0800 Subject: [PATCH 08/18] feat: add video understanding and Skills for more Agents Answer questions about a local video by sampling frames with ffmpeg and sending them to an image-understanding model, which lends the capability its own evidence. ffmpeg runs with an argv array, a timeout and PATH alone. Native video content parts stay probe-only: Moonshot takes video only as an uploaded file, not a data URL. Offer OpenCode as an install target on the private fallback key file, since neither OpenCode nor Hermes documents a way to set its shell tool's environment. Hermes is held back pending the decision on Node. Agents without a Skills directory can get the same tree as a guide in ~/.bootagent/multimodal// with a prompt pointing at it. The guide is found by its hash rather than registered as a target, and follows the Skill through key changes, Provider deletes and uninstall. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 2 +- README_ZH.md | 2 +- docs/multimodal-skills-design.md | 58 ++++++-- .../BootAgent/internal/app/models.ts | 36 +++++ frontend/src/backend/wails.ts | 2 + .../src/components/MultimodalSection.test.tsx | 33 ++++- frontend/src/components/MultimodalSection.tsx | 66 ++++++++- frontend/src/i18n.tsx | 6 + frontend/src/types/api.ts | 4 + internal/app/multimodal.go | 36 ++++- internal/app/multimodal_install.go | 139 ++++++++++++++++-- internal/app/multimodal_install_test.go | 137 ++++++++++++++++- internal/app/skill.go | 10 ++ internal/modality/capability.go | 15 +- internal/modality/resolve.go | 47 ++++++ internal/modality/resolve_test.go | 47 +++++- internal/skillgen/skillgen.go | 44 +++--- internal/skillgen/skillgen_test.go | 4 + internal/skillgen/templates/run.mjs | 90 +++++++++++- .../templates/video-understanding.md.tmpl | 25 ++++ .../golden/video-understanding/SKILL.md | 51 +++++++ .../golden/video-understanding/config.json | 13 ++ internal/skillgen/video_script_test.go | 108 ++++++++++++++ 23 files changed, 900 insertions(+), 75 deletions(-) create mode 100644 internal/skillgen/templates/video-understanding.md.tmpl create mode 100644 internal/skillgen/testdata/golden/video-understanding/SKILL.md create mode 100644 internal/skillgen/testdata/golden/video-understanding/config.json create mode 100644 internal/skillgen/video_script_test.go diff --git a/README.md b/README.md index bd2f1588..ffe70feb 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ Launch BootAgent - Keep a local Skill library, scan Skills already present on supported Agents, and choose which Agents receive each Skill. - See which models on your keyed Providers can understand images, video or audio, or generate images, and where each finding comes from (declared by the Provider, inferred from the model name, or probed). Listing models is free; a probe is a real request and is sent only when you click it, and an image-generation probe asks first. -- Install a multimodal Skill (image understanding, image generation, speech-to-text, text-to-speech, video generation) into Claude Code or Codex, with example prompts to copy. Built-in Providers add the models their docs name: MiniMax Speech on Novita, PPIO and JieKou.AI, GPT Image and async text-to-video on JieKou.AI. Video is never probed, because a probe would be a full billed generation. Its script needs Node.js 18 or later. BootAgent writes the Provider key into the Agent's own configuration as `BOOTAGENT__API_KEY`, so the Agent and its model can read it; the install confirmation says so. Uninstalling the last Skill that uses a Provider removes the variable. +- Install a multimodal Skill (image understanding, image generation, speech-to-text, text-to-speech, video generation, video understanding) into Claude Code, Codex or OpenCode, with example prompts to copy. Agents without a Skills directory get the same files under `~/.bootagent/multimodal/` and a prompt that points them there. Video understanding samples frames with ffmpeg, which must be on `PATH`. Built-in Providers add the models their docs name: MiniMax Speech on Novita, PPIO and JieKou.AI, GPT Image and async text-to-video on JieKou.AI. Video is never probed, because a probe would be a full billed generation. Its script needs Node.js 18 or later. BootAgent writes the Provider key into the Agent's own configuration as `BOOTAGENT__API_KEY`, or into a private file under `~/.bootagent/skill-env/` for Agents with no configurable environment, so the Agent and its model can read it; the install confirmation says so. Uninstalling the last Skill that uses a Provider removes the variable. - Discover MCP servers from initialized Claude Code, Codex, OpenCode, Kilo CLI, and Hermes installations. - Store MCP servers in a local user-level Registry, select sync targets explicitly, and apply changes only when you are ready. diff --git a/README_ZH.md b/README_ZH.md index 52f87913..6d2d7c64 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -62,7 +62,7 @@ BootAgent 是一个本地桌面工作台，用来统一管理 AI 编程 Agent。 - 在本机维护 Skill 管理库，扫描支持的 Agent 已有的 Skill，并选择每个 Skill 要同步到哪些 Agent。 - 查看已保存 API Key 的 Provider 中哪些模型能理解图片、视频或音频，或能生成图片，以及每项结论的来源（Provider 声明、按模型名推断或实测）。列出模型不收费；实测是一次真实请求，只在你点击时发出，图片生成实测会先确认。 -- 把多模态 Skill（图片理解、图片生成、语音识别、语音合成、视频生成）安装到 Claude Code 或 Codex，并提供可复制的示例提示词。内置 Provider 会补充其文档中列出的模型：Novita、PPIO 和 JieKou.AI 的 MiniMax Speech，JieKou.AI 的 GPT Image 与异步文生视频。视频生成从不实测，因为一次实测就是一次完整的计费生成。Skill 脚本需要 Node.js 18 或更高版本。BootAgent 会把 Provider 的 Key 以 `BOOTAGENT__API_KEY` 写进 Agent 自己的配置，因此 Agent 及其模型能读到它，安装确认里会说明。卸载最后一个用到该 Provider 的 Skill 时，这个变量会被移除。 +- 把多模态 Skill（图片理解、图片生成、语音识别、语音合成、视频生成、视频理解）安装到 Claude Code、Codex 或 OpenCode，并提供可复制的示例提示词。没有 Skills 目录的 Agent 会在 `~/.bootagent/multimodal/` 下得到同样的文件，并附一段指向它的提示词。视频理解用 ffmpeg 抽帧，需要 ffmpeg 在 `PATH` 上。内置 Provider 会补充其文档中列出的模型：Novita、PPIO 和 JieKou.AI 的 MiniMax Speech，JieKou.AI 的 GPT Image 与异步文生视频。视频生成从不实测，因为一次实测就是一次完整的计费生成。Skill 脚本需要 Node.js 18 或更高版本。BootAgent 会把 Provider 的 Key 以 `BOOTAGENT__API_KEY` 写进 Agent 自己的配置；没有可配置环境变量的 Agent 则写进 `~/.bootagent/skill-env/` 下的私有文件。因此 Agent 及其模型能读到它，安装确认里会说明。卸载最后一个用到该 Provider 的 Skill 时，这个变量会被移除。 - 从已初始化的 Claude Code、Codex、OpenCode、Kilo CLI 和 Hermes 中发现 MCP 服务器。 - 使用本机用户级 Registry 保存 MCP 服务器，明确选择同步目标，并在确认后再应用改动。 diff --git a/docs/multimodal-skills-design.md b/docs/multimodal-skills-design.md index b03b09d3..fd47dd2a 100644 --- a/docs/multimodal-skills-design.md +++ b/docs/multimodal-skills-design.md @@ -1,6 +1,6 @@ # Multimodal Skills: Design -Status: In progress. Phases 1 to 3 are implemented: detection; generation and install for Claude Code and Codex; native speech and async video on the routes the vendors document. Branch `codex/multimodal-skills`. +Status: Phases 1 to 4 are implemented, with one open decision (Hermes, under Open questions). Branch `codex/multimodal-skills`. ## Goal @@ -170,10 +170,17 @@ probe passes, and each gets a probe once its format is verified (Phase 3). | Capability | Adapter | Installable from a declaration | | --- | --- | --- | | Image understanding | `openai-chat-vision` | Yes | -| Video understanding (native) | `openai-chat-video` | Yes | +| Video understanding (native) | `openai-chat-video` | No, probe first; no probe exists yet | +| Video understanding (frames) | `frames-chat-vision` | Follows the model's image-understanding evidence | | Image generation via chat output | `openai-chat-image-output` | No, probe first. JieKou.AI's `gemini-*-image` and `gpt-image-*`, whose `endpoints` is empty | | Speech-to-text via chat audio input | `openai-chat-audio-input` | No, probe first. `*-omni-*`, `gemini-*`, `mimo-v2.6-*` | +Native video was first listed as installable from a declaration. Phase 4 found +that vendors disagree on the content part: Moonshot accepts video only as a +file uploaded to its own storage (`"video_url": {"url": "ms://"}`), +not as a data URL. So native video now installs only after a probe, which none +has yet, and video understanding is reached by frame sampling instead. + ### ID rules Each rule is a regular expression matched against the model ID lower-cased with @@ -257,7 +264,7 @@ refused / key / transient / network classification from | Image generation | One image from `/images/generations`, no size sent | One image; the button says so and a dialog asks first, and the backend refuses the probe without that confirmation | | Text-to-speech (native) | The text `ok` to `POST /v3/` with a documented system voice; the verdict is read from `base_resp.status_code` too | Minimal | | Video generation | Not probed | A probe is a full billed generation that cannot be cancelled reliably, so this capability is catalog-only | -| Video understanding | Native video models: catalog-only. Frame sampling: inherits the image-understanding verdict | None | +| Video understanding | Native video: no probe yet, so not installable. Frame sampling: the image-understanding probe, recorded as image understanding | Minimal | Implementation notes: @@ -352,6 +359,17 @@ or a colon cannot break the frontmatter. | Speech-to-text | `transcribe --audio [--language] [--prompt]` | The transcript on stdout | | Text-to-speech | `speak --text [--voice] [--format] [--out]` (native route: also `--speed`) | The saved file path on stdout | | Video generation | `video --prompt [options the model documents] [--out]`, and `video-result --task ` | The saved file path on stdout; `task: ` on stderr | +| Video understanding | `describe-video --video [--prompt] [--frames 1-16] [--max-tokens]` | The answer on stdout | + +`describe-video` reads the duration from ffmpeg's own output, so it needs no +ffprobe, samples frames evenly at the middle of each slice, scales them to at +most 1024 px wide, and sends them in order to the chat vision route with a note +of their timestamps. ffmpeg runs with an argv array, a timeout and an +environment of `PATH` alone, so it never sees the key; the frames go to a +temporary directory that is removed afterwards. Without ffmpeg on `PATH` the +script exits 3. Any model with image understanding gets this capability with +that evidence; the Skills page warns when BootAgent itself finds no ffmpeg, +which is only a hint, since the Agent may run elsewhere. Every Skill also has `check`, which confirms the key is available without sending a request. Exit codes: 0 success, 2 bad arguments or input, 3 setup @@ -409,8 +427,8 @@ secret and stay in `config.json`. | --- | --- | --- | | Claude Code | `env` in `~/.claude/settings.json` | Already a secret file under ADR-008; the Bash tool inherits `env`. | | Codex | `[shell_environment_policy.set]` in `~/.codex/config.toml` | `set` is applied after Codex's default `KEY`/`SECRET`/`TOKEN` exclusion, so the variable survives it. The table is merged with the user's own entries and placed after the user's tables; inline-table and dotted-key spellings of it are refused rather than duplicated. Every writer of `config.toml` (Profile, MCP, Skill) writes it as a secret whenever the old or the new content holds a `BOOTAGENT_*` key, since the backup it takes is of the old content. | -| OpenCode | To be confirmed | How its shell tool receives configured environment variables is not yet checked. | -| Hermes | To be confirmed | Same as OpenCode. | +| OpenCode | The fallback file | Its config documents `{env:VAR}` substitution into config values and an `environment` key for formatters only, nothing that reaches the shell tool. | +| Hermes | The fallback file, once offered | Its README and `.env.example` say nothing about what its terminal tool inherits; under the SSH backend the Agent cannot read `.env`. Not offered yet (Open questions). | Fallback for an Agent with no configurable environment: a private `~/.bootagent/skill-env/.env`, the same pattern ADR-008 uses for @@ -453,9 +471,9 @@ The Skills page gets a Multimodal capabilities section: declared by the Provider, ID rule, catalog, or probed. 2. Per capability, a model choice and, where the probe table allows, a Probe button. -3. Target Agents: those with `skills_path` in `agents.lock.json` and a native - writer for the variable. In Phase 2 that is Claude Code and Codex; OpenCode - and Hermes follow in Phase 4. +3. Target Agents: those with `skills_path` in `agents.lock.json`: Claude Code, + Codex and OpenCode. Hermes is held back (Open questions). The confirmation + names the fallback file when a chosen Agent reads it. 4. Install: a confirmation names the variable, states that the Agent and its model can read it, and notes the Node 18 requirement. Then the tree is generated, stored and published, the variables are written, and the result is @@ -474,11 +492,21 @@ also runs the reconcile, so no separate multimodal uninstall exists. ### Agents without Skills Kilo CLI, Aider, OpenClaw, Kimi Code, pi, dsh and the desktop apps have no Skills -directory. BootAgent writes the same tree to `~/.bootagent/multimodal//` -and the variables to the fallback env file, and shows a prompt such as -"Read ~/.bootagent/multimodal/vision/SKILL.md and use it to …". It does not edit +directory. The install dialog's "Other Agents (write a guide)" option writes the +same tree to `~/.bootagent/multimodal//` and the key to the fallback +env file, and shows a prompt in both languages such as "First read +/Users/me/.bootagent/multimodal/vision/SKILL.md and follow it to do this: …", +with the absolute path, since not every Agent expands `~`. It does not edit those Agents' instruction files. +The guide is not a registry target, because it is not an Agent's Skills +directory and would fail the eligibility checks the uninstall and apply paths +make. It is found again by its hash: a guide directory whose tree is a stored +generated variant counts as installed, keeps its Provider's key file in the +reconcile, is re-rendered when the Provider's endpoint changes, blocks deleting +the Provider, and is removed with the Skill. A folder there that BootAgent did +not write is left alone, as in an Agent's Skills directory. + ## Security and privacy - The key never enters the Skill tree, `config.json`, the Skill registry, logs, @@ -519,7 +547,8 @@ those Agents' instruction files. 3. Native speech and async video on the routes Novita, PPIO and JieKou.AI document; catalog image models on the images route. PPIO video is not implemented (no documented route). -4. Video understanding; OpenCode, Hermes and the guide for Agents without Skills. +4. Video understanding by frame sampling; OpenCode on the fallback file; the + guide for Agents without Skills. Hermes is held back pending a decision. ## Open questions @@ -530,7 +559,8 @@ those Agents' instruction files. tools. - Hermes is installed by its official script, so Node may be absent. If so, should `SKILL.md` point at BootAgent's managed Node runtime by absolute path, or should - the Skill not be offered to Hermes? + the Skill not be offered to Hermes? Until this is decided, Hermes is not an + install target (`pendingSkillTargets` in `internal/app/multimodal_install.go`). - Whether the UI shows a price per call. The catalog has no pricing today; v1 shows a generic cost note only. @@ -542,3 +572,5 @@ those Agents' instruction files. - [PPIO: task result](https://ppio.com/docs/models/reference-get-async-task-result), [MiniMax Speech 2.8 Turbo](https://ppio.com/docs/models/reference-minimax-speech-2.8-turbo) - [JieKou.AI: task result](https://docs.jiekou.ai/docs/models/reference-get-async-task-result), [Wan 2.7 text-to-video](https://docs.jiekou.ai/docs/models/reference-wan2.7-t2v), [Kling v3.0 4K text-to-video](https://docs.jiekou.ai/docs/models/reference-kling-v3.0-4k-t2v), [GPT Image](https://docs.jiekou.ai/docs/models/reference-gpt-image), [Gemini](https://docs.jiekou.ai/docs/providers/gemini), [MiniMax Speech 2.8 Turbo](https://docs.jiekou.ai/docs/models/reference-minimax-speech-2.8-turbo) - [Codex: configuration reference](https://developers.openai.com/codex/config-reference) +- [OpenCode: configuration](https://opencode.ai/docs/config/) +- [Hermes Agent: README](https://github.com/NousResearch/hermes-agent), [.env.example](https://github.com/NousResearch/hermes-agent/blob/main/.env.example) diff --git a/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/app/models.ts b/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/app/models.ts index b6ee9fd7..23326ed8 100644 --- a/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/app/models.ts +++ b/frontend/bindings/github.com/MaimoryLab/BootAgent/internal/app/models.ts @@ -238,6 +238,12 @@ export interface InstalledMultimodal { "model": string; "adapter": string; "agents": string[] | null; + + /** + * GuidePath is set when this variant is also the guide for Agents + * without Skills. + */ + "guidePath"?: string; } export interface MCPAgentApplyResult { @@ -405,6 +411,18 @@ export interface MultimodalDetection { * hand, the route it is probed on, and how that probe is sent. */ "manual": MultimodalManualRoute[] | null; + + /** + * Fallback lists the Targets with no configurable environment: their key + * goes to BootAgent's private fallback file, which the install says. + */ + "fallback": string[] | null; + + /** + * FFmpeg says ffmpeg is on BootAgent's PATH. Video understanding needs it + * where the Agent runs, which may differ, so this is a hint only. + */ + "ffmpeg": boolean; } /** @@ -416,11 +434,23 @@ export interface MultimodalInstallRequest { "model": string; "capability": string; "agents": string[] | null; + + /** + * Guide also writes the Skill to ~/.bootagent/multimodal// + * for Agents without a Skills directory, which are pointed at it by a + * prompt. Their key goes to the fallback file. + */ + "guide": boolean; } export interface MultimodalInstallResult { "skillId": string; "results": MultimodalAgentResult[] | null; + + /** + * GuidePath is the guide directory when one was written. + */ + "guidePath"?: string; } export interface MultimodalManualRoute { @@ -459,6 +489,12 @@ export interface MultimodalProvider { */ "unsupported": string[] | null; + /** + * KeyFile is the private fallback file an install for an Agent without a + * configurable environment writes the key into. + */ + "keyFile": string; + /** * Listed is false when /models could not be read. Message and ErrorCode * then say why; catalog generation models are still listed, since they do diff --git a/frontend/src/backend/wails.ts b/frontend/src/backend/wails.ts index af0d04e9..7f651dd9 100644 --- a/frontend/src/backend/wails.ts +++ b/frontend/src/backend/wails.ts @@ -365,6 +365,8 @@ export const wailsApi = { targets: detection?.targets ?? [], generatable: detection?.generatable ?? [], manual: detection?.manual ?? [], + fallback: detection?.fallback ?? [], + ffmpeg: detection?.ffmpeg ?? false, })) as Promise, // Sends a live (billed) request; only called from a button the user presses. // `confirmed` is set only after the user agreed to a probe that produces diff --git a/frontend/src/components/MultimodalSection.test.tsx b/frontend/src/components/MultimodalSection.test.tsx index ca639c20..133048f7 100644 --- a/frontend/src/components/MultimodalSection.test.tsx +++ b/frontend/src/components/MultimodalSection.test.tsx @@ -26,12 +26,13 @@ import { I18nProvider, sourceTranslate } from "../i18n"; import { describeProbe, groupByCapability, groupInstalled, installLabel, MultimodalSection } from "./MultimodalSection"; import { EXAMPLE_PROMPTS } from "./multimodalPrompts"; -const GENERATABLE = ["openai-chat-vision", "openai-images", "openai-audio-transcriptions", "openai-audio-speech"]; +const GENERATABLE = ["openai-chat-vision", "openai-images", "openai-audio-transcriptions", "openai-audio-speech", "frames-chat-vision"]; const gateway: MultimodalProvider = { id: "novita", name: "Novita", keyEnv: "BOOTAGENT_NOVITA_API_KEY", + keyFile: "/Users/me/.bootagent/skill-env/novita.env", unsupported: [], listed: true, models: [ @@ -59,6 +60,8 @@ function detection(partial: Partial = {}): MultimodalDetect { capability: "image-generation", adapter: "openai-images", probe: "confirm" }, { capability: "speech-to-text", adapter: "openai-audio-transcriptions", probe: "minimal" }, ], + fallback: [], + ffmpeg: true, ...partial, }; } @@ -205,7 +208,7 @@ describe("MultimodalSection", () => { fireEvent.click(within(dialog).getByLabelText("Codex")); fireEvent.click(within(dialog).getByLabelText("Codex")); fireEvent.click(within(dialog).getByRole("button", { name: "安装" })); - await waitFor(() => expect(installMultimodal).toHaveBeenCalledWith({ provider: "novita", model: "qwen/qwen3-vl-8b", capability: "vision", agents: ["claude-code", "codex"] })); + await waitFor(() => expect(installMultimodal).toHaveBeenCalledWith({ provider: "novita", model: "qwen/qwen3-vl-8b", capability: "vision", agents: ["claude-code", "codex"], guide: false })); expect(await within(dialog).findByText("已安装")).toBeTruthy(); expect(within(dialog).getByText(/did not write is installed here/)).toBeTruthy(); expect(within(dialog).getByText(/需要重启/)).toBeTruthy(); @@ -266,6 +269,30 @@ describe("MultimodalSection", () => { expect(within(row).getByRole("button", { name: /^安装$/ })).toBeTruthy(); }); + it("writes a guide for other Agents and names the fallback file", async () => { + detectMultimodal.mockResolvedValue(detection({ targets: ["claude-code", "codex", "opencode"], fallback: ["opencode"] })); + installMultimodal.mockResolvedValue({ skillId: "bootagent-vision", guidePath: "/Users/me/.bootagent/multimodal/vision", results: [{ agent: "guide", installed: true, envWritten: true }] }); + renderSection(); + fireEvent.click(await screen.findByRole("button", { name: /^安装$/ })); + const dialog = screen.getByRole("dialog"); + // OpenCode is preselected and has no configurable environment. + expect(within(dialog).getByText(/私有文件 \/Users\/me\/\.bootagent\/skill-env\/novita\.env/)).toBeTruthy(); + for (const name of ["Claude Code", "Codex", "opencode"]) fireEvent.click(within(dialog).getByLabelText(name)); + fireEvent.click(within(dialog).getByLabelText("其他 Agent（生成说明文件）")); + fireEvent.click(within(dialog).getByRole("button", { name: "安装" })); + await waitFor(() => expect(installMultimodal).toHaveBeenCalledWith({ provider: "novita", model: "qwen/qwen3-vl-8b", capability: "vision", agents: [], guide: true })); + expect(await within(dialog).findByText("其他 Agent（说明文件）")).toBeTruthy(); + expect(within(dialog).getByText("其他 Agent：说明文件在 /Users/me/.bootagent/multimodal/vision")).toBeTruthy(); + fireEvent.click(within(dialog).getAllByRole("button", { name: "复制提示词" })[0]); + expect(copyToClipboard).toHaveBeenCalledWith(expect.stringContaining("先读 /Users/me/.bootagent/multimodal/vision/SKILL.md")); + }); + + it("warns when ffmpeg is missing for video by frames", async () => { + detectMultimodal.mockResolvedValue(detection({ ffmpeg: false, providers: [{ ...gateway, models: [{ id: "qwen/qwen3-vl-8b", capabilities: [{ id: "video-understanding", source: "declared", adapter: "frames-chat-vision", supported: true, installable: true, probe: "minimal" }] }] }] })); + renderSection(); + expect(await screen.findByText(/没有在 PATH 上找到 ffmpeg/)).toBeTruthy(); + }); + it("points to the Providers page when no key is saved", async () => { detectMultimodal.mockResolvedValue(detection({ providers: [] })); renderSection(); @@ -273,7 +300,7 @@ describe("MultimodalSection", () => { }); it("reports a failed listing without hiding the Provider", async () => { - detectMultimodal.mockResolvedValue(detection({ providers: [{ id: "deepseek", name: "DeepSeek", keyEnv: "BOOTAGENT_DEEPSEEK_API_KEY", unsupported: [], listed: false, message: "API key was rejected (401).", errorCode: "API_KEY_REJECTED", models: [] }] })); + detectMultimodal.mockResolvedValue(detection({ providers: [{ id: "deepseek", name: "DeepSeek", keyEnv: "BOOTAGENT_DEEPSEEK_API_KEY", keyFile: "/Users/me/.bootagent/skill-env/deepseek.env", unsupported: [], listed: false, message: "API key was rejected (401).", errorCode: "API_KEY_REJECTED", models: [] }] })); renderSection(); expect(await screen.findByText(/无法读取模型列表/)).toBeTruthy(); }); diff --git a/frontend/src/components/MultimodalSection.tsx b/frontend/src/components/MultimodalSection.tsx index 2e809e87..cc2bce83 100644 --- a/frontend/src/components/MultimodalSection.tsx +++ b/frontend/src/components/MultimodalSection.tsx @@ -268,9 +268,14 @@ export function MultimodalSection({ agentLabels = {}, onSkillsChanged }: Multimo {card.variants.map((variant) => (

- {t("{model}（{provider}）已安装到 {agents}", { model: variant.model, provider: providerName(variant.providerId), agents: variant.agents.map(nameOf).join(separator) })} + {variant.agents.length + ? t("{model}（{provider}）已安装到 {agents}", { model: variant.model, provider: providerName(variant.providerId), agents: variant.agents.map(nameOf).join(separator) }) + : t("{model}（{provider}）", { model: variant.model, provider: providerName(variant.providerId) })}

))} + {card.variants.filter((variant) => variant.guidePath).map((variant) => ( + + ))} ))} @@ -333,6 +338,9 @@ export function MultimodalSection({ agentLabels = {}, onSkillsChanged }: Multimo {installedHere ? t("重新安装") : t("安装")} ) : null} + {chosen.capability.adapter === "frames-chat-vision" && detection && !detection.ffmpeg ? ( +

{t("视频理解要先用 ffmpeg 抽帧。BootAgent 没有在 PATH 上找到 ffmpeg；请在 Agent 运行的环境里安装它。")}

+ ) : null} {verdict ? : null} {state?.error ?

{state.error}

: null} @@ -383,6 +391,7 @@ export function MultimodalSection({ agentLabels = {}, onSkillsChanged }: Multimo item.capability === installTarget.capability)} providerName={providerName} nameOf={nameOf} @@ -430,6 +439,42 @@ function Verdict({ verdict }: { verdict: ReturnType }) { ); } +/** + * For Agents without Skills: the guide's path and a prompt that points the + * Agent at it, in both languages, with copy buttons. + */ +function GuidePrompt({ path, capability }: { path: string; capability: MultimodalCapabilityId }) { + const { t } = useI18n(); + const [copied, setCopied] = useState(""); + const example = EXAMPLE_PROMPTS[capability]?.[0]; + const prompts = [ + { lang: "zh-CN", text: `先读 ${path}/SKILL.md，按里面的说明完成：${example?.zh ?? ""}` }, + { lang: "en", text: `First read ${path}/SKILL.md and follow it to do this: ${example?.en ?? ""}` }, + ]; + const copy = (text: string) => { + void copyToClipboard(text).then((ok) => { + if (!ok) return; + setCopied(text); + window.setTimeout(() => setCopied((current) => (current === text ? "" : current)), 1500); + }); + }; + return ( +
+ {t("其他 Agent：说明文件在 {path}", { path })} +
    + {prompts.map((item) => ( +
  • + {item.text} + +
    • + ))} +
+
+ ); +} + function ExamplePrompts({ capability }: { capability: MultimodalCapabilityId }) { const { t } = useI18n(); const [copied, setCopied] = useState(""); @@ -465,6 +510,8 @@ function ExamplePrompts({ capability }: { capability: MultimodalCapabilityId }) interface InstallDialogProps { target: InstallTarget; targets: string[]; + /** Targets whose key goes to the private fallback file. */ + fallback: string[]; /** What is installed for this capability now, from any Provider. */ current: InstalledMultimodal[]; providerName: (id: string) => string; @@ -473,9 +520,10 @@ interface InstallDialogProps { onCancel: () => void; } -function InstallDialog({ target, targets, current, providerName, nameOf, onDone, onCancel }: InstallDialogProps) { +function InstallDialog({ target, targets, fallback, current, providerName, nameOf, onDone, onCancel }: InstallDialogProps) { const { t, locale } = useI18n(); const [agents, setAgents] = useState(targets); + const [guide, setGuide] = useState(false); const [busy, setBusy] = useState(false); const [result, setResult] = useState(null); const [error, setError] = useState(""); @@ -486,7 +534,7 @@ function InstallDialog({ target, targets, current, providerName, nameOf, onDone, const install = () => { setBusy(true); setError(""); - void api.installMultimodal({ provider: target.provider.id, model: target.model, capability: target.capability, agents }) + void api.installMultimodal({ provider: target.provider.id, model: target.model, capability: target.capability, agents, guide }) .then(setResult) .catch((reason: unknown) => setError(describeError(reason, t("安装失败")).message)) .finally(() => setBusy(false)); @@ -501,7 +549,7 @@ function InstallDialog({ target, targets, current, providerName, nameOf, onDone,
    {result.results.map((item) => (
  • - {nameOf(item.agent)} + {item.agent === "guide" ? t("其他 Agent（说明文件）") : nameOf(item.agent)} {item.installed && item.envWritten ? t("已安装") : item.error || t("安装失败")}
    • ))} @@ -509,6 +557,7 @@ function InstallDialog({ target, targets, current, providerName, nameOf, onDone, {result.results.some((item) => item.installed) ? ( <>

    {t("正在运行的 Agent 需要重启，才能读到新写入的环境变量。")}

    + {result.guidePath ? : null} ) : null} @@ -525,6 +574,10 @@ function InstallDialog({ target, targets, current, providerName, nameOf, onDone, {nameOf(agent)} ))} + {replacing.length ? (

    {t("将替换这些 Agent 当前使用的 {models}。", { models: replacing.map((item) => `${item.model}（${providerName(item.providerId)}）`).join(separator) })}

    @@ -533,13 +586,16 @@ function InstallDialog({ target, targets, current, providerName, nameOf, onDone,
    {t("API Key 对 Agent 可见")} {t("安装会把 {provider} 的 API Key 写进所选 Agent 自己的配置文件，变量名是 {name}。Agent 运行的任何命令都能读到它，Agent 使用的模型也可能看到。", { provider: providerName(target.provider.id), name: target.provider.keyEnv })} + {guide || agents.some((agent) => fallback.includes(agent)) ? ( + {t("{agents}没有可配置的环境变量，Key 会写进 BootAgent 的私有文件 {path}，由 Skill 的脚本读取。", { agents: [...agents.filter((agent) => fallback.includes(agent)).map(nameOf), ...(guide ? [t("其他 Agent（生成说明文件）")] : [])].join(separator), path: target.provider.keyFile })} + ) : null} {t("Skill 的脚本需要 Node.js 18 或更高版本。之后每次调用都由 Agent 发起，按模型服务的价格计费；生成类 Skill 会要求 Agent 先征得你的同意。")}
    {error ?

    {error}

    : null}
    - +
    )} diff --git a/frontend/src/i18n.tsx b/frontend/src/i18n.tsx index 2aade999..98609325 100644 --- a/frontend/src/i18n.tsx +++ b/frontend/src/i18n.tsx @@ -977,6 +977,12 @@ const english = { "确认实测": "Confirm the probe", "这次实测会让「{model}」真实生成一张图片，按模型服务的价格计费。确定要发送吗？": "This probe makes \"{model}\" generate one real image, billed at the provider's price. Send it?", "重新安装": "Reinstall", + "{model}（{provider}）": "{model} ({provider})", + "视频理解要先用 ffmpeg 抽帧。BootAgent 没有在 PATH 上找到 ffmpeg；请在 Agent 运行的环境里安装它。": "Video understanding samples frames with ffmpeg. BootAgent did not find ffmpeg on PATH; install it where the Agent runs.", + "其他 Agent：说明文件在 {path}": "Other Agents: the guide is at {path}", + "其他 Agent（说明文件）": "Other Agents (guide)", + "其他 Agent（生成说明文件）": "Other Agents (write a guide)", + "{agents}没有可配置的环境变量，Key 会写进 BootAgent 的私有文件 {path}，由 Skill 的脚本读取。": "{agents} have no configurable environment, so the key goes into BootAgent's private file {path}, which the Skill's script reads.", "{capabilities}：该模型服务目前没有可核实的公开接口，暂不支持。": "{capabilities}: this provider documents no public API for it yet, so it is not supported.", } as const; diff --git a/frontend/src/types/api.ts b/frontend/src/types/api.ts index 99146e97..dea4cce3 100644 --- a/frontend/src/types/api.ts +++ b/frontend/src/types/api.ts @@ -134,6 +134,10 @@ export type MultimodalDetection = { targets: string[]; generatable: string[]; manual: MultimodalManualRoute[]; + /** Targets whose key goes to the private fallback file. */ + fallback: string[]; + /** ffmpeg is on BootAgent's PATH; a hint for video understanding. */ + ffmpeg: boolean; }; export type MultimodalInstallRequest = Omit & { capability: MultimodalCapabilityId }; export type MultimodalInstallResult = Omit & { results: AppModels.MultimodalAgentResult[] }; diff --git a/internal/app/multimodal.go b/internal/app/multimodal.go index 23734823..7a9b90b3 100644 --- a/internal/app/multimodal.go +++ b/internal/app/multimodal.go @@ -28,6 +28,12 @@ type MultimodalDetection struct { // Manual lists the capabilities a user can reach with a model ID typed by // hand, the route it is probed on, and how that probe is sent. Manual []MultimodalManualRoute `json:"manual"` + // Fallback lists the Targets with no configurable environment: their key + // goes to BootAgent's private fallback file, which the install says. + Fallback []string `json:"fallback"` + // FFmpeg says ffmpeg is on BootAgent's PATH. Video understanding needs it + // where the Agent runs, which may differ, so this is a hint only. + FFmpeg bool `json:"ffmpeg"` } type MultimodalManualRoute struct { @@ -44,6 +50,9 @@ type MultimodalProvider struct { KeyEnv string `json:"keyEnv"` // Unsupported lists capabilities this Provider documents no API for. Unsupported []string `json:"unsupported"` + // KeyFile is the private fallback file an install for an Agent without a + // configurable environment writes the key into. + KeyFile string `json:"keyFile"` // Listed is false when /models could not be read. Message and ErrorCode // then say why; catalog generation models are still listed, since they do // not come from /models. @@ -121,13 +130,19 @@ func (u *UseCases) DetectMultimodal(ctx context.Context) (MultimodalDetection, e if err := contextError(ctx, "Multimodal detection was cancelled"); err != nil { return MultimodalDetection{}, err } - installTargets, _, err := u.multimodalInstallTargets() + installTargets, eligible, err := u.multimodalInstallTargets() if err != nil { return MultimodalDetection{}, err } if installTargets == nil { installTargets = []string{} } + fallback := []string{} + for _, agentID := range installTargets { + if !nativeSkillEnvAdapters[eligible[agentID].ConfigAdapter] { + fallback = append(fallback, agentID) + } + } manual := []MultimodalManualRoute{} for _, capability := range modality.Capabilities { if adapter := modality.ManualAdapter(capability); adapter != "" { @@ -136,12 +151,22 @@ func (u *UseCases) DetectMultimodal(ctx context.Context) (MultimodalDetection, e } return MultimodalDetection{ Providers: results, Installed: u.installedMultimodal(), Targets: installTargets, - Generatable: skillgen.SupportedAdapters(), Manual: manual, + Generatable: skillgen.SupportedAdapters(), Manual: manual, Fallback: fallback, FFmpeg: u.commandOnPath("ffmpeg"), }, nil } +// commandOnPath reports whether a command resolves the way status does. +func (u *UseCases) commandOnPath(command string) bool { + if u.status.Lookup != nil { + _, ok := u.status.Lookup(command) + return ok + } + _, ok := u.lookPath(command) + return ok +} + func (u *UseCases) detectProvider(ctx context.Context, rules modality.Rules, target provider.Entry) MultimodalProvider { - result := MultimodalProvider{ID: target.ID, Name: target.Name, KeyEnv: configWriter.SkillEnvName(target.ID), Unsupported: rules.Unsupported(target.ID), Models: []modality.Model{}} + result := MultimodalProvider{ID: target.ID, Name: target.Name, KeyEnv: configWriter.SkillEnvName(target.ID), KeyFile: configWriter.SkillEnvFilePath(u.status.Home, target.ID), Unsupported: rules.Unsupported(target.ID), Models: []modality.Model{}} if result.Unsupported == nil { result.Unsupported = []string{} } @@ -197,6 +222,11 @@ func (u *UseCases) ProbeMultimodal(ctx context.Context, providerID, model, capab if kind == modality.ProbeConfirm && !confirmed { return nil, oneerrors.New(oneerrors.InvalidRequest, "This check produces billed output; confirm it first") } + if adapter == modality.AdapterFramesVision { + // Video by frame sampling is a chat vision request carrying frames: + // probe that, and record it as what it is, image understanding. + capability, adapter = modality.Vision, modality.AdapterChatVision + } check := &MultimodalProbe{ProviderID: providerID, Model: model, Capability: capability, Adapter: adapter} target, err := u.providers.Get(providerID) if err != nil { diff --git a/internal/app/multimodal_install.go b/internal/app/multimodal_install.go index 20eb4315..a6f4cb55 100644 --- a/internal/app/multimodal_install.go +++ b/internal/app/multimodal_install.go @@ -27,8 +27,17 @@ type MultimodalInstallRequest struct { Model string `json:"model"` Capability string `json:"capability"` Agents []string `json:"agents"` + // Guide also writes the Skill to ~/.bootagent/multimodal// + // for Agents without a Skills directory, which are pointed at it by a + // prompt. Their key goes to the fallback file. + Guide bool `json:"guide"` } +// GuideTarget names the guide in per-target results. It is not an Agent ID +// and never enters the Skill registry: the guide directory is found again by +// its hash. +const GuideTarget = "guide" + type MultimodalAgentResult struct { Agent string `json:"agent"` // Installed: the Skill tree is in the Agent's Skills directory. @@ -41,6 +50,8 @@ type MultimodalAgentResult struct { type MultimodalInstallResult struct { SkillID string `json:"skillId"` Results []MultimodalAgentResult `json:"results"` + // GuidePath is the guide directory when one was written. + GuidePath string `json:"guidePath,omitempty"` } // InstalledMultimodal is one generated Skill variant that is published to at @@ -52,23 +63,34 @@ type InstalledMultimodal struct { Model string `json:"model"` Adapter string `json:"adapter"` Agents []string `json:"agents"` + // GuidePath is set when this variant is also the guide for Agents + // without Skills. + GuidePath string `json:"guidePath,omitempty"` } // nativeSkillEnvAdapters are the config adapters with a writer for Skill // variables (ADR-011). Other Agents get the private fallback file. var nativeSkillEnvAdapters = map[string]bool{"claude-code": true, "codex": true} +// pendingSkillTargets are Agents with a Skills directory that are not offered +// yet. Hermes is installed by its official script, so Node may be absent +// where it runs; whether its Skill should run BootAgent's managed Node by +// absolute path is an open decision (design doc, Open questions). +var pendingSkillTargets = map[string]bool{"hermes": true} + // multimodalInstallTargets are the Agents a multimodal Skill can be installed // into from the multimodal section: those with a Skills directory BootAgent -// can see, and a native place for the key variable. Phase 4 widens this. +// can see. Claude Code and Codex get the key in their own configuration; the +// others read the private fallback file, since neither OpenCode nor Hermes +// documents a way to set the environment of its shell tool. func (u *UseCases) multimodalInstallTargets() ([]string, map[string]catalog.Agent, error) { eligible, err := u.eligibleSkillAgents() if err != nil { return nil, nil, err } var targets []string - for id, agent := range eligible { - if nativeSkillEnvAdapters[agent.ConfigAdapter] { + for id := range eligible { + if !pendingSkillTargets[id] { targets = append(targets, id) } } @@ -76,6 +98,12 @@ func (u *UseCases) multimodalInstallTargets() ([]string, map[string]catalog.Agen return targets, eligible, nil } +// guideDir is where the Skill for a capability is written for Agents without +// a Skills directory. +func (u *UseCases) guideDir(capability string) string { + return filepath.Join(u.status.Home, ".bootagent", "multimodal", capability) +} + // InstallMultimodal generates the Skill, stores it as a variant with source // "generated", publishes it to each Agent and writes the key variable there. // It sends no billed request: the evidence it installs on is the free model @@ -95,7 +123,7 @@ func (u *UseCases) InstallMultimodal(ctx context.Context, request MultimodalInst return result, oneerrors.New(oneerrors.InvalidRequest, "A model and a known capability are required") } agents := slices.Compact(slices.Sorted(slices.Values(request.Agents))) - if len(agents) == 0 { + if len(agents) == 0 && !request.Guide { return result, oneerrors.New(oneerrors.InvalidRequest, "Choose at least one Agent") } target, err := u.providers.Get(request.Provider) @@ -141,13 +169,19 @@ func (u *UseCases) InstallMultimodal(ctx context.Context, request MultimodalInst result.Results = append(result.Results, MultimodalAgentResult{Agent: agentID, Error: "This Agent cannot take a multimodal Skill yet"}) } } - if len(install) == 0 { + if len(install) == 0 && !request.Guide { return result, nil } - published, err := u.publishGeneratedLocked(ctx, spec, install) + published, err := u.publishGeneratedLocked(ctx, spec, install, request.Guide) if err != nil { return result, err } + if request.Guide { + install = append(install, GuideTarget) + if published[GuideTarget] == "" { + result.GuidePath = u.guideDir(capability) + } + } envFailures := u.reconcileSkillEnvLocked(ctx) for _, agentID := range install { item := MultimodalAgentResult{Agent: agentID} @@ -190,9 +224,11 @@ func (u *UseCases) resolveForInstall(ctx context.Context, rules modality.Rules, } // publishGeneratedLocked renders the Skill, stores it as a variant and -// publishes it to each Agent. It returns a per-Agent error message ("" on -// success); an error return means nothing was published. Callers hold writeMu. -func (u *UseCases) publishGeneratedLocked(ctx context.Context, spec skillgen.Spec, agentIDs []string) (map[string]string, error) { +// publishes it to each Agent, and to the guide directory when guide is set. +// It returns a per-target error message ("" on success), the guide under +// GuideTarget; an error return means nothing was published. Callers hold +// writeMu. +func (u *UseCases) publishGeneratedLocked(ctx context.Context, spec skillgen.Spec, agentIDs []string, guide bool) (map[string]string, error) { id := skillgen.SkillID(spec.Capability) store := u.skillStore() registry, err := store.Load() @@ -277,9 +313,62 @@ func (u *UseCases) publishGeneratedLocked(ctx context.Context, spec skillgen.Spe } results[agentID] = "" } + if guide { + results[GuideTarget] = u.publishGuideLocked(ctx, store, id, stats.Hash, fact) + } return results, nil } +// publishGuideLocked writes a stored variant to the guide directory, with +// the same rule as an Agent target: only a variant of this Skill BootAgent +// stored may be replaced. It returns an error message, "" on success. +func (u *UseCases) publishGuideLocked(ctx context.Context, store skill.Store, id, hash string, fact skill.Fact) string { + destination := u.guideDir(skillgen.CapabilityFromSkillID(id)) + parent := filepath.Dir(destination) + if hasSymlinkComponent(u.status.Home, parent) { + return "Guide path is not private" + } + if err := u.filesystem.EnsurePrivateDir(ctx, parent); err != nil { + return "Cannot prepare the guide directory" + } + if err := verifyGeneratedTarget(ctx, destination, fact); err != nil { + return err.Error() + } + if err := skill.PublishTree(ctx, store.VariantPath(id, hash), destination); err != nil { + return "Cannot write the guide" + } + return "" +} + +// guideHash is the hash of a capability's guide directory, empty when there +// is none or it is not a plain directory. +func (u *UseCases) guideHash(ctx context.Context, capability string) string { + destination := u.guideDir(capability) + info, err := os.Lstat(destination) + if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return "" + } + stats, err := skill.HashTree(ctx, destination) + if err != nil { + return "" + } + return stats.Hash +} + +// removeGeneratedGuide deletes a Skill's guide directory when it holds one of +// the Skill's stored variants; anything else there is left alone. +func (u *UseCases) removeGeneratedGuide(ctx context.Context, id string, fact skill.Fact) error { + capability := skillgen.CapabilityFromSkillID(id) + if capability == "" { + return nil + } + hash := u.guideHash(ctx, capability) + if hash == "" || variantIndex(fact.Variants, hash) < 0 { + return nil + } + return removeManagedSkill(ctx, u.guideDir(capability), hash) +} + // verifyGeneratedTarget allows replacing what is at the destination only when // it is a variant of this Skill BootAgent stored. Anything else -- a Skill the // user wrote under the same name, or an edited copy -- is left alone. @@ -329,6 +418,8 @@ type generatedVariant struct { hash string config skillgen.Config targets []string + // guide: this variant is what the capability's guide directory holds. + guide bool } // generatedVariants lists the variants BootAgent rendered here. Only these @@ -338,9 +429,11 @@ func (u *UseCases) generatedVariants(registry skill.Registry) []generatedVariant store := u.skillStore() var result []generatedVariant for _, id := range slices.Sorted(maps.Keys(registry.Skills)) { - if skillgen.CapabilityFromSkillID(id) == "" { + capability := skillgen.CapabilityFromSkillID(id) + if capability == "" { continue } + guide := u.guideHash(context.Background(), capability) for _, variant := range registry.Skills[id].Variants { if !variant.Stored || !slices.Contains(variant.ImportSources, skill.ImportSourceGenerated) { continue @@ -349,7 +442,7 @@ func (u *UseCases) generatedVariants(registry skill.Registry) []generatedVariant if err != nil { continue } - result = append(result, generatedVariant{id: id, hash: variant.Hash, config: config, targets: variant.ManagedTargets}) + result = append(result, generatedVariant{id: id, hash: variant.Hash, config: config, targets: variant.ManagedTargets, guide: guide == variant.Hash}) } } return result @@ -390,6 +483,13 @@ func (u *UseCases) reconcileSkillEnvLocked(ctx context.Context) map[string]strin desired := map[string]map[string]string{} fallback := map[string][]string{} for _, variant := range u.generatedVariants(registry) { + if variant.guide { + if keyFor(variant.config.ProviderID) == "" { + failures[GuideTarget] = "Provider " + variant.config.ProviderID + " has no saved API key" + } else { + fallback[variant.config.ProviderID] = append(fallback[variant.config.ProviderID], GuideTarget) + } + } for _, agentID := range variant.targets { agent, ok := manifest.Agents[agentID] if !ok { @@ -462,13 +562,20 @@ func (u *UseCases) installedMultimodal() []InstalledMultimodal { } result := []InstalledMultimodal{} for _, variant := range u.generatedVariants(registry) { - if len(variant.targets) == 0 { + if len(variant.targets) == 0 && !variant.guide { continue } - result = append(result, InstalledMultimodal{ + installed := InstalledMultimodal{ SkillID: variant.id, Capability: variant.config.Capability, ProviderID: variant.config.ProviderID, Model: variant.config.Model, Adapter: variant.config.Adapter, Agents: slices.Clone(variant.targets), - }) + } + if installed.Agents == nil { + installed.Agents = []string{} + } + if variant.guide { + installed.GuidePath = u.guideDir(variant.config.Capability) + } + result = append(result, installed) } return result } @@ -482,7 +589,7 @@ func (u *UseCases) generatedSkillsUsing(providerID string) []generatedVariant { } var result []generatedVariant for _, variant := range u.generatedVariants(registry) { - if variant.config.ProviderID == providerID && len(variant.targets) > 0 { + if variant.config.ProviderID == providerID && (len(variant.targets) > 0 || variant.guide) { result = append(result, variant) } } @@ -504,7 +611,7 @@ func (u *UseCases) refreshGeneratedSkillsLocked(ctx context.Context, target prov ProviderID: target.ID, ProviderName: target.Name, Model: variant.config.Model, BaseURL: target.BaseURL, NativeBaseURL: variant.config.NativeBaseURL, Options: variant.config.Options, } - published, err := u.publishGeneratedLocked(ctx, spec, variant.targets) + published, err := u.publishGeneratedLocked(ctx, spec, variant.targets, variant.guide) if err != nil { for _, agentID := range variant.targets { failures[agentID] = oneerrors.As(err).Message diff --git a/internal/app/multimodal_install_test.go b/internal/app/multimodal_install_test.go index 89dd06ff..6c062b3d 100644 --- a/internal/app/multimodal_install_test.go +++ b/internal/app/multimodal_install_test.go @@ -157,8 +157,10 @@ func TestInstallMultimodalPublishesAndWritesTheKeyOnlyWhereItBelongs(t *testing. if len(detection.Installed) != 1 || detection.Installed[0].Model != "qwen/qwen3-vl-8b" || len(detection.Installed[0].Agents) != 2 { t.Fatalf("installed = %#v", detection.Installed) } - if strings.Join(detection.Targets, ",") != "claude-code,codex" { - t.Fatalf("targets = %v", detection.Targets) + // Hermes has a Skills directory but is not offered yet; see + // pendingSkillTargets. + if strings.Join(detection.Targets, ",") != "claude-code,codex,opencode" || strings.Join(detection.Fallback, ",") != "opencode" { + t.Fatalf("targets = %v, fallback = %v", detection.Targets, detection.Fallback) } } @@ -186,10 +188,10 @@ func TestInstallMultimodalRefusesWeakEvidence(t *testing.T) { t.Fatalf("%v after a probe: %#v", target, result) } } - // An Agent with no native place for the key is not offered here. - result := installSkill(t, core, "qwen/qwen3-vl-8b", modality.Vision, "opencode") + // Hermes is not offered until its Node question is settled. + result := installSkill(t, core, "qwen/qwen3-vl-8b", modality.Vision, "hermes") if len(result.Results) != 1 || result.Results[0].Installed || result.Results[0].Error == "" { - t.Fatalf("opencode result = %#v", result) + t.Fatalf("hermes result = %#v", result) } } @@ -476,3 +478,128 @@ func TestDetectionMarksUndocumentedCapabilities(t *testing.T) { } } } + +// OpenCode documents no way to set its shell tool's environment, so its +// Skill reads the private fallback file. +func TestOpenCodeInstallsWithTheFallbackFile(t *testing.T) { + gateway := &multimodalGateway{listing: gatewayListing} + core, home := multimodalInstallCore(t, gateway) + result := installSkill(t, core, "qwen/qwen3-vl-8b", modality.Vision, "opencode") + if !result.Results[0].Installed || !result.Results[0].EnvWritten { + t.Fatalf("result = %#v", result) + } + if _, err := os.Stat(filepath.Join(home, ".config", "opencode", "skills", "bootagent-vision", "SKILL.md")); err != nil { + t.Fatal(err) + } + fallback := filepath.Join(home, ".bootagent", "skill-env", "gw.env") + if data, err := os.ReadFile(fallback); err != nil || !strings.Contains(string(data), "BOOTAGENT_GW_API_KEY=") { + t.Fatalf("fallback = %q, %v", data, err) + } + // Nothing was written into OpenCode's own configuration. + if data, err := os.ReadFile(filepath.Join(home, ".config", "opencode", "opencode.json")); err == nil && strings.Contains(string(data), multimodalKey) { + t.Fatal("the key went into opencode.json") + } + if result := core.UninstallSkill(context.Background(), "bootagent-vision"); result.Error != "" { + t.Fatalf("uninstall = %#v", result) + } + if _, err := os.Stat(fallback); !os.IsNotExist(err) { + t.Fatal("the fallback file outlived the Skill") + } +} + +// Agents without Skills get the same tree as a guide and a prompt pointing at +// it. The guide is not in the registry's targets; it is found by its hash. +func TestGuideForAgentsWithoutSkills(t *testing.T) { + gateway := &multimodalGateway{listing: gatewayListing} + core, home := multimodalInstallCore(t, gateway) + result, err := core.InstallMultimodal(context.Background(), MultimodalInstallRequest{Provider: "gw", Model: "qwen/qwen3-vl-8b", Capability: modality.Vision, Guide: true}) + if err != nil { + t.Fatal(err) + } + guide := filepath.Join(home, ".bootagent", "multimodal", "vision") + if result.GuidePath != guide || len(result.Results) != 1 || result.Results[0].Agent != GuideTarget || !result.Results[0].Installed || !result.Results[0].EnvWritten { + t.Fatalf("result = %#v", result) + } + if cfg, err := skillgen.ReadConfig(guide); err != nil || cfg.Model != "qwen/qwen3-vl-8b" { + t.Fatalf("guide config = %#v, %v", cfg, err) + } + fallback := filepath.Join(home, ".bootagent", "skill-env", "gw.env") + if _, err := os.Stat(fallback); err != nil { + t.Fatal("the guide got no key file") + } + registry, _ := core.skillStore().Load() + if variant := registry.Skills["bootagent-vision"].Variants[0]; len(variant.ManagedTargets) != 0 { + t.Fatalf("the guide entered the registry targets: %#v", variant) + } + detection, err := core.DetectMultimodal(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(detection.Installed) != 1 || detection.Installed[0].GuidePath != guide || len(detection.Installed[0].Agents) != 0 { + t.Fatalf("installed = %#v", detection.Installed) + } + // A rotated key reaches the guide's file too. + if _, err := core.SaveProvider(context.Background(), provider.Entry{ID: "gw", Name: "Gateway", BaseURL: "https://gw.example/v1", APIKey: "sk-rotated-guide"}, false, false); err != nil { + t.Fatal(err) + } + if data, _ := os.ReadFile(fallback); !strings.Contains(string(data), "sk-rotated-guide") { + t.Fatalf("fallback after rotation = %q", data) + } + if err := core.DeleteProvider(context.Background(), "gw"); err == nil { + t.Fatal("deleting a Provider a guide uses was allowed") + } + if result := core.UninstallSkill(context.Background(), "bootagent-vision"); result.Error != "" { + t.Fatalf("uninstall = %#v", result) + } + for _, path := range []string{guide, fallback} { + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatalf("%s outlived the uninstall", path) + } + } +} + +func TestGuideLeavesAFolderBootAgentDidNotWrite(t *testing.T) { + gateway := &multimodalGateway{listing: gatewayListing} + core, home := multimodalInstallCore(t, gateway) + guide := filepath.Join(home, ".bootagent", "multimodal", "vision") + writeTestSkillAt(t, guide, "the user's notes") + result, err := core.InstallMultimodal(context.Background(), MultimodalInstallRequest{Provider: "gw", Model: "qwen/qwen3-vl-8b", Capability: modality.Vision, Guide: true}) + if err != nil { + t.Fatal(err) + } + if result.GuidePath != "" || result.Results[0].Installed || result.Results[0].Error == "" { + t.Fatalf("result = %#v", result) + } + if data, _ := os.ReadFile(filepath.Join(guide, "SKILL.md")); !strings.Contains(string(data), "the user's notes") { + t.Fatal("the user's folder was overwritten") + } + if _, err := os.Stat(filepath.Join(home, ".bootagent", "skill-env", "gw.env")); !os.IsNotExist(err) { + t.Fatal("a key file was written for a guide that was not installed") + } +} + +// Video by frame sampling is a chat vision request; its probe is recorded as +// image understanding and both rows follow it. +func TestVideoByFramesInstallsOnTheImageVerdict(t *testing.T) { + gateway := &multimodalGateway{listing: gatewayListing} + core, home := multimodalInstallCore(t, gateway) + // kimi-k3 is an ID-rule guess: neither row installs yet. + if _, err := core.InstallMultimodal(context.Background(), MultimodalInstallRequest{Provider: "gw", Model: "moonshotai/kimi-k3", Capability: modality.VideoUnderstanding, Agents: []string{"codex"}}); err == nil { + t.Fatal("video on a guess was installed") + } + check, err := core.ProbeMultimodal(context.Background(), "gw", "moonshotai/kimi-k3", modality.VideoUnderstanding, modality.AdapterFramesVision, false) + if err != nil || check.Status != ModelCheckAvailable || check.Capability != modality.Vision || check.Adapter != modality.AdapterChatVision { + t.Fatalf("probe = %#v, %v", check, err) + } + if posts := gateway.billed(); len(posts) != 1 || !strings.Contains(posts[0], "/chat/completions") { + t.Fatalf("probe sent %v", posts) + } + result, err := core.InstallMultimodal(context.Background(), MultimodalInstallRequest{Provider: "gw", Model: "moonshotai/kimi-k3", Capability: modality.VideoUnderstanding, Agents: []string{"codex"}}) + if err != nil || !result.Results[0].Installed { + t.Fatalf("install = %#v, %v", result, err) + } + cfg, err := skillgen.ReadConfig(filepath.Join(home, ".codex", "skills", "bootagent-video-understanding")) + if err != nil || cfg.Adapter != modality.AdapterFramesVision || cfg.Model != "moonshotai/kimi-k3" { + t.Fatalf("config = %#v, %v", cfg, err) + } +} diff --git a/internal/app/skill.go b/internal/app/skill.go index 3fe53dab..a2061b95 100644 --- a/internal/app/skill.go +++ b/internal/app/skill.go @@ -460,6 +460,10 @@ func (u *UseCases) ApplySkills(ctx context.Context, req SkillApplyRequest) Skill result.Results = append(result.Results, item) } if deleteSkill && !deleteFailed && !hasManagedTargets(fact) { + if err := u.removeGeneratedGuide(ctx, change.ID, fact); err != nil { + result.Results = append(result.Results, SkillAgentApplyResult{Error: "Cannot remove the Skill's guide: " + err.Error()}) + continue + } if err := store.RemoveSkill(ctx, change.ID); err != nil { result.Results = append(result.Results, SkillAgentApplyResult{Error: "Cannot remove Skill from Registry"}) continue @@ -543,6 +547,12 @@ func (u *UseCases) UninstallSkill(ctx context.Context, id string) SkillUninstall } } } + // The guide for Agents without Skills goes with the Skill, before its + // stored variants, which identify it, are removed. + if err := u.removeGeneratedGuide(ctx, id, fact); err != nil { + result.Error = err.Error() + return result + } if err := store.RemoveSkill(ctx, id); err != nil { result.Error = err.Error() return result diff --git a/internal/modality/capability.go b/internal/modality/capability.go index b797a8fa..9192db9f 100644 --- a/internal/modality/capability.go +++ b/internal/modality/capability.go @@ -25,8 +25,15 @@ const ( // AdapterChatVision sends images as Chat Completions image_url parts. AdapterChatVision = "openai-chat-vision" // AdapterChatVideo sends video as Chat Completions content parts, for models - // that take video natively. + // that take video natively. Vendors disagree on the part: Moonshot takes + // only a file uploaded to its own storage (ms://), not a data URL, + // so this route installs only after a probe, which none has yet. AdapterChatVideo = "openai-chat-video" + // AdapterFramesVision samples frames from a video with ffmpeg and sends + // them to an image-understanding model on the chat vision route. It + // inherits that model's image-understanding verdict and needs ffmpeg on + // PATH where the Agent runs. + AdapterFramesVision = "frames-chat-vision" // AdapterChatImageOutput asks a chat model for an image in its reply // (gemini-*-image, gpt-image-* on JieKou.AI). How gateways return the image // is not yet verified, so models on this route install only after a probe. @@ -78,7 +85,7 @@ var Options = []string{OptionDuration, OptionSize, OptionAspectRatio, OptionNega // Adapters is the closed vocabulary the catalog may name. var Adapters = []string{ - AdapterChatVision, AdapterChatVideo, AdapterChatImageOutput, AdapterChatAudioInput, + AdapterChatVision, AdapterChatVideo, AdapterFramesVision, AdapterChatImageOutput, AdapterChatAudioInput, AdapterImages, AdapterAudioTranscriptions, AdapterAudioSpeech, AdapterNovitaAsync, AdapterNovitaAudio, AdapterPPIOVideo, } @@ -108,6 +115,7 @@ var chatAdapter = map[string]string{ var probeOnly = map[string]bool{ AdapterChatImageOutput: true, AdapterChatAudioInput: true, + AdapterChatVideo: true, } // Probe kinds tell the UI whether a capability can be checked live and whether @@ -131,6 +139,8 @@ var probeKind = map[string]string{ AdapterAudioSpeech: ProbeMinimal, AdapterImages: ProbeConfirm, AdapterNovitaAudio: ProbeMinimal, + // The frames route sends a chat vision request; its probe is that one. + AdapterFramesVision: ProbeMinimal, } // ProbeKind reports how an adapter can be probed. @@ -143,6 +153,7 @@ func ProbeKind(adapter string) string { var adapterCapability = map[string]string{ AdapterChatVision: Vision, AdapterChatVideo: VideoUnderstanding, + AdapterFramesVision: VideoUnderstanding, AdapterChatImageOutput: ImageGeneration, AdapterChatAudioInput: SpeechToText, AdapterImages: ImageGeneration, diff --git a/internal/modality/resolve.go b/internal/modality/resolve.go index 41c7dc9e..85aa3688 100644 --- a/internal/modality/resolve.go +++ b/internal/modality/resolve.go @@ -241,6 +241,7 @@ func (r Rules) Resolve(providerID string, entries []provider.ModelEntry, verdict Probe: ProbeKind(adapter), }) } + capabilities = withFrameSampling(capabilities) if len(capabilities) > 0 { models = append(models, Model{ID: id, Capabilities: capabilities}) } @@ -276,6 +277,7 @@ func (r Rules) Resolve(providerID string, entries []provider.ModelEntry, verdict capabilities = append(capabilities, probedCapability(capabilityID, verdict)) } } + capabilities = withFrameSampling(capabilities) if len(capabilities) > 0 { models = append(models, Model{ID: id, Capabilities: capabilities}) } @@ -299,6 +301,51 @@ func Find(models []Model, modelID, capability string) (Capability, bool) { return Capability{}, false } +// withFrameSampling gives a model that understands images video +// understanding by frame sampling, with the image-understanding verdict as +// its own: the request is the same chat vision request, carrying frames. It +// replaces a native video claim, whose route is unverified, unless a probe of +// that route already answered. +func withFrameSampling(capabilities []Capability) []Capability { + visionIndex, videoIndex := -1, -1 + for index, capability := range capabilities { + switch capability.ID { + case Vision: + visionIndex = index + case VideoUnderstanding: + videoIndex = index + } + } + if visionIndex < 0 { + return capabilities + } + if videoIndex >= 0 && capabilities[videoIndex].Source == SourceProbe && capabilities[videoIndex].Adapter != AdapterFramesVision { + return capabilities + } + vision := capabilities[visionIndex] + frames := vision + frames.ID = VideoUnderstanding + frames.Adapter = AdapterFramesVision + frames.Probe = ProbeKind(AdapterFramesVision) + frames.Recommended = false + if videoIndex >= 0 { + capabilities[videoIndex] = frames + return capabilities + } + result := make([]Capability, 0, len(capabilities)+1) + for _, capabilityID := range Capabilities { + for _, capability := range capabilities { + if capability.ID == capabilityID { + result = append(result, capability) + } + } + if capabilityID == VideoUnderstanding { + result = append(result, frames) + } + } + return result +} + // catalogCapability is a capability on a verified route the catalog names. It // installs without a probe; the probe, where one exists, is still offered. func catalogCapability(capabilityID string, generation catalog.ModalityGenerationModel) Capability { diff --git a/internal/modality/resolve_test.go b/internal/modality/resolve_test.go index 49bb70c8..1aa44a17 100644 --- a/internal/modality/resolve_test.go +++ b/internal/modality/resolve_test.go @@ -158,15 +158,17 @@ func TestResolveOrder(t *testing.T) { if len(kimi) != 2 || kimi[0] != (Capability{ID: Vision, Source: SourceProbe, Adapter: AdapterChatVision, Supported: true, Installable: true, Probe: ProbeMinimal}) { t.Fatalf("kimi vision = %#v", kimi) } - if kimi[1].ID != VideoUnderstanding || kimi[1].Source != SourceIDRule || kimi[1].Installable { + // Video understanding by frame sampling inherits the probed image verdict. + if kimi[1].ID != VideoUnderstanding || kimi[1].Adapter != AdapterFramesVision || kimi[1].Source != SourceProbe || !kimi[1].Installable { t.Fatalf("kimi video = %#v", kimi[1]) } qwen := byID["qwen/qwen3-vl-8b"].Capabilities - if len(qwen) != 1 || qwen[0].Supported || qwen[0].Installable || qwen[0].Source != SourceProbe { + // The refusal carries to video by frame sampling: it is the same request. + if len(qwen) != 2 || qwen[0].Supported || qwen[0].Installable || qwen[0].Source != SourceProbe || qwen[1].ID != VideoUnderstanding || qwen[1].Installable { t.Fatalf("refused probe = %#v", qwen) } gemini := byID["gemini-3-pro-image"].Capabilities - if len(gemini) != 2 || !gemini[0].Installable || gemini[1].ID != ImageGeneration || gemini[1].Installable || gemini[1].Adapter != AdapterChatImageOutput { + if len(gemini) != 3 || !gemini[0].Installable || gemini[1].ID != ImageGeneration || gemini[1].Installable || gemini[1].Adapter != AdapterChatImageOutput || gemini[2].Adapter != AdapterFramesVision { t.Fatalf("declared image output = %#v", gemini) } } @@ -367,3 +369,42 @@ func TestNewRulesRejectsBrokenGenerationEntries(t *testing.T) { } } } + +func TestFrameSamplingFollowsTheImageVerdict(t *testing.T) { + rules := embeddedRules(t) + entries := []provider.ModelEntry{ + // Declares image and video input: native video is unverified, so + // frames over the declared image route are offered instead. + {ID: "qwen/qwen3.6-flash", InputModalities: []string{"text", "image", "video"}, OutputModalities: []string{"text"}}, + // Image input only: frames still give it video understanding. + {ID: "glm-4.6v", InputModalities: []string{"text", "image"}, OutputModalities: []string{"text"}}, + // An ID-rule guess stays a guess for video too. + {ID: "claude-sonnet-x"}, + // No image input: no frame sampling. + {ID: "text-only", InputModalities: []string{"text"}, OutputModalities: []string{"text"}}, + } + models := rules.Resolve("custom", entries, map[string]map[string]provider.ModalityVerdict{ + "refused-vision": {Vision: {Supported: false, Source: SourceProbe, Adapter: AdapterChatVision}}, + }) + for _, id := range []string{"qwen/qwen3.6-flash", "glm-4.6v"} { + video, ok := Find(models, id, VideoUnderstanding) + if !ok || video.Adapter != AdapterFramesVision || video.Source != SourceDeclared || !video.Installable || video.Probe != ProbeMinimal { + t.Errorf("%s video = %#v, %v", id, video, ok) + } + } + if video, _ := Find(models, "claude-sonnet-x", VideoUnderstanding); video.Installable || video.Source != SourceIDRule { + t.Errorf("guessed video = %#v", video) + } + if _, ok := Find(models, "text-only", VideoUnderstanding); ok { + t.Error("frame sampling offered without image input") + } + // A refused image probe makes the frames route refused too. + if video, ok := Find(models, "refused-vision", VideoUnderstanding); !ok || video.Supported || video.Installable { + t.Errorf("refused video = %#v, %v", video, ok) + } + // A declared native video route alone is not installable. + native := rules.Resolve("custom", []provider.ModelEntry{{ID: "video-only", InputModalities: []string{"text", "video"}, OutputModalities: []string{"text"}}}, nil) + if video, _ := Find(native, "video-only", VideoUnderstanding); video.Installable || video.Adapter != AdapterChatVideo { + t.Errorf("native video = %#v", video) + } +} diff --git a/internal/skillgen/skillgen.go b/internal/skillgen/skillgen.go index 1444ca71..7d8723ec 100644 --- a/internal/skillgen/skillgen.go +++ b/internal/skillgen/skillgen.go @@ -38,11 +38,12 @@ const IDPrefix = "bootagent-" // run.mjs implements exactly these; anything else is refused instead of // producing a Skill whose script cannot run it. var supported = map[string][]string{ - modality.Vision: {modality.AdapterChatVision}, - modality.ImageGeneration: {modality.AdapterImages}, - modality.SpeechToText: {modality.AdapterAudioTranscriptions}, - modality.TextToSpeech: {modality.AdapterAudioSpeech, modality.AdapterNovitaAudio}, - modality.VideoGeneration: {modality.AdapterNovitaAsync}, + modality.Vision: {modality.AdapterChatVision}, + modality.ImageGeneration: {modality.AdapterImages}, + modality.SpeechToText: {modality.AdapterAudioTranscriptions}, + modality.TextToSpeech: {modality.AdapterAudioSpeech, modality.AdapterNovitaAudio}, + modality.VideoGeneration: {modality.AdapterNovitaAsync}, + modality.VideoUnderstanding: {modality.AdapterFramesVision}, } // Supports reports whether a capability over an adapter can be generated. @@ -205,27 +206,30 @@ type skillText struct { } var titles = map[string]string{ - modality.Vision: "Image understanding", - modality.ImageGeneration: "Image generation", - modality.SpeechToText: "Speech to text", - modality.TextToSpeech: "Text to speech", - modality.VideoGeneration: "Video generation", + modality.Vision: "Image understanding", + modality.ImageGeneration: "Image generation", + modality.SpeechToText: "Speech to text", + modality.TextToSpeech: "Text to speech", + modality.VideoGeneration: "Video generation", + modality.VideoUnderstanding: "Video understanding", } var descriptions = map[string]string{ - modality.Vision: "Look at local image files (screenshots, photos, diagrams) and answer questions about them through %s on %s. Use when the user asks about an image and you cannot see images yourself.", - modality.ImageGeneration: "Generate images from a text description through %s on %s and save them as files. Use when the user asks you to create an image. Each run is billed: confirm with the user first.", - modality.SpeechToText: "Transcribe a local audio file to text through %s on %s. Use when the user asks for the text of a recording.", - modality.TextToSpeech: "Turn text into a spoken audio file through %s on %s. Use when the user asks for audio of some text. Each run is billed: confirm with the user first.", - modality.VideoGeneration: "Generate a short video from a text description through %s on %s and save it as a file. Use when the user asks you to create a video. Each run is billed by the second and cannot be cancelled: confirm with the user first.", + modality.Vision: "Look at local image files (screenshots, photos, diagrams) and answer questions about them through %s on %s. Use when the user asks about an image and you cannot see images yourself.", + modality.ImageGeneration: "Generate images from a text description through %s on %s and save them as files. Use when the user asks you to create an image. Each run is billed: confirm with the user first.", + modality.SpeechToText: "Transcribe a local audio file to text through %s on %s. Use when the user asks for the text of a recording.", + modality.TextToSpeech: "Turn text into a spoken audio file through %s on %s. Use when the user asks for audio of some text. Each run is billed: confirm with the user first.", + modality.VideoGeneration: "Generate a short video from a text description through %s on %s and save it as a file. Use when the user asks you to create a video. Each run is billed by the second and cannot be cancelled: confirm with the user first.", + modality.VideoUnderstanding: "Answer questions about a local video file by sampling frames with ffmpeg and sending them to %s on %s. Use when the user asks what happens in a video, or to find a moment in a screen recording.", } var costRules = map[string]string{ - modality.Vision: "Each run is a billed request to the user's Provider. Run it when the user asks about specific images; do not run it speculatively over many files without asking.", - modality.ImageGeneration: "Every run costs money. Before each run, tell the user what you are about to generate and how many images, and wait for them to agree.", - modality.SpeechToText: "Each run is billed by the length of the audio. Run it when the user asks for a transcript; ask first before transcribing long or many recordings.", - modality.TextToSpeech: "Every run costs money, billed by the length of the text. Before each run, tell the user what text you are about to speak and wait for them to agree.", - modality.VideoGeneration: "Every run costs money, billed by the second of video, and a submitted run cannot be cancelled. Before each run, tell the user the prompt, the duration and the resolution or aspect ratio you will use, and wait for them to agree. Never retry a failed or timed-out run on your own: a timed-out run may still finish and be billed.", + modality.Vision: "Each run is a billed request to the user's Provider. Run it when the user asks about specific images; do not run it speculatively over many files without asking.", + modality.ImageGeneration: "Every run costs money. Before each run, tell the user what you are about to generate and how many images, and wait for them to agree.", + modality.SpeechToText: "Each run is billed by the length of the audio. Run it when the user asks for a transcript; ask first before transcribing long or many recordings.", + modality.TextToSpeech: "Every run costs money, billed by the length of the text. Before each run, tell the user what text you are about to speak and wait for them to agree.", + modality.VideoGeneration: "Every run costs money, billed by the second of video, and a submitted run cannot be cancelled. Before each run, tell the user the prompt, the duration and the resolution or aspect ratio you will use, and wait for them to agree. Never retry a failed or timed-out run on your own: a timed-out run may still finish and be billed.", + modality.VideoUnderstanding: "Each run is a billed request carrying every sampled frame as an image, so it costs about as much as that many image questions. Run it when the user asks about a specific video; ask before raising --frames or running it over many files.", } // Render writes the Skill tree into dir, which must exist and be empty. The diff --git a/internal/skillgen/skillgen_test.go b/internal/skillgen/skillgen_test.go index 662e1571..7707caac 100644 --- a/internal/skillgen/skillgen_test.go +++ b/internal/skillgen/skillgen_test.go @@ -37,6 +37,10 @@ var goldenSpecs = map[string]Spec{ Capability: modality.TextToSpeech, Adapter: modality.AdapterNovitaAudio, ProviderID: "ppio", ProviderName: "PPIO", Model: "minimax-speech-2.8-turbo", BaseURL: "https://api.ppio.com/openai", NativeBaseURL: "https://api.ppio.com", }, + modality.VideoUnderstanding: { + Capability: modality.VideoUnderstanding, Adapter: modality.AdapterFramesVision, + ProviderID: "novita", ProviderName: "Novita", Model: "qwen/qwen3-vl-8b", BaseURL: "https://api.novita.ai/openai", + }, modality.VideoGeneration: { Capability: modality.VideoGeneration, Adapter: modality.AdapterNovitaAsync, ProviderID: "jiekou", ProviderName: "JieKou.AI", Model: "wan2.7-t2v", BaseURL: "https://api.highwayapi.ai/openai", NativeBaseURL: "https://api.highwayapi.ai", diff --git a/internal/skillgen/templates/run.mjs b/internal/skillgen/templates/run.mjs index 2b19cb94..a38b5e7e 100644 --- a/internal/skillgen/templates/run.mjs +++ b/internal/skillgen/templates/run.mjs @@ -6,9 +6,10 @@ // dependencies. The API key comes from the environment variable named in // config.json, or from BootAgent's private fallback file; it is never printed. -import { createWriteStream, existsSync, mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs"; -import { homedir } from "node:os"; -import { basename, dirname, extname, join, resolve } from "node:path"; +import { execFile } from "node:child_process"; +import { accessSync, constants, createWriteStream, existsSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { homedir, tmpdir } from "node:os"; +import { basename, delimiter, dirname, extname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; const EXIT = { ok: 0, internal: 1, usage: 2, setup: 3, refused: 4, key: 5, unavailable: 6, pending: 7 }; @@ -482,6 +483,88 @@ async function downloadTo(url, path, timeoutMs) { // Each capability and adapter this script implements: the subcommand, its // handler and default timeout in seconds, plus any extra subcommands. +// findExecutable looks a command up on PATH the way a shell would. +function findExecutable(name) { + const names = process.platform === "win32" ? [`${name}.exe`, `${name}.cmd`, name] : [name]; + for (const directory of (process.env.PATH ?? "").split(delimiter)) { + if (!directory) continue; + for (const candidate of names) { + const path = join(directory, candidate); + try { + accessSync(path, constants.X_OK); + if (statSync(path).isFile()) return path; + } catch { + // Not here. + } + } + } + return ""; +} + +// runTool runs a helper with an argv array, a timeout, and an environment of +// PATH alone: the child never sees the API key. +function runTool(file, args, timeoutMs) { + const env = { PATH: process.env.PATH ?? "" }; + if (process.env.SystemRoot) env.SystemRoot = process.env.SystemRoot; + return new Promise((done) => { + execFile(file, args, { env, timeout: timeoutMs, maxBuffer: 4 * MiB, windowsHide: true }, (error, stdout, stderr) => { + done({ code: error ? (typeof error.code === "number" ? error.code : -1) : 0, stderr: String(stderr ?? "") }); + }); + }); +} + +async function videoDuration(ffmpeg, path) { + // ffmpeg with an input and no output prints the input's details, then + // exits non-zero; the duration line is what is wanted. + const { stderr } = await runTool(ffmpeg, ["-hide_banner", "-i", path], 30000); + const match = /Duration:\s*(\d+):(\d{2}):(\d{2}(?:\.\d+)?)/.exec(stderr); + return match ? Number(match[1]) * 3600 + Number(match[2]) * 60 + Number(match[3]) : 0; +} + +async function describeVideo(config, key, options, timeoutMs) { + if (!options.video) fail(EXIT.usage, "Give a --video file."); + const path = resolve(options.video); + let info; + try { + info = statSync(path); + } catch { + fail(EXIT.usage, `Video file not found: ${options.video}`); + } + if (!info.isFile()) fail(EXIT.usage, `Video path is not a file: ${options.video}`); + const count = positiveInt(options.frames, "frames", 8, 16); + const ffmpeg = findExecutable("ffmpeg"); + if (!ffmpeg) fail(EXIT.setup, "ffmpeg was not found on PATH. Install ffmpeg to describe videos."); + const duration = await videoDuration(ffmpeg, path); + const work = mkdtempSync(join(tmpdir(), "bootagent-frames-")); + try { + const frames = []; + for (let index = 0; index < count; index++) { + // Evenly spaced, at the middle of each slice; without a duration, one + // frame a second from the start. + const at = duration > 0 ? Math.min((duration / count) * (index + 0.5), Math.max(duration - 0.05, 0)) : index; + const out = join(work, `frame-${index}.jpg`); + await runTool(ffmpeg, ["-hide_banner", "-loglevel", "error", "-ss", at.toFixed(2), "-i", path, "-frames:v", "1", "-vf", "scale='min(1024,iw)':-2", "-q:v", "4", "-y", out], 60000); + if (existsSync(out) && statSync(out).size > 0) frames.push({ at, data: readFileSync(out) }); + } + if (!frames.length) fail(EXIT.usage, `No frames could be read from ${options.video}.`); + const when = frames.map((frame) => `${frame.at.toFixed(1)} s`).join(", "); + const preface = `${options.prompt || "Describe what happens in this video."}\n\nThe images are ${frames.length} frames sampled in order from a${duration > 0 ? ` ${duration.toFixed(1)} s` : ""} video, at ${when}.`; + const content = [{ type: "text", text: preface }, ...frames.map((frame) => ({ type: "image_url", image_url: { url: `data:image/jpeg;base64,${frame.data.toString("base64")}` } }))]; + const response = await call(config, key, "/chat/completions", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ model: config.model, messages: [{ role: "user", content }], max_tokens: positiveInt(options["max-tokens"], "max-tokens", 1024, 32768) }), + }, timeoutMs); + const body = await json(response); + const text = body?.choices?.[0]?.message?.content; + const answer = Array.isArray(text) ? text.map((part) => part?.text ?? "").join("") : text; + if (typeof answer !== "string") fail(EXIT.unavailable, "The provider returned no text."); + process.stdout.write(answer.trim() + "\n"); + } finally { + rmSync(work, { recursive: true, force: true }); + } +} + const COMMANDS = { "vision/openai-chat-vision": { name: "describe", run: describe, timeout: 120 }, "image-generation/openai-images": { name: "generate", run: generate, timeout: 300 }, @@ -489,6 +572,7 @@ const COMMANDS = { "text-to-speech/openai-audio-speech": { name: "speak", run: speak, timeout: 120 }, "text-to-speech/novita-audio": { name: "speak", run: speakNative, timeout: 120 }, "video-generation/novita-async": { name: "video", run: generateVideo, timeout: 600, extra: { "video-result": videoResult } }, + "video-understanding/frames-chat-vision": { name: "describe-video", run: describeVideo, timeout: 300 }, }; async function main() { diff --git a/internal/skillgen/templates/video-understanding.md.tmpl b/internal/skillgen/templates/video-understanding.md.tmpl new file mode 100644 index 00000000..7522a69b --- /dev/null +++ b/internal/skillgen/templates/video-understanding.md.tmpl @@ -0,0 +1,25 @@ +{{template "header" .}} +## When to use + +When the user asks about the contents of a local video file: what happens in a screen recording, where an error appears, what a clip shows. It samples still frames, so it sees what is on screen but not the audio. + +## Requirements + +`ffmpeg` must be on `PATH` where you run commands. If it is missing the script exits with code 3; tell the user to install ffmpeg (for example `brew install ffmpeg` on macOS) and try again. + +## Command + +Run the script in this Skill's directory (the directory containing this `SKILL.md`): + +```bash +node /scripts/run.mjs describe-video --video [--prompt ""] [--frames <1-16>] [--max-tokens ] +``` + +- `--video`: a local video file ffmpeg can read (mp4, mov, webm and similar). +- `--prompt`: what to ask about the video. Put the user's actual question here. Default: "Describe what happens in this video." +- `--frames`: how many frames to sample, spread evenly over the video. Default 8. More frames see more but cost more. +- `--max-tokens`: the longest answer to allow. Default 1024. +- `--timeout`: seconds to wait. Default 300. + +The answer is printed to stdout as plain text. The frames are written to a temporary directory and removed afterwards. +{{template "footer" .}} diff --git a/internal/skillgen/testdata/golden/video-understanding/SKILL.md b/internal/skillgen/testdata/golden/video-understanding/SKILL.md new file mode 100644 index 00000000..b2f8ffa8 --- /dev/null +++ b/internal/skillgen/testdata/golden/video-understanding/SKILL.md @@ -0,0 +1,51 @@ +--- +name: bootagent-video-understanding +description: "Answer questions about a local video file by sampling frames with ffmpeg and sending them to qwen/qwen3-vl-8b on Novita. Use when the user asks what happens in a video, or to find a moment in a screen recording." +--- + +# Video understanding + +Generated by BootAgent. It calls `qwen/qwen3-vl-8b` on the user's Novita account. BootAgent rewrites this Skill when the model or Provider changes, so do not edit it. + +## When to use + +When the user asks about the contents of a local video file: what happens in a screen recording, where an error appears, what a clip shows. It samples still frames, so it sees what is on screen but not the audio. + +## Requirements + +`ffmpeg` must be on `PATH` where you run commands. If it is missing the script exits with code 3; tell the user to install ffmpeg (for example `brew install ffmpeg` on macOS) and try again. + +## Command + +Run the script in this Skill's directory (the directory containing this `SKILL.md`): + +```bash +node /scripts/run.mjs describe-video --video [--prompt ""] [--frames <1-16>] [--max-tokens ] +``` + +- `--video`: a local video file ffmpeg can read (mp4, mov, webm and similar). +- `--prompt`: what to ask about the video. Put the user's actual question here. Default: "Describe what happens in this video." +- `--frames`: how many frames to sample, spread evenly over the video. Default 8. More frames see more but cost more. +- `--max-tokens`: the longest answer to allow. Default 1024. +- `--timeout`: seconds to wait. Default 300. + +The answer is printed to stdout as plain text. The frames are written to a temporary directory and removed afterwards. + +## Rules + +- Each run is a billed request carrying every sampled frame as an image, so it costs about as much as that many image questions. Run it when the user asks about a specific video; ask before raising --frames or running it over many files. +- Never print, echo or log environment variables, `~/.bootagent/skill-env/novita.env`, or anything that could reveal the API key. The script reads the key itself; you never need it. +- Only send files the user asked about. They go to the user's own Provider and nowhere else. + +## Exit codes + +| Code | Meaning | What to tell the user | +| --- | --- | --- | +| 0 | Success | Report the result. | +| 2 | Bad arguments or input file | Fix the command, or ask the user for the right file. | +| 3 | Setup problem: Node.js older than 18, or no API key found | Ask the user to restart the Agent so it picks up `BOOTAGENT_NOVITA_API_KEY`, or to reinstall this Skill from BootAgent. | +| 4 | The Provider refused the request | Show the Provider's message. The model may not support this input; the user can choose another model in BootAgent. | +| 5 | The Provider rejected the API key | Ask the user to check the Novita key in BootAgent. | +| 6 | The Provider was unreachable, busy or timed out | Suggest trying again later. | + +`node /scripts/run.mjs check` confirms the key is available without sending a request. diff --git a/internal/skillgen/testdata/golden/video-understanding/config.json b/internal/skillgen/testdata/golden/video-understanding/config.json new file mode 100644 index 00000000..6ad6e96e --- /dev/null +++ b/internal/skillgen/testdata/golden/video-understanding/config.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "generator": "bootagent", + "capability": "video-understanding", + "adapter": "frames-chat-vision", + "provider_id": "novita", + "provider_name": "Novita", + "model": "qwen/qwen3-vl-8b", + "base_url": "https://api.novita.ai/openai/v1", + "key_env": "BOOTAGENT_NOVITA_API_KEY", + "key_file": "~/.bootagent/skill-env/novita.env", + "output_dir": "bootagent-output" +} diff --git a/internal/skillgen/video_script_test.go b/internal/skillgen/video_script_test.go new file mode 100644 index 00000000..3ee7da8e --- /dev/null +++ b/internal/skillgen/video_script_test.go @@ -0,0 +1,108 @@ +package skillgen + +import ( + "encoding/json" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/MaimoryLab/BootAgent/internal/modality" +) + +// fakeFFmpeg is a shell script standing in for ffmpeg: asked for details it +// prints a Duration line and fails, as ffmpeg does with no output; asked for a +// frame it writes a tiny JPEG. It records its environment so the test can +// prove the key never reaches it. +func fakeFFmpeg(t *testing.T, envDump string) string { + t.Helper() + if runtime.GOOS == "windows" { + t.Skip("the fake ffmpeg is a shell script") + } + dir := resolvedTempDir(t) + script := `#!/bin/sh +env >> '` + envDump + `' +for arg in "$@"; do last="$arg"; done +case " $* " in + *" -frames:v "*) printf '\377\330\377\340fakejpeg' > "$last"; exit 0 ;; + *) echo " Duration: 00:00:10.00, start: 0.000000, bitrate: 100 kb/s" >&2; exit 1 ;; +esac +` + if err := os.WriteFile(filepath.Join(dir, "ffmpeg"), []byte(script), 0o700); err != nil { + t.Fatal(err) + } + return dir +} + +var framesSpec = Spec{Capability: modality.VideoUnderstanding, Adapter: modality.AdapterFramesVision, ProviderID: "local", ProviderName: "Local", Model: "vision-model"} + +func TestScriptDescribesAVideoFromSampledFrames(t *testing.T) { + fake, dir, work, home := scriptSetup(t, framesSpec.Capability, framesSpec.Adapter, framesSpec.Model) + envDump := filepath.Join(resolvedTempDir(t), "ffmpeg-env") + tools := fakeFFmpeg(t, envDump) + if err := os.WriteFile(filepath.Join(work, "clip.mp4"), []byte("not really a video"), 0o600); err != nil { + t.Fatal(err) + } + path := "PATH=" + tools + string(os.PathListSeparator) + os.Getenv("PATH") + run := runScript(t, dir, work, home, []string{keyEnv, path}, "describe-video", "--video", "clip.mp4", "--frames", "4", "--prompt", "Where does the error appear?") + if run.code != 0 || run.stdout != "A red error banner.\n" { + t.Fatalf("run = %#v", run) + } + request, _ := fake.last("/v1/chat/completions") + var body struct { + Messages []struct { + Content []map[string]any `json:"content"` + } `json:"messages"` + } + if err := json.Unmarshal(request.body, &body); err != nil { + t.Fatal(err) + } + parts := body.Messages[0].Content + if len(parts) != 5 { + t.Fatalf("want a text part and 4 frames, got %d parts", len(parts)) + } + text, _ := parts[0]["text"].(string) + if !strings.Contains(text, "Where does the error appear?") || !strings.Contains(text, "10.0 s video") || !strings.Contains(text, "1.3 s, 3.8 s, 6.3 s, 8.8 s") { + t.Fatalf("preface = %q", text) + } + if url := parts[1]["image_url"].(map[string]any)["url"].(string); !strings.HasPrefix(url, "data:image/jpeg;base64,") { + t.Fatalf("frame = %q", url) + } + // ffmpeg runs with PATH alone: never the key, never its variable name. + dump, err := os.ReadFile(envDump) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(dump), scriptKey) || strings.Contains(string(dump), "BOOTAGENT_") { + t.Fatalf("ffmpeg saw the key:\n%s", dump) + } + // The temporary frame directory is gone; ffmpeg wrote its frames there. + if matches, _ := filepath.Glob(filepath.Join(os.TempDir(), "bootagent-frames-*", "frame-*.jpg")); len(matches) != 0 { + t.Fatalf("frames left behind: %v", matches) + } +} + +func TestScriptVideoNeedsFFmpeg(t *testing.T) { + _, dir, work, home := scriptSetup(t, framesSpec.Capability, framesSpec.Adapter, framesSpec.Model) + if err := os.WriteFile(filepath.Join(work, "clip.mp4"), []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + node, _ := exec.LookPath("node") + // A PATH holding only node: no ffmpeg anywhere. + only := resolvedTempDir(t) + if err := os.Symlink(node, filepath.Join(only, "node")); err != nil { + t.Fatal(err) + } + run := runScript(t, dir, work, home, []string{keyEnv, "PATH=" + only}, "describe-video", "--video", "clip.mp4") + if run.code != 3 || !strings.Contains(run.stderr, "ffmpeg was not found") { + t.Fatalf("run = %#v", run) + } + if run := runScript(t, dir, work, home, []string{keyEnv}, "describe-video", "--video", "missing.mp4"); run.code != 2 { + t.Fatalf("missing file = %#v", run) + } + if run := runScript(t, dir, work, home, []string{keyEnv}, "describe-video", "--video", "clip.mp4", "--frames", "40"); run.code != 2 { + t.Fatalf("too many frames = %#v", run) + } +} From d5be1b5ab0929ca2368bc40075c638c0b70580f3 Mon Sep 17 00:00:00 2001 From: yujiezhang-ops Date: Fri, 9 Oct 2026 17:38:17 +0800 Subject: [PATCH 09/18] docs: record the decision not to offer multimodal Skills to Hermes Co-Authored-By: Claude Opus 5 (1M context) --- docs/multimodal-skills-design.md | 27 ++++++++++++++++++--------- internal/app/multimodal_install.go | 9 +++++---- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/docs/multimodal-skills-design.md b/docs/multimodal-skills-design.md index fd47dd2a..51fc1561 100644 --- a/docs/multimodal-skills-design.md +++ b/docs/multimodal-skills-design.md @@ -1,6 +1,6 @@ # Multimodal Skills: Design -Status: Phases 1 to 4 are implemented, with one open decision (Hermes, under Open questions). Branch `codex/multimodal-skills`. +Status: Phases 1 to 4 are implemented. Branch `codex/multimodal-skills`. ## Goal @@ -428,7 +428,7 @@ secret and stay in `config.json`. | Claude Code | `env` in `~/.claude/settings.json` | Already a secret file under ADR-008; the Bash tool inherits `env`. | | Codex | `[shell_environment_policy.set]` in `~/.codex/config.toml` | `set` is applied after Codex's default `KEY`/`SECRET`/`TOKEN` exclusion, so the variable survives it. The table is merged with the user's own entries and placed after the user's tables; inline-table and dotted-key spellings of it are refused rather than duplicated. Every writer of `config.toml` (Profile, MCP, Skill) writes it as a secret whenever the old or the new content holds a `BOOTAGENT_*` key, since the backup it takes is of the old content. | | OpenCode | The fallback file | Its config documents `{env:VAR}` substitution into config values and an `environment` key for formatters only, nothing that reaches the shell tool. | -| Hermes | The fallback file, once offered | Its README and `.env.example` say nothing about what its terminal tool inherits; under the SSH backend the Agent cannot read `.env`. Not offered yet (Open questions). | +| Hermes | Not offered | Its README and `.env.example` say nothing about what its terminal tool inherits; under the SSH backend the Agent cannot read `.env`. See Hermes below. | Fallback for an Agent with no configurable environment: a private `~/.bootagent/skill-env/.env`, the same pattern ADR-008 uses for @@ -472,8 +472,8 @@ The Skills page gets a Multimodal capabilities section: 2. Per capability, a model choice and, where the probe table allows, a Probe button. 3. Target Agents: those with `skills_path` in `agents.lock.json`: Claude Code, - Codex and OpenCode. Hermes is held back (Open questions). The confirmation - names the fallback file when a chosen Agent reads it. + Codex and OpenCode. Hermes is not offered (see Hermes below). The + confirmation names the fallback file when a chosen Agent reads it. 4. Install: a confirmation names the variable, states that the Agent and its model can read it, and notes the Node 18 requirement. Then the tree is generated, stored and published, the variables are written, and the result is @@ -489,6 +489,19 @@ the error message in the UI" for image understanding. Uninstall reuses the existing Skill uninstall (`SkillService.Uninstall`), which keeps a backup and now also runs the reconcile, so no separate multimodal uninstall exists. +### Hermes + +Decided on 2026-10-09: the multimodal Skill is not offered to Hermes as an +install target. Hermes is installed by its official script, so Node may be +absent from its PATH. Pointing `SKILL.md` at BootAgent's managed Node by absolute +path was considered and not taken: that path is versioned +(`~/.bootagent/runtimes/node/v/…`), exists only when the user installed +Node through BootAgent, and moves when the lock file bumps Node, so every Hermes +Skill would need re-rendering on each upgrade. Neither the path nor the key file +exists when Hermes runs commands on a remote backend (SSH, Docker) either. +Hermes users can still use the guide below. `pendingSkillTargets` in +`internal/app/multimodal_install.go` holds the exclusion. + ### Agents without Skills Kilo CLI, Aider, OpenClaw, Kimi Code, pi, dsh and the desktop apps have no Skills @@ -548,7 +561,7 @@ not write is left alone, as in an Agent's Skills directory. document; catalog image models on the images route. PPIO video is not implemented (no documented route). 4. Video understanding by frame sampling; OpenCode on the fallback file; the - guide for Agents without Skills. Hermes is held back pending a decision. + guide for Agents without Skills. Hermes is not offered (see Hermes). ## Open questions @@ -557,10 +570,6 @@ not write is left alone, as in an Agent's Skills directory. route, and PPIO and Novita video generation, need documenting by the vendors. - How OpenCode and Hermes pass configured environment variables to their shell tools. -- Hermes is installed by its official script, so Node may be absent. If so, should - `SKILL.md` point at BootAgent's managed Node runtime by absolute path, or should - the Skill not be offered to Hermes? Until this is decided, Hermes is not an - install target (`pendingSkillTargets` in `internal/app/multimodal_install.go`). - Whether the UI shows a price per call. The catalog has no pricing today; v1 shows a generic cost note only. diff --git a/internal/app/multimodal_install.go b/internal/app/multimodal_install.go index a6f4cb55..00d5b1ff 100644 --- a/internal/app/multimodal_install.go +++ b/internal/app/multimodal_install.go @@ -72,10 +72,11 @@ type InstalledMultimodal struct { // variables (ADR-011). Other Agents get the private fallback file. var nativeSkillEnvAdapters = map[string]bool{"claude-code": true, "codex": true} -// pendingSkillTargets are Agents with a Skills directory that are not offered -// yet. Hermes is installed by its official script, so Node may be absent -// where it runs; whether its Skill should run BootAgent's managed Node by -// absolute path is an open decision (design doc, Open questions). +// pendingSkillTargets are Agents with a Skills directory that are not offered. +// Hermes is installed by its official script, so Node may be absent where it +// runs, and BootAgent's managed Node sits at a versioned path that moves on +// every upgrade and does not exist on Hermes's remote backends. Decided not to +// offer it (design doc, Hermes); Hermes users can use the guide. var pendingSkillTargets = map[string]bool{"hermes": true} // multimodalInstallTargets are the Agents a multimodal Skill can be installed From 2a2d98d240d60400e88381e22f3de34dcda0a767 Mon Sep 17 00:00:00 2001 From: yujiezhang-ops Date: Fri, 9 Oct 2026 17:58:08 +0800 Subject: [PATCH 10/18] fix: recover ChatGPT desktop version from the registered AppX package When Get-AppxPackage returns nothing or fails, look the package family up through PackageManager.FindPackagesForUser before reporting the version as unavailable. The family is passed through the environment, never interpolated into the script. If metadata is still missing, the card says launch can still be attempted and that version detection is unrelated to model API support. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/components/DesktopAppSection.test.tsx | 14 ++++ frontend/src/components/DesktopAppSection.tsx | 5 +- internal/desktopapp/desktopapp.go | 56 ++++++++++++++- internal/desktopapp/desktopapp_test.go | 71 ++++++++++++++++++- 4 files changed, 142 insertions(+), 4 deletions(-) diff --git a/frontend/src/components/DesktopAppSection.test.tsx b/frontend/src/components/DesktopAppSection.test.tsx index aa63d15a..c22e8cf5 100644 --- a/frontend/src/components/DesktopAppSection.test.tsx +++ b/frontend/src/components/DesktopAppSection.test.tsx @@ -285,6 +285,20 @@ describe("DesktopAppSection", () => { expect(screen.queryByRole("heading", { name: "桌面 Agent" })).toBeNull(); }); + it("explains missing version metadata without blocking launch", async () => { + bridge.openDesktopAgent.mockResolvedValue(undefined); + render( + + + , + ); + + expect(screen.getByText("已检测到应用，但版本信息不可用")).toBeTruthy(); + expect(screen.getByText("可继续尝试启动；版本检测与模型 API 兼容性无关。")).toBeTruthy(); + fireEvent.click(screen.getByRole("button", { name: "启动" })); + await waitFor(() => expect(bridge.openDesktopAgent).toHaveBeenCalledWith("chatgpt-desktop")); + }); + // This component renders every desktop Agent, and it used to hardcode // agentId="codex" for all of them. WorkBuddy is a Tencent product, so its card // displayed OpenAI's mark -- a trademark problem rather than a styling one. diff --git a/frontend/src/components/DesktopAppSection.tsx b/frontend/src/components/DesktopAppSection.tsx index 2c9c5fe2..e04262e9 100644 --- a/frontend/src/components/DesktopAppSection.tsx +++ b/frontend/src/components/DesktopAppSection.tsx @@ -143,7 +143,10 @@ export function DesktopAppSection({ app: desktopApp, onChanged, onSetup, onConfi {desktopApp.inspectionUnavailable ? (
    ) : null}
    diff --git a/internal/desktopapp/desktopapp.go b/internal/desktopapp/desktopapp.go index 48a41f14..5ec8caa9 100644 --- a/internal/desktopapp/desktopapp.go +++ b/internal/desktopapp/desktopapp.go @@ -456,6 +456,7 @@ func inspectChatGPTWindows(ctx context.Context, options Options) (Status, error) if status.Version == nil { status.Version = nonEmptyPointer(versionFromPackageFullName(packageInfo.PackageFullName)) } + recoverChatGPTWindowsMetadata(ctx, options, &status) return status, nil } // An empty JSON result means no registered package. The StartApps query @@ -482,8 +483,7 @@ func inspectChatGPTWindows(ctx context.Context, options Options) (Status, error) } status.Installed = true status.PackageFamily = appID[:strings.IndexByte(appID, '!')] - message := "Windows registered the app, but AppX version metadata was unavailable" - status.InspectionUnavailable = &message + recoverChatGPTWindowsMetadata(ctx, options, &status) return status, nil } if packageQueryErr != nil { @@ -492,6 +492,37 @@ func inspectChatGPTWindows(ctx context.Context, options Options) (Status, error) return status, nil } +func recoverChatGPTWindowsMetadata(ctx context.Context, options Options, status *Status) { + if status.Version != nil { + return + } + if status.PackageFamily != "" { + result, err := runWithEnvironment(options, ctx, chatGPTWindowsMetadataQuery(), map[string]string{ + "BOOTAGENT_DESKTOP_PACKAGE_FAMILY": status.PackageFamily, + }, inspectTimeout) + if err == nil && result.ExitCode == 0 { + metadata, ok := parseWindowsPackage(result.Stdout) + family := strings.TrimSpace(metadata.PackageFamilyName) + if family == "" { + family = packageFamilyFromFullName(metadata.PackageFullName) + } + if ok && isChatGPTPackage(metadata.Name, metadata.PackageFullName) && strings.EqualFold(family, status.PackageFamily) { + status.Version = nonEmptyPointer(metadata.Version) + if status.Version == nil { + status.Version = nonEmptyPointer(versionFromPackageFullName(metadata.PackageFullName)) + } + if path := strings.TrimSpace(metadata.InstallLocation); path != "" { + status.Path = path + } + } + } + } + if status.Version == nil { + message := "Windows registered the app, but AppX version metadata was unavailable" + status.InspectionUnavailable = &message + } +} + type windowsPackage struct { Name string `json:"Name"` PackageFullName string `json:"PackageFullName"` @@ -517,6 +548,27 @@ Get-StartApps | Where-Object { $_.AppID -like 'OpenAI.Codex_*!App' -or $_.AppID return []string{"powershell.exe", "-NoProfile", "-NonInteractive", "-Command", script} } +func chatGPTWindowsMetadataQuery() []string { + script := `[Console]::OutputEncoding = [Text.Encoding]::UTF8 +$ErrorActionPreference = 'Stop' +$manager = [Windows.Management.Deployment.PackageManager,Windows.Management.Deployment,ContentType=WindowsRuntime]::new() +$items = @($manager.FindPackagesForUser('', $env:BOOTAGENT_DESKTOP_PACKAGE_FAMILY)) +$items | ForEach-Object { + $identity = $_.Id + $version = $identity.Version + $installLocation = '' + try { $installLocation = $_.InstalledLocation.Path } catch {} + [PSCustomObject]@{ + Name = $identity.Name + PackageFullName = $identity.FullName + PackageFamilyName = $identity.FamilyName + Version = '{0}.{1}.{2}.{3}' -f $version.Major,$version.Minor,$version.Build,$version.Revision + InstallLocation = $installLocation + } +} | Sort-Object { [version]$_.Version } -Descending | Select-Object -First 1 | ConvertTo-Json -Compress` + return []string{"powershell.exe", "-NoProfile", "-NonInteractive", "-Command", script} +} + func parseWindowsPackage(output string) (windowsPackage, bool) { trimmed := strings.TrimPrefix(strings.TrimSpace(output), "\ufeff") if trimmed == "" { diff --git a/internal/desktopapp/desktopapp_test.go b/internal/desktopapp/desktopapp_test.go index 3a7fb433..6d92a7f0 100644 --- a/internal/desktopapp/desktopapp_test.go +++ b/internal/desktopapp/desktopapp_test.go @@ -388,12 +388,81 @@ func TestInspectWindowsStartAppsFallbackReportsUnknownMetadata(t *testing.T) { if status.InspectionUnavailable == nil { t.Fatalf("appID=%q fallback did not explain unavailable version metadata", appID) } - if len(runner.calls) != 2 { + if len(runner.calls) != 3 { t.Fatalf("appID=%q queries = %#v", appID, runner.calls) } } } +func TestInspectWindowsRecoversRegisteredPackageMetadata(t *testing.T) { + for _, initial := range []process.Result{ + {ExitCode: 0}, + {ExitCode: 1, Stderr: "AppX module unavailable"}, + } { + runner := &scriptedRunner{results: []process.Result{ + initial, + {ExitCode: 0, Stdout: "OpenAI.Codex_2p2nqsd0c76g0!App\n"}, + {ExitCode: 0, Stdout: `{"Name":"OpenAI.Codex","PackageFullName":"OpenAI.Codex_26.10.3.0_neutral__2p2nqsd0c76g0","PackageFamilyName":"OpenAI.Codex_2p2nqsd0c76g0","Version":"26.10.3.0","InstallLocation":"C:\\Apps\\Codex"}`}, + }} + status := Inspect(context.Background(), ChatGPTDesktopID, Options{Platform: platform.For("windows", "amd64"), Runner: runner}) + if !status.Installed || status.Version == nil || *status.Version != "26.10.3.0" || status.Path != `C:\Apps\Codex` || status.InspectionUnavailable != nil { + t.Fatalf("initial=%#v status = %#v", initial, status) + } + if len(runner.calls) != 3 { + t.Fatalf("queries = %#v", runner.calls) + } + script := strings.Join(runner.calls[2], " ") + if !strings.Contains(script, "FindPackagesForUser") || !strings.Contains(script, "$env:BOOTAGENT_DESKTOP_PACKAGE_FAMILY") { + t.Fatalf("metadata query = %q", script) + } + if runner.environments[2]["BOOTAGENT_DESKTOP_PACKAGE_FAMILY"] != "OpenAI.Codex_2p2nqsd0c76g0" || strings.Contains(script, "OpenAI.Codex_2p2nqsd0c76g0") { + t.Fatalf("metadata query must receive the package family through the environment: %#v", runner.environments) + } + for _, forbidden := range []string{"WindowsApps", "Get-ChildItem", "AllUsers"} { + if strings.Contains(script, forbidden) { + t.Fatalf("metadata query contains %q: %s", forbidden, script) + } + } + } +} + +func TestInspectWindowsRecoversAMissingPackageVersion(t *testing.T) { + runner := &scriptedRunner{results: []process.Result{ + {ExitCode: 0, Stdout: `{"Name":"OpenAI.Codex","PackageFamilyName":"OpenAI.Codex_2p2nqsd0c76g0"}`}, + {ExitCode: 0, Stdout: `{"Name":"OpenAI.Codex","PackageFullName":"OpenAI.Codex_26.10.3.0_neutral__2p2nqsd0c76g0","PackageFamilyName":"OpenAI.Codex_2p2nqsd0c76g0"}`}, + }} + status := Inspect(context.Background(), ChatGPTDesktopID, Options{Platform: platform.For("windows", "amd64"), Runner: runner}) + if !status.Installed || status.Version == nil || *status.Version != "26.10.3.0" || status.InspectionUnavailable != nil { + t.Fatalf("status = %#v", status) + } + if len(runner.calls) != 2 || !strings.Contains(strings.Join(runner.calls[1], " "), "FindPackagesForUser") { + t.Fatalf("queries = %#v", runner.calls) + } +} + +func TestInspectWindowsMetadataFailureKeepsTheRegisteredApp(t *testing.T) { + for _, metadata := range []process.Result{ + {ExitCode: 1, Stderr: "Access denied"}, + {ExitCode: 0, Stdout: "not JSON"}, + {ExitCode: 0, Stdout: `{"Name":"Other.App","Version":"1.2.3.0","PackageFamilyName":"Other.App_publisher"}`}, + {ExitCode: 0, Stdout: `{"Name":"OpenAI.CodexBeta","Version":"1.2.3.0","PackageFamilyName":"OpenAI.CodexBeta_2p2nqsd0c76g0"}`}, + {ExitCode: 0, Stdout: `{"Name":"OpenAI.Codex","PackageFamilyName":"OpenAI.Codex_2p2nqsd0c76g0"}`}, + } { + runner := &scriptedRunner{results: []process.Result{ + {ExitCode: 0}, + {ExitCode: 0, Stdout: "OpenAI.Codex_2p2nqsd0c76g0!App\n"}, + metadata, + }} + status := Inspect(context.Background(), ChatGPTDesktopID, Options{Platform: platform.For("windows", "arm64"), Runner: runner}) + if !status.Installed || status.PackageFamily != "OpenAI.Codex_2p2nqsd0c76g0" || status.Version != nil || status.InspectionUnavailable == nil { + t.Fatalf("metadata=%#v status = %#v", metadata, status) + } + if len(runner.calls) != 3 { + t.Fatalf("queries = %#v", runner.calls) + } + } +} + func TestParseMountPointKeepsSpaces(t *testing.T) { got, err := parseMountPoint("/dev/disk4s1\tApple_HFS\t/Volumes/ChatGPT Installer\n") if err != nil || got != "/Volumes/ChatGPT Installer" { From 3a77c908a506aa79a843fe233c5df7515c72a2a9 Mon Sep 17 00:00:00 2001 From: yujiezhang-ops Date: Fri, 9 Oct 2026 17:58:08 +0800 Subject: [PATCH 11/18] feat: point refused Codex Responses checks at the protocol adapter A model check refused over the Responses API now explains that Codex needs a Responses model and offers a button to the API protocol adapter. Other reasons and protocols keep their existing remedies. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/components/ModelCheckNotice.test.tsx | 65 +++++++++++++++++++ frontend/src/components/ModelCheckNotice.tsx | 10 +++ internal/app/profile_model_check_test.go | 29 +++++++++ 3 files changed, 104 insertions(+) create mode 100644 frontend/src/components/ModelCheckNotice.test.tsx diff --git a/frontend/src/components/ModelCheckNotice.test.tsx b/frontend/src/components/ModelCheckNotice.test.tsx new file mode 100644 index 00000000..a07df9bc --- /dev/null +++ b/frontend/src/components/ModelCheckNotice.test.tsx @@ -0,0 +1,65 @@ +import { fireEvent, render, screen } from "@testing-library/react"; +import { MemoryRouter, Route, Routes } from "react-router-dom"; +import { describe, expect, it, vi } from "vitest"; + +import { translate } from "../i18n"; +import type { ProfileModelCheck } from "../types/api"; +import { describeModelCheck, ModelCheckNotice } from "./ModelCheckNotice"; + +function check(overrides: Partial = {}): ProfileModelCheck { + return { + profileId: "codex-gateway", + model: "aphrodite/venus-marble-5-5-es-cc-dst", + status: "unavailable", + reason: "refused", + protocol: "responses", + httpStatus: 400, + detail: "model: aphrodite/venus-marble-5-5-es-cc-dst does not support endpoint: responses", + agents: ["codex", "chatgpt-desktop"], + ...overrides, + }; +} + +function renderNotice(value: ProfileModelCheck) { + return render( + + + "Codex"} returnTo="/profiles" onDismiss={vi.fn()} />} /> + API 协议适配设置
    } /> + + , + ); +} + +describe("Codex model check guidance", () => { + it("offers an explicit adapter path when the Responses request is refused", () => { + renderNotice(check()); + + expect(screen.getByRole("alert").textContent).toContain("HTTP 400"); + expect(screen.getByRole("alert").textContent).toContain("does not support endpoint: responses"); + expect(screen.getByText("Codex 使用 Responses API。请选择支持 Responses 的模型；如果模型服务仅支持 Chat Completions，可使用 API 协议适配，并应用适配器生成的 Codex 配置模版。")).toBeTruthy(); + fireEvent.click(screen.getByRole("button", { name: "打开 API 协议适配" })); + expect(screen.getByText("API 协议适配设置")).toBeTruthy(); + }); + + it.each>([ + { reason: "key", httpStatus: 401 }, + { reason: "no_key", status: "unverified" }, + { reason: "network", status: "unverified" }, + { reason: "transient", status: "unverified", httpStatus: 500 }, + { protocol: "anthropic" }, + { protocol: "openai" }, + { status: "available", reason: undefined }, + ])("does not suggest protocol adaptation for $reason over $protocol", (overrides) => { + renderNotice(check(overrides)); + + expect(screen.queryByRole("button", { name: "打开 API 协议适配" })).toBeNull(); + expect(screen.queryByText(/Codex 使用 Responses API/)).toBeNull(); + }); + + it("provides the Responses remedy in English", () => { + const parts = describeModelCheck(check(), (key, values) => translate("en", key, values), () => "Codex", ", "); + + expect(parts.hint).toBe("Codex uses the Responses API. Choose a model that supports Responses. If the provider only supports Chat Completions, use the API protocol adapter and apply its generated Codex Profile."); + }); +}); diff --git a/frontend/src/components/ModelCheckNotice.tsx b/frontend/src/components/ModelCheckNotice.tsx index c23f793a..d48a5b6a 100644 --- a/frontend/src/components/ModelCheckNotice.tsx +++ b/frontend/src/components/ModelCheckNotice.tsx @@ -27,6 +27,7 @@ export function describeModelCheck(check: ProfileModelCheck, t: Translate, nameO reason: protocol ? t("模型服务已通过 {protocol} 正常响应。", { protocol }) : t("模型服务已正常响应。"), agents: agents ? t("已检测的 Agent：{agents}", { agents }) : "", detail: "", + hint: "", }; } const title = check.status === "unverified" @@ -61,6 +62,9 @@ export function describeModelCheck(check: ProfileModelCheck, t: Translate, nameO reason, agents: agents ? t("受影响的 Agent：{agents}", { agents }) : "", detail: check.detail ? t("模型服务返回：{detail}", { detail: check.detail }) : "", + hint: check.reason === "refused" && check.protocol === "responses" + ? t("Codex 使用 Responses API。请选择支持 Responses 的模型；如果模型服务仅支持 Chat Completions，可使用 API 协议适配，并应用适配器生成的 Codex 配置模版。") + : "", }; } @@ -141,8 +145,14 @@ export function ModelCheckNotice({ check, providerId, nameOf, returnTo, profileN {parts.reason} {parts.agents ? {parts.agents} : null} {parts.detail ? {parts.detail} : null} + {parts.hint ? {parts.hint} : null}
    + {parts.hint ? ( + + ) : null} {canRevert ? (
    } primaryLabel={dirtyCount ? t("应用 Skills（{count}）", { count: dirtyCount }) : t("应用 Skills")} onPrimary={apply} primaryDisabled={!dirty || busy}> -

    Skills

    {message ?

    {message}

    : null}
    +

    Skills

    {message ?

    {message}

    : null}
    {rows.length ?
    : null} {!rows.length ? {t("或者去工具市场发现")}} /> : !visibleRows.length ? :
    {visibleRows.map((row) => { const candidate = candidateMap.get(row.id); const hash = candidate?.hash ?? changes[row.id]?.variant_hash ?? row.variant_hashes?.[0] ?? ""; const detail = row.description || t("暂无详情"); const isDeleted = Boolean(pendingDeletes[row.id]); return ; })}
    {t("名称")}{t("详情")}{t("状态")}{t("选择目标 Agent")}
    {row.name || row.id}{compactDescription(detail)}{row.id}{isDeleted ? {t("待删除")} : row.conflict ? {t("冲突")} : {row.variants} variant} selectTarget(row.id, hash, agent, checked, row.agents ?? [])} />
    }
    - {previewOpen ? setPreviewOpen(false)}>

    {t("导入预览")}

    {candidates.map((candidate) => { const detail = candidate.description || candidate.hash.slice(0, 12); return
    ; })}
     : null} ; } diff --git a/frontend/src/styles/app.css b/frontend/src/styles/app.css index e629a08f..b6aced90 100644 --- a/frontend/src/styles/app.css +++ b/frontend/src/styles/app.css @@ -3946,7 +3946,6 @@ /* Multimodal capabilities on the Skills page: one block per keyed Provider, one row per capability. */ -.multimodal-section { margin-top: 28px; } .multimodal-section .section-heading h2 { display: flex; align-items: center; gap: 6px; } .multimodal-provider { display: grid; gap: 8px; padding: 12px 14px; border: 1px solid var(--border); border-radius: var(--radius-panel); } .multimodal-provider > header { display: flex; flex-wrap: wrap; align-items: baseline; gap: 10px; font-size: 13px; } From d8486e3f9eaba4c73dab5baa45b1a8200a802d07 Mon Sep 17 00:00:00 2001 From: yujiezhang-ops Date: Sat, 10 Oct 2026 10:38:41 +0800 Subject: [PATCH 13/18] feat: add Kimi Code Desktop as an installable desktop Agent Install reads the vendor's electron-updater feed, resolving its relative URLs against code.kimi.com and keeping downloads on code.kimi.com or cdn.kimi.com. On macOS it verifies the zip's SHA-512 and checks the bundle ID com.kimi.code.desktop, the Developer ID team 2J9472RW75 and notarization before copying the app into Applications. On Windows it verifies the installer's digest and its Authenticode publisher, NOVASCENT PRIVATE LIMITED (the signer of the vendor's published installer, pinned deliberately), then starts the NSIS installer. The app shares ~/.kimi-code/config.toml with the Kimi Code CLI, so applying a Profile to it activates the kimi-code Agent. The card reuses the Kimi mark. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 4 +- README_ZH.md | 4 +- frontend/src/components/icons/agents.test.tsx | 8 +- frontend/src/components/icons/agents.tsx | 10 + internal/app/desktopapp_test.go | 48 ++ .../testdata/status-empty-linux-arm64.json | 14 + internal/desktopapp/kimi.go | 552 ++++++++++++++++++ internal/desktopapp/kimi_test.go | 305 ++++++++++ internal/desktopapp/macos_verify.go | 6 + internal/desktopapp/registry.go | 16 + internal/desktopapp/windows_verify.go | 7 + 11 files changed, 969 insertions(+), 5 deletions(-) create mode 100644 internal/desktopapp/kimi.go create mode 100644 internal/desktopapp/kimi_test.go diff --git a/README.md b/README.md index ffe70feb..d8e473b9 100644 --- a/README.md +++ b/README.md @@ -100,10 +100,12 @@ BootAgent supports detection, configuration, launch, or installation guidance ac | Codex · Claude Code · OpenCode | DeepSeek Harness · Claude Desktop | | Kilo CLI · Aider · OpenClaw | ChatGPT Desktop · WorkBuddy | | Hermes Agent · Kimi Code · Pi | WorkBuddy AI · ZCode | -| DeepSeek Harness (local web app) | | +| DeepSeek Harness (local web app) | Kimi Code Desktop | Claude Desktop can be detected and launched on macOS and Windows, but it must be installed by the user from the [official download page](https://claude.com/download). BootAgent writes compatible configuration and does not download or proxy Claude Desktop. +Kimi Code Desktop is installed automatically on macOS and Windows. Before running the installer, BootAgent checks the download's digest and the app's signature: the Developer ID team on macOS, and the Authenticode publisher on Windows. It shares `~/.kimi-code/config.toml` with the Kimi Code CLI, so both Agents read the same configuration. + DeepSeek Harness can use its shipped DeepSeek route or a configured compatible Provider. It opens a local web app after installation; its own onboarding commands remain the source of truth. JieKou.AI, PPIO, Novita, DeepSeek, and Moonshot are built-in Providers. Custom OpenAI-compatible and Anthropic-compatible Providers can be added from the Provider page. diff --git a/README_ZH.md b/README_ZH.md index 6d2d7c64..c8bc4df3 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -98,10 +98,12 @@ BootAgent 会根据每个 Agent 的官方约定提供检测、配置、启动、 | Codex · Claude Code · OpenCode | DeepSeek Harness · Claude Desktop | | Kilo CLI · Aider · OpenClaw | ChatGPT Desktop · WorkBuddy | | Hermes Agent · Kimi Code · Pi | WorkBuddy AI · ZCode | -| DeepSeek Harness（本地 Web 应用） | | +| DeepSeek Harness（本地 Web 应用） | Kimi Code Desktop | Claude Desktop 可以在 macOS 和 Windows 上被检测和启动，但需要用户从[官方下载页面](https://claude.com/download)自行安装。BootAgent 只写入兼容配置，不下载也不代理 Claude Desktop。 +Kimi Code Desktop 在 macOS 和 Windows 上由 BootAgent 自动安装。运行安装程序前，BootAgent 会校验下载文件的摘要以及应用签名：macOS 校验 Developer ID 团队，Windows 校验 Authenticode 发布者。它与 Kimi Code CLI 共享 `~/.kimi-code/config.toml`，两个 Agent 读取的是同一份配置。 + DeepSeek Harness 可以使用自带的 DeepSeek 官方线路，也可以使用已配置的兼容 Provider。安装后它会打开本地 Web 应用，具体引导命令以其官方文档为准。 内置 Provider 包括 JieKou.AI、PPIO、Novita、DeepSeek 和 Moonshot；也可以在 Provider 页面添加自定义的 OpenAI 兼容或 Anthropic 兼容服务。 diff --git a/frontend/src/components/icons/agents.test.tsx b/frontend/src/components/icons/agents.test.tsx index d85a1798..d8525928 100644 --- a/frontend/src/components/icons/agents.test.tsx +++ b/frontend/src/components/icons/agents.test.tsx @@ -32,8 +32,9 @@ describe("AgentIcon", () => { // chatgpt-desktop is a desktop Agent rather than a CLI, and it reuses the // OpenAI mark because it is OpenAI's own product sharing Codex's config. // dsh-desktop reuses the DeepSeek mark for the same reason: it is - // DeepSeek's own desktop shell around the dsh CLI. - expect(assetIds.sort()).toEqual(["chatgpt-desktop", "claude-code", "claude-desktop", "codex", "dsh", "dsh-desktop", "hermes", "kilo-cli", "kimi-code", "openclaw", "opencode", "pi"]); + // DeepSeek's own desktop shell around the dsh CLI. kimi-code-desktop reuses the + // Kimi mark for the same reason: it is Moonshot's own app over the kimi-code CLI's config. + expect(assetIds.sort()).toEqual(["chatgpt-desktop", "claude-code", "claude-desktop", "codex", "dsh", "dsh-desktop", "hermes", "kilo-cli", "kimi-code", "kimi-code-desktop", "openclaw", "opencode", "pi"]); for (const id of assetIds) { const rights = agentMarkRights(id); expect(agentMarkKind(id)).toBe("asset"); @@ -124,6 +125,7 @@ describe("AgentIcon", () => { "kilo-cli": "0 0 24 24", hermes: "0 0 24 24", "kimi-code": "0 0 24 24", + "kimi-code-desktop": "0 0 24 24", pi: "0 0 24 24", openclaw: "0 0 120 120", }; @@ -208,7 +210,7 @@ describe("AgentIcon", () => { // Every Agent the app can show needs a mark it was actually given, so a new one // does not quietly land on the unknown-Agent fallback. it("registers a mark for every Agent and desktop Agent", () => { - for (const id of [...ALL, "chatgpt-desktop", "claude-desktop", "dsh-desktop", "workbuddy"]) { + for (const id of [...ALL, "chatgpt-desktop", "claude-desktop", "dsh-desktop", "kimi-code-desktop", "workbuddy"]) { expect(agentMarkKind(id), `${id} has no registered mark`).not.toBe("fallback"); } }); diff --git a/frontend/src/components/icons/agents.tsx b/frontend/src/components/icons/agents.tsx index 3d26efde..18f53c93 100644 --- a/frontend/src/components/icons/agents.tsx +++ b/frontend/src/components/icons/agents.tsx @@ -194,6 +194,16 @@ const MARKS: Record = { source: assetRightsManifest.assets["kimi-code"].source, rights: assetRightsManifest.assets["kimi-code"], }, + // Kimi Code Desktop is Moonshot's own app over the same ~/.kimi-code/config.toml + // the kimi-code CLI reads, so it carries the same Kimi product mark. Keyed by + // desktop Agent id because the desktop card looks itself up by id, as with + // dsh-desktop above -- without this entry the card fell back to the generic Bot glyph. + "kimi-code-desktop": { + kind: "asset", + markup: kimiCodeMark, + source: assetRightsManifest.assets["kimi-code"].source, + rights: assetRightsManifest.assets["kimi-code"], + }, }; /** One-line positioning shown on hover; never a restatement of the name. */ diff --git a/internal/app/desktopapp_test.go b/internal/app/desktopapp_test.go index 3cb010b2..35a91ae8 100644 --- a/internal/app/desktopapp_test.go +++ b/internal/app/desktopapp_test.go @@ -275,6 +275,54 @@ func TestConfigureDesktopAgentAcceptsAnyProfileWithAnAPIMode(t *testing.T) { } } +// Kimi Code Desktop shares the CLI's ~/.kimi-code/config.toml, so a Profile +// applied to it is the CLI Agent's activation: the same file, the same binding, +// and an Anthropic Profile is refused because the CLI writer speaks OpenAI. +func TestConfigureKimiCodeDesktopWritesTheSharedCLIConfig(t *testing.T) { + home := t.TempDir() + core := activationCore(t, home, provider.NewClient(nil), "macos") + if _, err := core.SaveProfile(context.Background(), SaveProfileOptions{ + ID: "kimi", Label: "Kimi", Provider: "ppio", APIKey: "kimi-secret", + Model: "model-a", ConfigMode: "provider", Protocol: "openai", + }); err != nil { + t.Fatal(err) + } + result, err := core.ConfigureDesktopAgent(context.Background(), desktopapp.KimiCodeDesktopID, "kimi") + if err != nil { + t.Fatal(err) + } + path := filepath.Join(home, ".kimi-code", "config.toml") + if result.Config != path || result.ProfileAgentID != "kimi-code" { + t.Fatalf("configure result = %#v", result) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"kimi-secret", `type = "openai"`, `default_model = "bootagent/model-a"`} { + if !strings.Contains(string(data), want) { + t.Fatalf("shared config missing %q: %s", want, data) + } + } + binding, err := core.profiles.ReadAgentBinding("kimi-code") + if err != nil || binding == nil || binding.ProfileRef != "kimi" { + t.Fatalf("CLI binding = %#v, err=%v", binding, err) + } + status, err := core.DesktopAgentStatus(context.Background(), desktopapp.KimiCodeDesktopID) + if err != nil || status.ConfigPath != path || status.ConfigSharedWith != "Kimi Code" || status.Protocol != provider.ProtocolOpenAI || status.ProfileID == nil || *status.ProfileID != "kimi" { + t.Fatalf("desktop status = %#v, err=%v", status, err) + } + + if _, err := core.SaveProfile(context.Background(), SaveProfileOptions{ + ID: "kimi-anthropic", Label: "Kimi", Provider: "ppio", Model: "model-a", ConfigMode: "provider", Protocol: "anthropic", + }); err != nil { + t.Fatal(err) + } + if _, err := core.ConfigureDesktopAgent(context.Background(), desktopapp.KimiCodeDesktopID, "kimi-anthropic"); err == nil { + t.Fatal("an Anthropic Profile was applied to the OpenAI-only shared config") + } +} + func TestConfigureWorkBuddyWritesModelsJSONFromProvider(t *testing.T) { home := t.TempDir() core := NewUseCases(StatusOptions{Home: home, Platform: platform.For("macos", "arm64")}) diff --git a/internal/app/testdata/status-empty-linux-arm64.json b/internal/app/testdata/status-empty-linux-arm64.json index 634fe519..9afc2245 100644 --- a/internal/app/testdata/status-empty-linux-arm64.json +++ b/internal/app/testdata/status-empty-linux-arm64.json @@ -625,6 +625,20 @@ "source": "unknown", "supported": false, "version": null + }, + { + "configPath": "${HOME}/.kimi-code/config.toml", + "configSharedWith": "Kimi Code", + "home": "https://www.kimi.com/code", + "id": "kimi-code-desktop", + "installed": false, + "name": "Kimi Code Desktop", + "profileAgentId": "kimi-code", + "profileId": null, + "protocol": "openai", + "source": "unknown", + "supported": false, + "version": null } ] } diff --git a/internal/desktopapp/kimi.go b/internal/desktopapp/kimi.go new file mode 100644 index 00000000..c28c7c73 --- /dev/null +++ b/internal/desktopapp/kimi.go @@ -0,0 +1,552 @@ +package desktopapp + +import ( + "context" + "crypto/sha512" + "crypto/subtle" + "encoding/base64" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + + "gopkg.in/yaml.v3" +) + +const ( + KimiCodeDesktopID = "kimi-code-desktop" + KimiCodeDesktopName = "Kimi Code Desktop" + KimiCodeDesktopHome = "https://www.kimi.com/code" + // KimiCodeAgentID is the CLI Agent whose ~/.kimi-code/config.toml the + // desktop app shares: the vendor documents the two as sharing accounts, + // models and providers under ~/.kimi-code. + KimiCodeAgentID = "kimi-code" + + // KimiCodeDesktopBundleID and KimiCodeDesktopTeamID are read off the signed + // 1.0.4 release: codesign reports Identifier=com.kimi.code.desktop and + // TeamIdentifier=2J9472RW75 under "Developer ID Application: Beijing Moonshot + // Technology Co., Ltd (2J9472RW75)", and spctl accepts it as notarized. The + // team is pinned; the authority's common name is left free, as for ZCode. + KimiCodeDesktopBundleID = "com.kimi.code.desktop" + KimiCodeDesktopTeamID = "2J9472RW75" + + // KimiCodeDesktopWindowsPublisher is the O and CN of the certificate that + // signs KimiCode-1.0.4-win-x64.exe (issuer Microsoft ID Verified CS EOC CA + // 04). It is not the Moonshot entity behind the macOS team: it was read out + // of the vendor's own published installer, and pinning it was a deliberate + // decision rather than an assumption that the two are the same company. If + // the vendor changes signer, installs stop with a publisher error instead of + // trusting the new one. + KimiCodeDesktopWindowsPublisher = "NOVASCENT PRIVATE LIMITED" + + // KimiCodeDesktopFeedBase is what the installed app's own + // Contents/Resources/app-update.yml points electron-updater at (generic + // provider, channel latest). The feed lists its files relative to it, and + // code.kimi.com answers both with a redirect to cdn.kimi.com. + KimiCodeDesktopFeedHost = "code.kimi.com" + KimiCodeDesktopCDNHost = "cdn.kimi.com" + KimiCodeDesktopFeedBase = "https://" + KimiCodeDesktopFeedHost + "/kimi-code/desktop/" + + kimiCodeDesktopAppName = "Kimi Code.app" + kimiCodeDesktopExeName = "Kimi Code.exe" +) + +// kimiCodeFeed is the electron-updater manifest. Only what BootAgent acts on is +// decoded. +type kimiCodeFeed struct { + Version string `yaml:"version"` + Files []kimiCodeFeedFile `yaml:"files"` +} + +type kimiCodeFeedFile struct { + URL string `yaml:"url"` + SHA512 string `yaml:"sha512"` + Size int64 `yaml:"size"` +} + +// kimiCodeTarget names the build for a platform, or "" when the vendor +// publishes none BootAgent can install. It decides both what Status.Supported +// reports and which file the install picks, so the UI cannot offer an install +// the lookup then refuses. Windows on ARM gets the x64 installer, the vendor's +// only Windows build. +func kimiCodeTarget(osID, arch string) string { + switch osID { + case "macos": + switch strings.ToLower(strings.TrimSpace(arch)) { + case "arm64", "aarch64": + return "mac-arm64" + case "x64", "amd64", "x86_64": + return "mac-x64" + } + case "windows": + return "win-x64" + } + return "" +} + +func kimiCodeFeedURL(osID, arch string) (string, error) { + switch target := kimiCodeTarget(osID, arch); { + case strings.HasPrefix(target, "mac-"): + return KimiCodeDesktopFeedBase + "latest-mac.yml", nil + case target == "win-x64": + return KimiCodeDesktopFeedBase + "latest.yml", nil + } + if osID == "macos" || osID == "windows" { + return "", fmt.Errorf("%s has no package for %s/%s", KimiCodeDesktopName, osID, arch) + } + return "", fmt.Errorf("%s is not supported on %s", KimiCodeDesktopName, osID) +} + +func baseKimiCodeStatus(osID, arch string) Status { + status := Status{ID: KimiCodeDesktopID, Name: KimiCodeDesktopName, Source: SourceUnknown} + if kimiCodeTarget(osID, arch) == "" { + return status + } + switch osID { + case "macos": + status.Supported, status.Source = true, SourceMacOSZIP + case "windows": + status.Supported, status.Source = true, SourceWindowsInstaller + } + return status +} + +func inspectKimiCode(ctx context.Context, options Options) Status { + status := baseKimiCodeStatus(options.Platform.OS, options.Platform.Arch) + if err := contextError(ctx); err != nil { + status.InspectionUnavailable = nonEmptyPointer(err.Error()) + return status + } + if !status.Supported { + return status + } + var found Status + var err error + switch options.Platform.OS { + case "macos": + found, err = inspectKimiCodeMacOS(ctx, options) + case "windows": + found, err = inspectKimiCodeWindows(options) + } + if err != nil { + found.InspectionUnavailable = nonEmptyPointer(err.Error()) + } + return found +} + +func inspectKimiCodeMacOS(ctx context.Context, options Options) (Status, error) { + status := baseKimiCodeStatus("macos", options.Platform.Arch) + roots := options.SearchRoots + if len(roots) == 0 { + roots = []string{"/Applications"} + if options.Home != "" { + roots = append(roots, filepath.Join(options.Home, "Applications")) + } + } + var lastErr error + for _, root := range roots { + candidate := root + if !strings.EqualFold(filepath.Ext(root), ".app") { + candidate = filepath.Join(root, kimiCodeDesktopAppName) + } + info, err := os.Stat(candidate) + if os.IsNotExist(err) { + continue + } + if err != nil { + lastErr = err + continue + } + if !info.IsDir() { + continue + } + metadata, err := readMacMetadata(ctx, options, candidate) + if err != nil { + lastErr = err + continue + } + // Read from the plist, so a renamed .app cannot pass as Kimi Code. + if metadata.bundleID != KimiCodeDesktopBundleID { + continue + } + status.Installed, status.Path, status.Version = true, candidate, metadata.version + return status, nil + } + return status, lastErr +} + +func inspectKimiCodeWindows(options Options) (Status, error) { + status := baseKimiCodeStatus("windows", options.Platform.Arch) + for _, candidate := range kimiCodeWindowsCandidates(options) { + info, err := os.Stat(candidate) + if os.IsNotExist(err) { + continue + } + if err != nil { + return status, err + } + if info.IsDir() { + continue + } + status.Installed, status.Path = true, candidate + return status, nil + } + return status, nil +} + +// kimiCodeWindowsCandidates are where an electron-builder NSIS installer puts +// the app: per-user under %LOCALAPPDATA%\Programs by default, machine-wide +// under Program Files when the user picks that. The executable name is +// verified -- the installer's app-64.7z carries "Kimi Code.exe" -- but the +// directory is electron-builder's default for that product name, not an +// observed installation: the NSIS script is compressed and this machine is +// macOS. +func kimiCodeWindowsCandidates(options Options) []string { + if len(options.SearchRoots) > 0 { + candidates := make([]string, 0, len(options.SearchRoots)) + for _, root := range options.SearchRoots { + if strings.EqualFold(filepath.Ext(root), ".exe") { + candidates = append(candidates, root) + continue + } + candidates = append(candidates, filepath.Join(root, kimiCodeDesktopExeName)) + } + return candidates + } + candidates := make([]string, 0, 2) + if options.Home != "" { + candidates = append(candidates, filepath.Join(options.Home, "AppData", "Local", "Programs", "Kimi Code", kimiCodeDesktopExeName)) + } + return append(candidates, filepath.Join("C:\\", "Program Files", "Kimi Code", kimiCodeDesktopExeName)) +} + +// fetchKimiCodeFeed reads the manifest with no-cache: it is the rolling pointer +// to the current release, and a CDN in front of it would otherwise be free to +// pin BootAgent to a stale one. +func fetchKimiCodeFeed(ctx context.Context, options Options) (kimiCodeFeed, string, error) { + endpoint, err := kimiCodeFeedURL(options.Platform.OS, options.Platform.Arch) + if err != nil { + return kimiCodeFeed{}, "", err + } + requestCtx, cancel := context.WithTimeout(ctx, installTimeout) + defer cancel() + request, err := http.NewRequestWithContext(requestCtx, http.MethodGet, endpoint, nil) + if err != nil { + return kimiCodeFeed{}, "", err + } + request.Header.Set("Cache-Control", "no-cache") + client := options.Downloader + if client == nil { + client = http.DefaultClient + } + response, err := client.Do(request) + if err != nil { + return kimiCodeFeed{}, "", err + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return kimiCodeFeed{}, "", fmt.Errorf("%s update request returned HTTP %d", KimiCodeDesktopName, response.StatusCode) + } + var feed kimiCodeFeed + if err := yaml.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&feed); err != nil { + return kimiCodeFeed{}, "", fmt.Errorf("decode %s update response: %w", KimiCodeDesktopName, err) + } + return feed, endpoint, nil +} + +// kimiCodeArtifact picks this platform's file from the feed and resolves its +// URL. The feed lists files relative to itself, as electron-updater's generic +// provider allows; they are resolved against the feed URL BootAgent asked for, +// never against wherever a redirect landed, and the result still has to be on +// the vendor's hosts. +// +// On macOS this is the .zip, the file electron-updater itself consumes and the +// one the feed's top-level path names. +func kimiCodeArtifact(feed kimiCodeFeed, feedURL, osID, arch string) (kimiCodeFeedFile, error) { + target := kimiCodeTarget(osID, arch) + if target == "" { + return kimiCodeFeedFile{}, fmt.Errorf("%s has no package for %s/%s", KimiCodeDesktopName, osID, arch) + } + wanted := ".exe" + if osID == "macos" { + wanted = ".zip" + } + base, err := url.Parse(feedURL) + if err != nil { + return kimiCodeFeedFile{}, fmt.Errorf("parse %s feed URL: %w", KimiCodeDesktopName, err) + } + for _, file := range feed.Files { + reference, err := url.Parse(strings.TrimSpace(file.URL)) + if err != nil { + continue + } + name := strings.ToLower(filepath.Base(reference.Path)) + if !strings.HasSuffix(name, "-"+target+wanted) { + continue + } + approved, err := approvedDownloadURL(base.ResolveReference(reference).String(), KimiCodeDesktopFeedHost, KimiCodeDesktopCDNHost) + if err != nil { + return kimiCodeFeedFile{}, fmt.Errorf("validate %s package URL: %w", KimiCodeDesktopName, err) + } + if strings.TrimSpace(file.SHA512) == "" { + return kimiCodeFeedFile{}, fmt.Errorf("%s update feed lists no digest for %s", KimiCodeDesktopName, target+wanted) + } + file.URL = approved + return file, nil + } + return kimiCodeFeedFile{}, fmt.Errorf("%s update feed lists no %s package", KimiCodeDesktopName, target+wanted) +} + +// kimiCodePackage resolves what to download. DownloadURL is the test and +// self-hosting seam the other agents use; it still has to pass the host +// allowlist, and it carries no digest, so it verifies by signature alone. +func kimiCodePackage(ctx context.Context, options Options) (kimiCodeFeed, kimiCodeFeedFile, error) { + if raw := strings.TrimSpace(options.DownloadURL); raw != "" { + approved, err := approvedDownloadURL(raw, KimiCodeDesktopFeedHost, KimiCodeDesktopCDNHost) + return kimiCodeFeed{}, kimiCodeFeedFile{URL: approved}, err + } + feed, feedURL, err := fetchKimiCodeFeed(ctx, options) + if err != nil { + return kimiCodeFeed{}, kimiCodeFeedFile{}, err + } + artifact, err := kimiCodeArtifact(feed, feedURL, options.Platform.OS, options.Platform.Arch) + if err != nil { + return kimiCodeFeed{}, kimiCodeFeedFile{}, err + } + return feed, artifact, nil +} + +// verifyKimiCodeDigest checks the downloaded bytes against the feed. The digest +// is what authenticates the archive: the host allowlist only says who was +// asked. Size first, so a truncated transfer reports the useful error. +func verifyKimiCodeDigest(path string, expected kimiCodeFeedFile) error { + want, err := base64.StdEncoding.DecodeString(strings.TrimSpace(expected.SHA512)) + if err != nil { + return fmt.Errorf("decode expected %s digest: %w", KimiCodeDesktopName, err) + } + if len(want) != sha512.Size { + return fmt.Errorf("expected %s digest is not a SHA-512 value", KimiCodeDesktopName) + } + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + info, err := file.Stat() + if err != nil { + return err + } + if expected.Size > 0 && info.Size() != expected.Size { + return fmt.Errorf("downloaded %s package is %d bytes, expected %d", KimiCodeDesktopName, info.Size(), expected.Size) + } + digest := sha512.New() + if _, err := io.Copy(digest, file); err != nil { + return err + } + if subtle.ConstantTimeCompare(digest.Sum(nil), want) != 1 { + return fmt.Errorf("downloaded %s package failed its SHA-512 check", KimiCodeDesktopName) + } + return nil +} + +func installKimiCode(ctx context.Context, options Options) (ActionResult, error) { + if err := contextError(ctx); err != nil { + return ActionResult{}, err + } + status := inspectKimiCode(ctx, options) + if status.Installed { + return ActionResult{Status: "already-installed", Message: KimiCodeDesktopName + " is already installed", App: status}, nil + } + if !status.Supported { + return ActionResult{}, fmt.Errorf("%s is not supported on %s/%s", KimiCodeDesktopName, options.Platform.OS, options.Platform.Arch) + } + switch options.Platform.OS { + case "macos": + return installKimiCodeMacOS(ctx, options) + case "windows": + return installKimiCodeWindows(ctx, options) + } + return ActionResult{}, fmt.Errorf("%s is not supported on %s", KimiCodeDesktopName, options.Platform.OS) +} + +func installKimiCodeMacOS(ctx context.Context, options Options) (ActionResult, error) { + feed, artifact, err := kimiCodePackage(ctx, options) + if err != nil { + return ActionResult{}, err + } + tempDir, err := os.MkdirTemp("", "bootagent-kimi-code-") + if err != nil { + return ActionResult{}, fmt.Errorf("create temporary %s installer directory: %w", KimiCodeDesktopName, err) + } + defer os.RemoveAll(tempDir) + archive := filepath.Join(tempDir, "KimiCode.zip") + if err := downloadFile(ctx, options, artifact.URL, archive, KimiCodeDesktopID); err != nil { + return ActionResult{}, fmt.Errorf("download %s installer: %w", KimiCodeDesktopName, err) + } + if artifact.SHA512 != "" { + if err := verifyKimiCodeDigest(archive, artifact); err != nil { + return ActionResult{}, err + } + } + extracted := filepath.Join(tempDir, "extracted") + if err := os.MkdirAll(extracted, 0o700); err != nil { + return ActionResult{}, err + } + result, err := run(options, ctx, []string{"/usr/bin/ditto", "-x", "-k", archive, extracted}, installTimeout) + if err != nil { + return ActionResult{}, fmt.Errorf("extract %s installer: %w", KimiCodeDesktopName, err) + } + if result.ExitCode != 0 { + return ActionResult{}, commandFailure("extract "+KimiCodeDesktopName+" installer", result) + } + appPath, err := findKimiCodeApp(extracted) + if err != nil { + return ActionResult{}, err + } + metadata, err := readMacMetadata(ctx, options, appPath) + if err != nil { + return ActionResult{}, fmt.Errorf("inspect downloaded %s app: %w", KimiCodeDesktopName, err) + } + if metadata.bundleID != KimiCodeDesktopBundleID { + return ActionResult{}, fmt.Errorf("downloaded app has unexpected bundle identifier %q", metadata.bundleID) + } + if err := verifyKimiCodeMacOSApp(ctx, options, appPath); err != nil { + return ActionResult{}, fmt.Errorf("verify downloaded %s app: %w", KimiCodeDesktopName, err) + } + var lastErr error + for _, destination := range kimiCodeDestinations(options) { + if err := os.MkdirAll(filepath.Dir(destination), 0o755); err != nil { + lastErr = err + continue + } + if _, err := os.Stat(destination); err == nil { + lastErr = fmt.Errorf("destination already exists: %s", destination) + continue + } else if !os.IsNotExist(err) { + lastErr = err + continue + } + copied, copyErr := run(options, ctx, []string{"/usr/bin/ditto", appPath, destination}, installTimeout) + if copyErr != nil { + lastErr = copyErr + continue + } + if copied.ExitCode != 0 { + lastErr = commandFailure("copy "+KimiCodeDesktopName+" app", copied) + continue + } + installed := baseKimiCodeStatus("macos", options.Platform.Arch) + installed.Installed, installed.Path = true, destination + installed.Version = metadata.version + if installed.Version == nil { + installed.Version = nonEmptyPointer(feed.Version) + } + return ActionResult{Status: "installed", Message: KimiCodeDesktopName + " was installed", RefreshNeeded: true, App: installed}, nil + } + if lastErr == nil { + lastErr = errors.New("no writable macOS Applications directory") + } + return ActionResult{}, lastErr +} + +// installKimiCodeWindows verifies the digest and Authenticode, then starts the +// vendor's NSIS installer, which owns its own placement and shortcuts. +func installKimiCodeWindows(ctx context.Context, options Options) (ActionResult, error) { + feed, artifact, err := kimiCodePackage(ctx, options) + if err != nil { + return ActionResult{}, err + } + installer, err := os.CreateTemp("", "bootagent-kimi-code-*.exe") + if err != nil { + return ActionResult{}, fmt.Errorf("create temporary %s installer: %w", KimiCodeDesktopName, err) + } + installerPath := installer.Name() + if err := installer.Close(); err != nil { + _ = os.Remove(installerPath) + return ActionResult{}, err + } + keep := false + defer func() { + if !keep { + _ = os.Remove(installerPath) + } + }() + if err := downloadFile(ctx, options, artifact.URL, installerPath, KimiCodeDesktopID); err != nil { + return ActionResult{}, fmt.Errorf("download %s installer: %w", KimiCodeDesktopName, err) + } + if artifact.SHA512 != "" { + if err := verifyKimiCodeDigest(installerPath, artifact); err != nil { + return ActionResult{}, err + } + } + if err := contextError(ctx); err != nil { + return ActionResult{}, err + } + if err := verifyKimiCodeWindowsInstaller(ctx, options, installerPath); err != nil { + return ActionResult{}, fmt.Errorf("verify downloaded %s installer with Authenticode: %w", KimiCodeDesktopName, err) + } + if err := start(options, []string{installerPath}); err != nil { + return ActionResult{}, fmt.Errorf("start %s installer: %w", KimiCodeDesktopName, err) + } + keep = true + status := baseKimiCodeStatus("windows", options.Platform.Arch) + status.Version = nonEmptyPointer(feed.Version) + return ActionResult{Status: "installer-started", Message: "The downloaded " + KimiCodeDesktopName + " installer was started", RefreshNeeded: true, App: status}, nil +} + +func kimiCodeDestinations(options Options) []string { + dirs := options.ApplicationDirs + if len(dirs) == 0 { + dirs = []string{"/Applications"} + if options.Home != "" { + dirs = append(dirs, filepath.Join(options.Home, "Applications")) + } + } + result := make([]string, 0, len(dirs)) + for _, dir := range dirs { + result = append(result, filepath.Join(dir, kimiCodeDesktopAppName)) + } + return result +} + +func findKimiCodeApp(root string) (string, error) { + var found string + err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() && strings.EqualFold(filepath.Base(path), kimiCodeDesktopAppName) { + found = path + return filepath.SkipAll + } + return nil + }) + if err != nil { + return "", err + } + if found == "" { + return "", fmt.Errorf("downloaded %s archive contains no %s", KimiCodeDesktopName, kimiCodeDesktopAppName) + } + return found, nil +} + +func openKimiCode(ctx context.Context, options Options) error { + status := inspectKimiCode(ctx, options) + if !status.Installed { + if status.InspectionUnavailable != nil { + return errors.New(*status.InspectionUnavailable) + } + return errors.New(KimiCodeDesktopName + " is not installed") + } + switch options.Platform.OS { + case "macos": + return start(options, []string{"/usr/bin/open", "-a", status.Path}) + case "windows": + return start(options, []string{status.Path}) + } + return fmt.Errorf("%s is not supported on %s", KimiCodeDesktopName, options.Platform.OS) +} diff --git a/internal/desktopapp/kimi_test.go b/internal/desktopapp/kimi_test.go new file mode 100644 index 00000000..05af93ba --- /dev/null +++ b/internal/desktopapp/kimi_test.go @@ -0,0 +1,305 @@ +package desktopapp + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/MaimoryLab/BootAgent/internal/platform" + "github.com/MaimoryLab/BootAgent/internal/process" +) + +// kimiCodeRunner replays scripted results and creates the extracted .app when +// it sees the unpacking ditto call, which the install then walks for. +type kimiCodeRunner struct { + results []process.Result + calls [][]string + started [][]string + t *testing.T +} + +func (r *kimiCodeRunner) LookPath(string) (string, bool) { return "", false } + +func (r *kimiCodeRunner) Run(_ context.Context, argv []string, _ map[string]string, _ time.Duration) (process.Result, error) { + r.calls = append(r.calls, append([]string(nil), argv...)) + if len(argv) >= 5 && argv[0] == "/usr/bin/ditto" && argv[1] == "-x" { + if err := os.MkdirAll(filepath.Join(argv[4], kimiCodeDesktopAppName, "Contents"), 0o755); err != nil { + r.t.Fatal(err) + } + } + if len(r.results) == 0 { + return process.Result{Args: argv, ExitCode: 0}, nil + } + result := r.results[0] + r.results = r.results[1:] + result.Args = argv + return result, nil +} + +func (r *kimiCodeRunner) Start(argv []string, _ map[string]string) error { + r.started = append(r.started, append([]string(nil), argv...)) + return nil +} + +// kimiCodeMacFeed is shaped like the vendor's latest-mac.yml: relative URLs, +// both architectures, a zip and a dmg for each. +func kimiCodeMacFeed(armZipDigest string, armZipSize int64) []byte { + return fmt.Appendf(nil, `version: 1.0.4 +files: + - url: binaries/1.0.4/KimiCode-1.0.4-mac-arm64.zip + sha512: %s + size: %d + - url: binaries/1.0.4/KimiCode-1.0.4-mac-arm64.dmg + sha512: ZG1n + size: 3 + - url: binaries/1.0.4/KimiCode-1.0.4-mac-x64.zip + sha512: eDY0 + size: 3 + - url: binaries/1.0.4/KimiCode-1.0.4-mac-x64.dmg + sha512: ZG1n + size: 3 +path: binaries/1.0.4/KimiCode-1.0.4-mac-arm64.zip +`, armZipDigest, armZipSize) +} + +func TestKimiCodeTargetAndFeedPerPlatform(t *testing.T) { + tests := []struct { + osID, arch, target, feed string + }{ + {"macos", "arm64", "mac-arm64", KimiCodeDesktopFeedBase + "latest-mac.yml"}, + {"macos", "x86_64", "mac-x64", KimiCodeDesktopFeedBase + "latest-mac.yml"}, + {"windows", "x64", "win-x64", KimiCodeDesktopFeedBase + "latest.yml"}, + // The vendor ships no ARM Windows build; x64 is what such a machine runs. + {"windows", "arm64", "win-x64", KimiCodeDesktopFeedBase + "latest.yml"}, + } + for _, test := range tests { + if got := kimiCodeTarget(test.osID, test.arch); got != test.target { + t.Fatalf("kimiCodeTarget(%q, %q) = %q, want %q", test.osID, test.arch, got, test.target) + } + if got, err := kimiCodeFeedURL(test.osID, test.arch); err != nil || got != test.feed { + t.Fatalf("kimiCodeFeedURL(%q, %q) = %q, %v", test.osID, test.arch, got, err) + } + } + if _, err := kimiCodeFeedURL("linux", "x64"); err == nil { + t.Fatal("kimiCodeFeedURL() accepted linux, which the vendor does not ship") + } + if status := Inspect(context.Background(), KimiCodeDesktopID, Options{Platform: platform.For("linux", "amd64")}); status.Supported { + t.Fatalf("linux status = %#v, want unsupported", status) + } +} + +// The feed's URLs are relative. They resolve against the feed URL and must +// stay on the vendor's hosts; each architecture gets its own zip, never the +// dmg listed beside it. +func TestKimiCodeArtifactResolvesRelativeURLsPerArchitecture(t *testing.T) { + feed := kimiCodeFeed{Files: []kimiCodeFeedFile{ + {URL: "binaries/1.0.4/KimiCode-1.0.4-mac-arm64.dmg", SHA512: "ZG1n"}, + {URL: "binaries/1.0.4/KimiCode-1.0.4-mac-arm64.zip", SHA512: "YXJt"}, + {URL: "binaries/1.0.4/KimiCode-1.0.4-mac-x64.zip", SHA512: "eDY0"}, + }} + feedURL := KimiCodeDesktopFeedBase + "latest-mac.yml" + arm, err := kimiCodeArtifact(feed, feedURL, "macos", "arm64") + if err != nil || arm.URL != KimiCodeDesktopFeedBase+"binaries/1.0.4/KimiCode-1.0.4-mac-arm64.zip" || arm.SHA512 != "YXJt" { + t.Fatalf("arm64 artifact = %#v, %v", arm, err) + } + intel, err := kimiCodeArtifact(feed, feedURL, "macos", "x64") + if err != nil || !strings.HasSuffix(intel.URL, "-mac-x64.zip") { + t.Fatalf("x64 artifact = %#v, %v", intel, err) + } + if _, err := kimiCodeArtifact(kimiCodeFeed{Files: feed.Files[:1]}, feedURL, "macos", "arm64"); err == nil { + t.Fatal("kimiCodeArtifact() fell back to the dmg when no zip was listed") + } +} + +func TestKimiCodeArtifactRejectsOffHostAndMissingDigest(t *testing.T) { + feedURL := KimiCodeDesktopFeedBase + "latest.yml" + for _, url := range []string{"https://example.test/KimiCode-1.0.4-win-x64.exe", "//example.test/KimiCode-1.0.4-win-x64.exe", "http://code.kimi.com/KimiCode-1.0.4-win-x64.exe"} { + feed := kimiCodeFeed{Files: []kimiCodeFeedFile{{URL: url, SHA512: "ZXhl"}}} + if _, err := kimiCodeArtifact(feed, feedURL, "windows", "x64"); err == nil { + t.Fatalf("kimiCodeArtifact() accepted %q", url) + } + } + noDigest := kimiCodeFeed{Files: []kimiCodeFeedFile{{URL: "binaries/1.0.4/KimiCode-1.0.4-win-x64.exe"}}} + if _, err := kimiCodeArtifact(noDigest, feedURL, "windows", "x64"); err == nil || !strings.Contains(err.Error(), "no digest") { + t.Fatalf("kimiCodeArtifact() error = %v", err) + } +} + +func TestKimiCodeMacOSInstallVerifiesDigestBundleIDAndSignature(t *testing.T) { + payload := []byte("Kimi Code macOS archive") + feedURL := KimiCodeDesktopFeedBase + "latest-mac.yml" + zipURL := KimiCodeDesktopFeedBase + "binaries/1.0.4/KimiCode-1.0.4-mac-arm64.zip" + downloader := &routeDownloader{routes: map[string][]byte{ + feedURL: kimiCodeMacFeed(zcodeDigest(payload), int64(len(payload))), + zipURL: payload, + }} + applications := t.TempDir() + runner := &kimiCodeRunner{t: t, results: []process.Result{ + {ExitCode: 0}, // ditto -x -k + {ExitCode: 0, Stdout: "com.kimi.code.desktop\n"}, // plutil CFBundleIdentifier + {ExitCode: 0, Stdout: "1.0.4\n"}, // plutil CFBundleShortVersionString + {ExitCode: 0}, // codesign --verify + {ExitCode: 0, Stdout: "Identifier=com.kimi.code.desktop\nTeamIdentifier=2J9472RW75\nAuthority=Developer ID Application: Beijing Moonshot Technology Co., Ltd (2J9472RW75)\n"}, + {ExitCode: 0, Stdout: "source=Notarized Developer ID\n"}, // spctl + {ExitCode: 0}, // ditto copy + }} + result, err := Install(context.Background(), KimiCodeDesktopID, Options{ + Home: t.TempDir(), Platform: platform.For("macos", "arm64"), Runner: runner, Downloader: downloader, + SearchRoots: []string{t.TempDir()}, ApplicationDirs: []string{applications}, + }) + if err != nil { + t.Fatal(err) + } + if result.Status != "installed" || result.App.Path != filepath.Join(applications, kimiCodeDesktopAppName) || result.App.Version == nil || *result.App.Version != "1.0.4" { + t.Fatalf("Install() = %#v", result) + } + if len(downloader.hits) != 2 || downloader.hits[0] != feedURL || downloader.hits[1] != zipURL { + t.Fatalf("downloads = %#v", downloader.hits) + } +} + +// A different team must stop the install even though the digest and bundle +// identifier matched: the feed proves integrity, the signature proves origin. +func TestKimiCodeMacOSInstallRejectsAnotherTeam(t *testing.T) { + payload := []byte("Kimi Code macOS archive") + feedURL := KimiCodeDesktopFeedBase + "latest-mac.yml" + zipURL := KimiCodeDesktopFeedBase + "binaries/1.0.4/KimiCode-1.0.4-mac-arm64.zip" + downloader := &routeDownloader{routes: map[string][]byte{ + feedURL: kimiCodeMacFeed(zcodeDigest(payload), int64(len(payload))), + zipURL: payload, + }} + applications := t.TempDir() + runner := &kimiCodeRunner{t: t, results: []process.Result{ + {ExitCode: 0}, + {ExitCode: 0, Stdout: "com.kimi.code.desktop\n"}, + {ExitCode: 0, Stdout: "1.0.4\n"}, + {ExitCode: 0}, + {ExitCode: 0, Stdout: "Identifier=com.kimi.code.desktop\nTeamIdentifier=ABCDE12345\nAuthority=Developer ID Application: Someone Else (ABCDE12345)\n"}, + }} + _, err := Install(context.Background(), KimiCodeDesktopID, Options{ + Home: t.TempDir(), Platform: platform.For("macos", "arm64"), Runner: runner, Downloader: downloader, + SearchRoots: []string{t.TempDir()}, ApplicationDirs: []string{applications}, + }) + if err == nil || !strings.Contains(err.Error(), "TeamIdentifier=2J9472RW75") { + t.Fatalf("Install() error = %v", err) + } + if _, statErr := os.Stat(filepath.Join(applications, kimiCodeDesktopAppName)); !os.IsNotExist(statErr) { + t.Fatalf("an app from another team was copied into Applications: %v", statErr) + } +} + +func TestKimiCodeMacOSInstallStopsBeforeExtractingOnDigestMismatch(t *testing.T) { + feedURL := KimiCodeDesktopFeedBase + "latest-mac.yml" + zipURL := KimiCodeDesktopFeedBase + "binaries/1.0.4/KimiCode-1.0.4-mac-arm64.zip" + downloader := &routeDownloader{routes: map[string][]byte{ + feedURL: kimiCodeMacFeed(zcodeDigest([]byte("expected")), 8), + zipURL: []byte("replaced"), + }} + runner := &kimiCodeRunner{t: t} + _, err := Install(context.Background(), KimiCodeDesktopID, Options{ + Home: t.TempDir(), Platform: platform.For("macos", "arm64"), Runner: runner, Downloader: downloader, + SearchRoots: []string{t.TempDir()}, ApplicationDirs: []string{t.TempDir()}, + }) + if err == nil || !strings.Contains(err.Error(), "SHA-512") { + t.Fatalf("Install() error = %v", err) + } + if len(runner.calls) != 0 { + t.Fatalf("install ran commands after a digest mismatch: %#v", runner.calls) + } +} + +func kimiCodeWindowsFeed(digest string, size int) []byte { + return fmt.Appendf(nil, "version: 1.0.4\nfiles:\n - url: binaries/1.0.4/KimiCode-1.0.4-win-x64.exe\n sha512: %s\n size: %d\npath: binaries/1.0.4/KimiCode-1.0.4-win-x64.exe\n", digest, size) +} + +func TestKimiCodeWindowsInstallVerifiesAuthenticodeBeforeStarting(t *testing.T) { + payload := []byte("Kimi Code Windows installer") + feedURL := KimiCodeDesktopFeedBase + "latest.yml" + exeURL := KimiCodeDesktopFeedBase + "binaries/1.0.4/KimiCode-1.0.4-win-x64.exe" + downloader := &routeDownloader{routes: map[string][]byte{feedURL: kimiCodeWindowsFeed(zcodeDigest(payload), len(payload)), exeURL: payload}} + runner := &kimiCodeRunner{t: t, results: []process.Result{ + {ExitCode: 0, Stdout: `{"Status":"Valid","StatusMessage":"Signature verified.","Publisher":"NOVASCENT PRIVATE LIMITED","Subject":"CN=NOVASCENT PRIVATE LIMITED, O=NOVASCENT PRIVATE LIMITED, L=Singapore, C=SG","Issuer":"CN=Microsoft ID Verified CS EOC CA 04, O=Microsoft Corporation, C=US"}`}, + }} + result, err := Install(context.Background(), KimiCodeDesktopID, Options{ + Home: t.TempDir(), Platform: platform.For("windows", "x64"), Runner: runner, Downloader: downloader, + SearchRoots: []string{t.TempDir()}, + }) + if err != nil { + t.Fatal(err) + } + if result.Status != "installer-started" || len(runner.started) != 1 || result.App.Version == nil || *result.App.Version != "1.0.4" { + t.Fatalf("Install() = %#v started=%#v", result, runner.started) + } + // The install leaves the installer on disk for Windows to run. + if err := os.Remove(runner.started[0][0]); err != nil { + t.Fatal(err) + } +} + +// Moonshot's own name is not what signs the Windows build, so accepting it +// would be accepting a signer the vendor does not use; only the pinned +// publisher passes. +func TestKimiCodeWindowsInstallRejectsAnyOtherPublisher(t *testing.T) { + payload := []byte("Kimi Code Windows installer") + feedURL := KimiCodeDesktopFeedBase + "latest.yml" + exeURL := KimiCodeDesktopFeedBase + "binaries/1.0.4/KimiCode-1.0.4-win-x64.exe" + for _, publisher := range []string{"Beijing Moonshot Technology Co., Ltd", "Someone Else"} { + downloader := &routeDownloader{routes: map[string][]byte{feedURL: kimiCodeWindowsFeed(zcodeDigest(payload), len(payload)), exeURL: payload}} + runner := &kimiCodeRunner{t: t, results: []process.Result{ + {ExitCode: 0, Stdout: fmt.Sprintf(`{"Status":"Valid","StatusMessage":"Signature verified.","Publisher":%q,"Subject":"CN=%s, O=%s","Issuer":"CN=Trusted CA"}`, publisher, publisher, publisher)}, + }} + _, err := Install(context.Background(), KimiCodeDesktopID, Options{ + Home: t.TempDir(), Platform: platform.For("windows", "x64"), Runner: runner, Downloader: downloader, + SearchRoots: []string{t.TempDir()}, + }) + if err == nil || !strings.Contains(err.Error(), "not approved") { + t.Fatalf("publisher %q: Install() error = %v", publisher, err) + } + if len(runner.started) != 0 { + t.Fatalf("publisher %q: installer started: %#v", publisher, runner.started) + } + } +} + +func TestKimiCodeInspectRequiresTheBundleIdentifier(t *testing.T) { + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, kimiCodeDesktopAppName, "Contents"), 0o755); err != nil { + t.Fatal(err) + } + options := func(bundleID string) Options { + return Options{Platform: platform.For("macos", "arm64"), SearchRoots: []string{root}, Runner: &kimiCodeRunner{t: t, results: []process.Result{ + {ExitCode: 0, Stdout: bundleID + "\n"}, + {ExitCode: 0, Stdout: "1.0.4\n"}, + }}} + } + if status := Inspect(context.Background(), KimiCodeDesktopID, options("com.example.renamed")); status.Installed { + t.Fatalf("a renamed app passed as Kimi Code: %#v", status) + } + status := Inspect(context.Background(), KimiCodeDesktopID, options(KimiCodeDesktopBundleID)) + if !status.Installed || status.Version == nil || *status.Version != "1.0.4" || status.Path != filepath.Join(root, kimiCodeDesktopAppName) { + t.Fatalf("Inspect() = %#v", status) + } +} + +func TestKimiCodeWindowsInspectAndOpen(t *testing.T) { + root := t.TempDir() + exe := filepath.Join(root, kimiCodeDesktopExeName) + if err := os.WriteFile(exe, []byte("exe"), 0o600); err != nil { + t.Fatal(err) + } + runner := &kimiCodeRunner{t: t} + options := Options{Platform: platform.For("windows", "x64"), SearchRoots: []string{root}, Runner: runner} + if status := Inspect(context.Background(), KimiCodeDesktopID, options); !status.Installed || status.Path != exe { + t.Fatalf("Inspect() = %#v", status) + } + if err := Open(context.Background(), KimiCodeDesktopID, options); err != nil { + t.Fatal(err) + } + if len(runner.started) != 1 || runner.started[0][0] != exe { + t.Fatalf("started = %#v", runner.started) + } +} diff --git a/internal/desktopapp/macos_verify.go b/internal/desktopapp/macos_verify.go index 4ef0ad3b..627591b3 100644 --- a/internal/desktopapp/macos_verify.go +++ b/internal/desktopapp/macos_verify.go @@ -39,6 +39,12 @@ func verifyDSHMacOSApp(ctx context.Context, options Options, appPath string) err return verifyMacOSIdentity(ctx, options, appPath, DSHDesktopBundleID, DSHDesktopTeamID, "") } +// verifyKimiCodeMacOSApp pins Moonshot's Developer ID team and the bundle +// identifier, and requires notarization, like ZCode. +func verifyKimiCodeMacOSApp(ctx context.Context, options Options, appPath string) error { + return verifyMacOSIdentity(ctx, options, appPath, KimiCodeDesktopBundleID, KimiCodeDesktopTeamID, "") +} + func verifyMacOSIdentity(ctx context.Context, options Options, appPath, bundleID, teamID, authority string) error { result, err := run(options, ctx, []string{"/usr/bin/codesign", "--verify", "--deep", "--strict", "--verbose=2", appPath}, installTimeout) if err != nil { diff --git a/internal/desktopapp/registry.go b/internal/desktopapp/registry.go index 3a0f7411..d7f690ee 100644 --- a/internal/desktopapp/registry.go +++ b/internal/desktopapp/registry.go @@ -143,6 +143,22 @@ var implementations = []implementation{ install: installZCode, open: openZCode, }, + { + // Shares ~/.kimi-code/config.toml with the Kimi Code CLI, as ChatGPT + // Desktop shares Codex's: a Profile applied here is applied to the CLI + // Agent, whose writer owns that file. + Definition: Definition{ + ID: KimiCodeDesktopID, + Name: KimiCodeDesktopName, + ProfileAgentID: KimiCodeAgentID, + SharedConfigAgentID: KimiCodeAgentID, + ConfigAdapter: KimiCodeAgentID, + Home: KimiCodeDesktopHome, + }, + inspect: inspectKimiCode, + install: installKimiCode, + open: openKimiCode, + }, } func Definitions() []Definition { diff --git a/internal/desktopapp/windows_verify.go b/internal/desktopapp/windows_verify.go index 1d1b54be..dddd02fc 100644 --- a/internal/desktopapp/windows_verify.go +++ b/internal/desktopapp/windows_verify.go @@ -48,6 +48,13 @@ func verifyZCodeWindowsInstaller(ctx context.Context, options Options, installer return verifyWindowsInstallerPublisher(ctx, options, installerPath, []string{"北京智谱华章科技股份有限公司"}) } +// verifyKimiCodeWindowsInstaller pins the publisher read out of the vendor's +// published installer; see KimiCodeDesktopWindowsPublisher for why it differs +// from the macOS team. +func verifyKimiCodeWindowsInstaller(ctx context.Context, options Options, installerPath string) error { + return verifyWindowsInstallerPublisher(ctx, options, installerPath, []string{KimiCodeDesktopWindowsPublisher}) +} + func verifyWindowsInstallerPublisher(ctx context.Context, options Options, installerPath string, allowed []string) error { if strings.TrimSpace(installerPath) == "" { return errors.New("Windows installer path is empty") From 83e7a3a9daa75515db823a987856af2db6febba7 Mon Sep 17 00:00:00 2001 From: yujiezhang-ops Date: Sat, 10 Oct 2026 11:19:04 +0800 Subject: [PATCH 14/18] fix: keep generated Skill output outside the Skill tree Generated images, audio and video now default to ~/.bootagent/output instead of bootagent-output in the working directory. The Skill templates told Agents to run from the Skill directory, so the first generation changed the managed tree's hash and blocked later reinstall, model switches, endpoint refreshes and uninstall. The script also: - redacts successful text responses (vision, transcription, video understanding) before printing them - sets process.exitCode instead of calling process.exit, so long transcripts written to a pipe are not truncated - writes each result to its own temporary directory and publishes it with a hard link, so concurrent downloads in the same second never share a .part file or replace each other's output Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 1 + README_ZH.md | 1 + docs/multimodal-skills-design.md | 6 +- internal/skillgen/native_script_test.go | 4 +- internal/skillgen/runtime_regression_test.go | 214 ++++++++++++++++++ internal/skillgen/script_test.go | 4 +- internal/skillgen/skillgen.go | 2 +- .../templates/image-generation.md.tmpl | 4 +- internal/skillgen/templates/run.mjs | 85 ++++--- .../skillgen/templates/speech-to-text.md.tmpl | 2 +- .../skillgen/templates/text-to-speech.md.tmpl | 6 +- .../templates/video-generation.md.tmpl | 4 +- .../templates/video-understanding.md.tmpl | 2 +- internal/skillgen/templates/vision.md.tmpl | 2 +- .../testdata/golden/image-generation/SKILL.md | 4 +- .../golden/image-generation/config.json | 2 +- .../testdata/golden/speech-to-text/SKILL.md | 2 +- .../golden/speech-to-text/config.json | 2 +- .../golden/text-to-speech-native/SKILL.md | 4 +- .../golden/text-to-speech-native/config.json | 2 +- .../testdata/golden/text-to-speech/SKILL.md | 4 +- .../golden/text-to-speech/config.json | 2 +- .../testdata/golden/video-generation/SKILL.md | 4 +- .../golden/video-generation/config.json | 2 +- .../golden/video-understanding/SKILL.md | 2 +- .../golden/video-understanding/config.json | 2 +- .../skillgen/testdata/golden/vision/SKILL.md | 2 +- .../testdata/golden/vision/config.json | 2 +- 28 files changed, 306 insertions(+), 67 deletions(-) create mode 100644 internal/skillgen/runtime_regression_test.go diff --git a/README.md b/README.md index d8e473b9..c37a9e61 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,7 @@ Launch BootAgent - Keep a local Skill library, scan Skills already present on supported Agents, and choose which Agents receive each Skill. - See which models on your keyed Providers can understand images, video or audio, or generate images, and where each finding comes from (declared by the Provider, inferred from the model name, or probed). Listing models is free; a probe is a real request and is sent only when you click it, and an image-generation probe asks first. +- Generated images, audio and video default to `~/.bootagent/output/`, outside the installed Skill directory, and never overwrite existing files. Use `--out` to choose another destination. - Install a multimodal Skill (image understanding, image generation, speech-to-text, text-to-speech, video generation, video understanding) into Claude Code, Codex or OpenCode, with example prompts to copy. Agents without a Skills directory get the same files under `~/.bootagent/multimodal/` and a prompt that points them there. Video understanding samples frames with ffmpeg, which must be on `PATH`. Built-in Providers add the models their docs name: MiniMax Speech on Novita, PPIO and JieKou.AI, GPT Image and async text-to-video on JieKou.AI. Video is never probed, because a probe would be a full billed generation. Its script needs Node.js 18 or later. BootAgent writes the Provider key into the Agent's own configuration as `BOOTAGENT__API_KEY`, or into a private file under `~/.bootagent/skill-env/` for Agents with no configurable environment, so the Agent and its model can read it; the install confirmation says so. Uninstalling the last Skill that uses a Provider removes the variable. - Discover MCP servers from initialized Claude Code, Codex, OpenCode, Kilo CLI, and Hermes installations. - Store MCP servers in a local user-level Registry, select sync targets explicitly, and apply changes only when you are ready. diff --git a/README_ZH.md b/README_ZH.md index c8bc4df3..e9c1ce6f 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -62,6 +62,7 @@ BootAgent 是一个本地桌面工作台，用来统一管理 AI 编程 Agent。 - 在本机维护 Skill 管理库，扫描支持的 Agent 已有的 Skill，并选择每个 Skill 要同步到哪些 Agent。 - 查看已保存 API Key 的 Provider 中哪些模型能理解图片、视频或音频，或能生成图片，以及每项结论的来源（Provider 声明、按模型名推断或实测）。列出模型不收费；实测是一次真实请求，只在你点击时发出，图片生成实测会先确认。 +- 生成的图片、音频和视频默认保存到 `~/.bootagent/output/`，位于已安装的 Skill 目录之外，且不会覆盖已有文件。可用 `--out` 指定其他位置。 - 把多模态 Skill（图片理解、图片生成、语音识别、语音合成、视频生成、视频理解）安装到 Claude Code、Codex 或 OpenCode，并提供可复制的示例提示词。没有 Skills 目录的 Agent 会在 `~/.bootagent/multimodal/` 下得到同样的文件，并附一段指向它的提示词。视频理解用 ffmpeg 抽帧，需要 ffmpeg 在 `PATH` 上。内置 Provider 会补充其文档中列出的模型：Novita、PPIO 和 JieKou.AI 的 MiniMax Speech，JieKou.AI 的 GPT Image 与异步文生视频。视频生成从不实测，因为一次实测就是一次完整的计费生成。Skill 脚本需要 Node.js 18 或更高版本。BootAgent 会把 Provider 的 Key 以 `BOOTAGENT__API_KEY` 写进 Agent 自己的配置；没有可配置环境变量的 Agent 则写进 `~/.bootagent/skill-env/` 下的私有文件。因此 Agent 及其模型能读到它，安装确认里会说明。卸载最后一个用到该 Provider 的 Skill 时，这个变量会被移除。 - 从已初始化的 Claude Code、Codex、OpenCode、Kilo CLI 和 Hermes 中发现 MCP 服务器。 - 使用本机用户级 Registry 保存 MCP 服务器，明确选择同步目标，并在确认后再应用改动。 diff --git a/docs/multimodal-skills-design.md b/docs/multimodal-skills-design.md index 3b1ae618..db9dae24 100644 --- a/docs/multimodal-skills-design.md +++ b/docs/multimodal-skills-design.md @@ -382,8 +382,10 @@ retried; a refused or failed task is final, since retrying spends money. A timeout prints the task ID so `video-result` can collect it later without submitting again, and `SKILL.md` tells the Agent never to retry on its own. Results are streamed to disk as they download, without the key, and a partial -file is removed on failure. Outputs default to `bootagent-output/` in the -working directory and never overwrite an existing file. Result URLs are +file is removed on failure. Outputs default to `~/.bootagent/output/`, outside +the installed Skill tree, and never overwrite an existing file, including when +downloads run concurrently. Run scripts from the user's working directory so +relative input paths refer to their files. Result URLs are downloaded without the key, because they may be on another host. `config.json` is non-secret: Provider ID, adapter, model, base URLs and the diff --git a/internal/skillgen/native_script_test.go b/internal/skillgen/native_script_test.go index 1c111cb6..6d70629e 100644 --- a/internal/skillgen/native_script_test.go +++ b/internal/skillgen/native_script_test.go @@ -164,13 +164,13 @@ func TestScriptGeneratesVideoAndDownloadsWithoutTheKey(t *testing.T) { t.Fatalf("run = %#v", run) } saved := strings.TrimSpace(run.stdout) - if !strings.HasPrefix(saved, filepath.Join(work, "bootagent-output")) || !strings.HasSuffix(saved, ".mp4") { + if !strings.HasPrefix(saved, filepath.Join(home, ".bootagent", "output")+string(os.PathSeparator)) || !strings.HasSuffix(saved, ".mp4") { t.Fatalf("saved = %q", saved) } if data, _ := os.ReadFile(saved); string(data) != "fake mp4 bytes" { t.Fatalf("video = %q", data) } - if matches, _ := filepath.Glob(filepath.Join(work, "bootagent-output", "*.part")); len(matches) != 0 { + if matches, _ := filepath.Glob(filepath.Join(home, ".bootagent", "output", ".bootagent-output-*")); len(matches) != 0 { t.Fatalf("partial files left: %v", matches) } body, _ := fake.body("/v3/async/wan2.7-t2v") diff --git a/internal/skillgen/runtime_regression_test.go b/internal/skillgen/runtime_regression_test.go new file mode 100644 index 00000000..6fbcac79 --- /dev/null +++ b/internal/skillgen/runtime_regression_test.go @@ -0,0 +1,214 @@ +package skillgen + +import ( + "context" + "encoding/json" + "encoding/pem" + "io" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/MaimoryLab/BootAgent/internal/modality" + "github.com/MaimoryLab/BootAgent/internal/skill" +) + +func TestScriptRedactsSuccessfulTextResponses(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + text := "Echo: " + r.Header.Get("Authorization") + _ = json.NewEncoder(w).Encode(map[string]any{ + "text": text, "choices": []any{map[string]any{"message": map[string]any{"content": text}}}, + }) + })) + defer server.Close() + for _, capability := range []string{modality.Vision, modality.SpeechToText, modality.VideoUnderstanding} { + t.Run(capability, func(t *testing.T) { + adapter := map[string]string{modality.Vision: modality.AdapterChatVision, modality.SpeechToText: modality.AdapterAudioTranscriptions, modality.VideoUnderstanding: modality.AdapterFramesVision}[capability] + dir := renderTo(t, Spec{Capability: capability, Adapter: adapter, ProviderID: "local", Model: "m", BaseURL: server.URL}) + work, home := resolvedTempDir(t), resolvedTempDir(t) + env := []string{keyEnv} + args := []string{"describe", "--image", "input.png"} + input := "input.png" + if capability == modality.SpeechToText { + input, args = "input.wav", []string{"transcribe", "--audio", "input.wav"} + } else if capability == modality.VideoUnderstanding { + input, args = "input.mp4", []string{"describe-video", "--video", "input.mp4", "--frames", "1"} + tools := fakeFFmpeg(t, filepath.Join(resolvedTempDir(t), "env")) + env = append(env, "PATH="+tools+string(os.PathListSeparator)+os.Getenv("PATH")) + } + if err := os.WriteFile(filepath.Join(work, input), []byte("input"), 0o600); err != nil { + t.Fatal(err) + } + run := runScript(t, dir, work, home, env, args...) + if run.code != 0 || !strings.Contains(run.stdout, "[redacted]") || strings.Contains(run.stdout+run.stderr, scriptKey) { + t.Fatalf("successful response was not redacted: exit %d, redaction present %t, key present %t", run.code, strings.Contains(run.stdout, "[redacted]"), strings.Contains(run.stdout+run.stderr, scriptKey)) + } + }) + } +} + +func TestScriptFlushesLongTranscriptsToAPipe(t *testing.T) { + transcript := strings.Repeat("long transcript ", 20000) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + _ = json.NewEncoder(w).Encode(map[string]string{"text": transcript}) + })) + defer server.Close() + dir := renderTo(t, Spec{Capability: modality.SpeechToText, Adapter: modality.AdapterAudioTranscriptions, ProviderID: "local", Model: "m", BaseURL: server.URL}) + work, home := resolvedTempDir(t), resolvedTempDir(t) + if err := os.WriteFile(filepath.Join(work, "input.wav"), []byte("RIFF WAVE"), 0o600); err != nil { + t.Fatal(err) + } + run := runScript(t, dir, work, home, []string{keyEnv}, "transcribe", "--audio", "input.wav") + want := strings.TrimSpace(transcript) + "\n" + if run.code != 0 || run.stdout != want { + t.Fatalf("transcript: exit %d, received %d bytes, want %d bytes", run.code, len(run.stdout), len(want)) + } +} + +func TestScriptDefaultOutputPreservesTheSkillTree(t *testing.T) { + for _, capability := range []string{modality.ImageGeneration, modality.TextToSpeech} { + t.Run(capability, func(t *testing.T) { + adapter := modality.AdapterImages + args := []string{"generate", "--prompt", "a lighthouse"} + if capability == modality.TextToSpeech { + adapter, args = modality.AdapterAudioSpeech, []string{"speak", "--text", "hello"} + } + _, dir, _, home := scriptSetup(t, capability, adapter, "m") + before, err := skill.HashTree(context.Background(), dir) + if err != nil { + t.Fatal(err) + } + // The Skill may be invoked from its own directory by an Agent. + run := runScript(t, dir, dir, home, []string{keyEnv}, args...) + if run.code != 0 { + t.Fatalf("run = %#v", run) + } + for _, path := range strings.Split(strings.TrimSpace(run.stdout), "\n") { + if !strings.HasPrefix(path, filepath.Join(home, ".bootagent", "output")+string(os.PathSeparator)) { + t.Errorf("default output is outside the user's output folder: %s", path) + } + } + after, err := skill.HashTree(context.Background(), dir) + if err != nil || before != after { + t.Fatalf("generation changed the managed Skill tree: before %#v, after %#v, error %v", before, after, err) + } + }) + } +} + +func TestScriptConcurrentVideosKeepBothResults(t *testing.T) { + if _, err := exec.LookPath("node"); err != nil { + t.Skip("node is not installed") + } + var mu sync.Mutex + downloads := 0 + ready := make(chan struct{}) + var server *httptest.Server + server = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v3/async/task-result" { + task := r.URL.Query().Get("task_id") + _ = json.NewEncoder(w).Encode(map[string]any{ + "task": map[string]string{"status": "TASK_STATUS_SUCCEED"}, + "videos": []any{map[string]string{"video_url": server.URL + "/files/" + task, "video_type": "mp4"}}, + }) + return + } + mu.Lock() + downloads++ + if downloads == 2 { + close(ready) + } + mu.Unlock() + select { + case <-ready: + case <-time.After(3 * time.Second): + http.Error(w, "the other download did not start", http.StatusGatewayTimeout) + return + case <-r.Context().Done(): + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(r.URL.Path + ":first")) + w.(http.Flusher).Flush() + // Keep both downloads open so each writer has begun before either finishes. + time.Sleep(200 * time.Millisecond) + _, _ = w.Write([]byte(":last")) + })) + defer server.Close() + ca := filepath.Join(resolvedTempDir(t), "ca.pem") + if err := os.WriteFile(ca, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw}), 0o600); err != nil { + t.Fatal(err) + } + spec := videoSpec + spec.BaseURL, spec.NativeBaseURL = server.URL, server.URL + dir := renderTo(t, spec) + work, home := resolvedTempDir(t), resolvedTempDir(t) + outputDir := filepath.Join(home, ".bootagent", "output") + if err := os.MkdirAll(outputDir, 0o700); err != nil { + t.Fatal(err) + } + existing := filepath.Join(outputDir, "video-20261010-000000.mp4") + if err := os.WriteFile(existing, []byte("existing video"), 0o600); err != nil { + t.Fatal(err) + } + clock := filepath.Join(resolvedTempDir(t), "clock.cjs") + if err := os.WriteFile(clock, []byte(`Date.prototype.toISOString = () => "2026-10-10T00:00:00.000Z";`), 0o600); err != nil { + t.Fatal(err) + } + env := []string{keyEnv, "NODE_EXTRA_CA_CERTS=" + ca, "NODE_OPTIONS=--require " + strconv.Quote(clock)} + results := make(chan scriptRun, 2) + for _, task := range []string{"task-A", "task-B"} { + go func(task string) { + results <- runScript(t, dir, work, home, env, "video-result", "--task", task) + }(task) + } + paths := map[string]bool{} + contents := map[string]bool{} + for range 2 { + run := <-results + if run.code != 0 { + t.Errorf("concurrent download: exit %d, stderr %s", run.code, run.stderr) + continue + } + path := strings.TrimSpace(run.stdout) + paths[path] = true + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + contents[string(data)] = true + } + if len(paths) != 2 || !contents["/files/task-A:first:last"] || !contents["/files/task-B:first:last"] { + t.Errorf("both distinct videos were not preserved: %d paths, content keys %v", len(paths), contents) + } + if data, err := os.ReadFile(existing); err != nil || string(data) != "existing video" { + t.Errorf("existing video was replaced: %q, %v", data, err) + } + if matches, _ := filepath.Glob(filepath.Join(home, ".bootagent", "output", ".bootagent-output-*")); len(matches) != 0 { + t.Errorf("temporary download directories left behind: %v", matches) + } +} + +func TestScriptExplicitSpeechOutputKeepsAnExistingFile(t *testing.T) { + _, dir, work, home := scriptSetup(t, modality.TextToSpeech, modality.AdapterAudioSpeech, "tts-1") + existing := filepath.Join(work, "done.mp3") + if err := os.WriteFile(existing, []byte("existing audio"), 0o600); err != nil { + t.Fatal(err) + } + run := runScript(t, dir, work, home, []string{keyEnv}, "speak", "--text", "hello", "--out", "done.mp3") + if run.code != 0 || strings.TrimSpace(run.stdout) != filepath.Join(work, "done-2.mp3") { + t.Fatalf("explicit output = %#v", run) + } + if data, err := os.ReadFile(existing); err != nil || string(data) != "existing audio" { + t.Fatalf("existing audio was replaced: %q, %v", data, err) + } +} diff --git a/internal/skillgen/script_test.go b/internal/skillgen/script_test.go index 4b4a0740..8a7c4634 100644 --- a/internal/skillgen/script_test.go +++ b/internal/skillgen/script_test.go @@ -175,8 +175,8 @@ func TestScriptGeneratesImagesAndDownloadsWithoutTheKey(t *testing.T) { t.Fatalf("saved = %q", paths) } for _, path := range paths { - if !strings.HasPrefix(path, filepath.Join(work, "bootagent-output")) { - t.Fatalf("%s is outside the working directory's output folder", path) + if !strings.HasPrefix(path, filepath.Join(home, ".bootagent", "output")+string(os.PathSeparator)) { + t.Fatalf("%s is outside the user's output folder", path) } if _, err := os.Stat(path); err != nil { t.Fatal(err) diff --git a/internal/skillgen/skillgen.go b/internal/skillgen/skillgen.go index 7d8723ec..3342dfce 100644 --- a/internal/skillgen/skillgen.go +++ b/internal/skillgen/skillgen.go @@ -188,7 +188,7 @@ func NewConfig(spec Spec) (Config, error) { Options: options, KeyEnv: config.SkillEnvName(spec.ProviderID), KeyFile: "~/.bootagent/skill-env/" + spec.ProviderID + ".env", - OutputDir: "bootagent-output", + OutputDir: "~/.bootagent/output", }, nil } diff --git a/internal/skillgen/templates/image-generation.md.tmpl b/internal/skillgen/templates/image-generation.md.tmpl index f2f3bb20..e7acc161 100644 --- a/internal/skillgen/templates/image-generation.md.tmpl +++ b/internal/skillgen/templates/image-generation.md.tmpl @@ -5,7 +5,7 @@ When the user asks you to create an image: an illustration, an icon draft, a moc ## Command -Run the script in this Skill's directory (the directory containing this `SKILL.md`): +Run the script from the user's working directory, using the absolute path to this Skill: ```bash node /scripts/run.mjs generate --prompt "" [--size ] [--n ] [--out ] @@ -14,7 +14,7 @@ node /scripts/run.mjs generate --prompt "" - `--prompt`: a full description of the image to create. - `--size`: for example `1024x1024`. Leave it out unless the user asks; valid sizes depend on the model. - `--n`: how many images, 1 to 4. Default 1. -- `--out`: directory for the files. Default `bootagent-output` in the current directory. +- `--out`: directory for the files. Default `~/.bootagent/output`, outside the Skill directory. - `--timeout`: seconds to wait. Default 300. The saved file paths are printed to stdout, one per line. Existing files are never overwritten. diff --git a/internal/skillgen/templates/run.mjs b/internal/skillgen/templates/run.mjs index a38b5e7e..bf017475 100644 --- a/internal/skillgen/templates/run.mjs +++ b/internal/skillgen/templates/run.mjs @@ -7,10 +7,12 @@ // config.json, or from BootAgent's private fallback file; it is never printed. import { execFile } from "node:child_process"; -import { accessSync, constants, createWriteStream, existsSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { accessSync, constants, createWriteStream, existsSync, linkSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; import { homedir, tmpdir } from "node:os"; import { basename, delimiter, dirname, extname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import { Readable } from "node:stream"; +import { pipeline } from "node:stream/promises"; const EXIT = { ok: 0, internal: 1, usage: 2, setup: 3, refused: 4, key: 5, unavailable: 6, pending: 7 }; const MiB = 1024 * 1024; @@ -148,21 +150,40 @@ function stamp() { return new Date().toISOString().replace(/[-:]/g, "").replace(/\..+$/, "").replace("T", "-"); } -function freePath(path) { - if (!existsSync(path)) return path; +// Linking a completed temporary file publishes it atomically and refuses an +// existing destination. Concurrent runs retry with a suffix instead of +// checking first and then accidentally replacing another run's output. +function publishOutput(partial, path) { const extension = extname(path); const stem = path.slice(0, path.length - extension.length); - for (let counter = 2; ; counter++) { - const candidate = `${stem}-${counter}${extension}`; - if (!existsSync(candidate)) return candidate; + for (let counter = 1; ; counter++) { + const candidate = counter === 1 ? path : `${stem}-${counter}${extension}`; + try { + linkSync(partial, candidate); + return candidate; + } catch (error) { + if (error?.code !== "EEXIST") { + fail(EXIT.unavailable, `Cannot save the result without overwriting an existing file (${error?.code ?? "filesystem error"}). Choose an output directory on a filesystem that supports hard links.`); + } + } } } function writeOutput(path, data) { - const target = freePath(resolve(path)); + const target = resolve(path); mkdirSync(dirname(target), { recursive: true }); - writeFileSync(target, data); - return target; + const work = mkdtempSync(join(dirname(target), ".bootagent-output-")); + try { + const partial = join(work, "result.part"); + writeFileSync(partial, data, { mode: 0o600, flag: "wx" }); + return publishOutput(partial, target); + } finally { + rmSync(work, { recursive: true, force: true }); + } +} + +function outputDirectory(config, options) { + return resolve(options.out || expandHome(config.output_dir || "~/.bootagent/output")); } function detail(text) { @@ -258,7 +279,7 @@ async function describe(config, key, options, timeoutMs) { const text = body?.choices?.[0]?.message?.content; const answer = Array.isArray(text) ? text.map((part) => part?.text ?? "").join("") : text; if (typeof answer !== "string") fail(EXIT.unavailable, "The provider returned no text."); - process.stdout.write(answer.trim() + "\n"); + process.stdout.write(redact(answer.trim()) + "\n"); } async function generate(config, key, options, timeoutMs) { @@ -274,7 +295,7 @@ async function generate(config, key, options, timeoutMs) { const body = await json(response); const items = Array.isArray(body?.data) ? body.data : []; if (!items.length) fail(EXIT.unavailable, "The provider returned no image."); - const directory = resolve(options.out || config.output_dir || "bootagent-output"); + const directory = outputDirectory(config, options); const base = `image-${stamp()}`; const saved = []; for (const [index, item] of items.entries()) { @@ -300,7 +321,7 @@ async function transcribe(config, key, options, timeoutMs) { const response = await call(config, key, "/audio/transcriptions", { method: "POST", body: form }, timeoutMs); const body = await json(response); if (typeof body?.text !== "string") fail(EXIT.unavailable, "The provider returned no transcript."); - process.stdout.write(body.text.trim() + "\n"); + process.stdout.write(redact(body.text.trim()) + "\n"); } const SPEECH_FORMATS = ["mp3", "wav", "opus", "aac", "flac", "pcm"]; @@ -320,7 +341,7 @@ async function speak(config, key, options, timeoutMs) { if (data.length > LIMITS.speech) fail(EXIT.unavailable, "The audio is larger than expected and was not saved."); const target = options.out && extname(options.out) ? options.out - : join(options.out || config.output_dir || "bootagent-output", `speech-${stamp()}.${format}`); + : join(outputDirectory(config, options), `speech-${stamp()}.${format}`); process.stdout.write(writeOutput(target, data) + "\n"); } @@ -363,7 +384,7 @@ async function speakNative(config, key, options, timeoutMs) { if (data.length > LIMITS.speech) fail(EXIT.unavailable, "The audio is larger than expected and was not saved."); const target = options.out && extname(options.out) ? options.out - : join(options.out || config.output_dir || "bootagent-output", `speech-${stamp()}.${format}`); + : join(outputDirectory(config, options), `speech-${stamp()}.${format}`); process.stdout.write(writeOutput(target, data) + "\n"); } @@ -394,7 +415,7 @@ async function generateVideo(config, key, options, timeoutMs) { const body = await json(response); const task = body?.task_id; if (typeof task !== "string" || !task) fail(EXIT.unavailable, "The provider returned no task id."); - process.stderr.write(`task: ${task}\n`); + process.stderr.write(redact(`task: ${task}\n`)); await finishVideo(config, key, task, options, deadline); } @@ -422,7 +443,7 @@ async function finishVideo(config, key, task, options, deadline) { if (status === "TASK_STATUS_SUCCEED") { const videos = Array.isArray(body?.videos) ? body.videos.filter((item) => typeof item?.video_url === "string") : []; if (!videos.length) fail(EXIT.unavailable, `Task ${task} finished without a video.`); - const directory = resolve(options.out || config.output_dir || "bootagent-output"); + const directory = outputDirectory(config, options); const saved = []; for (const [index, item] of videos.entries()) { const extension = item.video_type === "gif" ? "gif" : "mp4"; @@ -459,26 +480,26 @@ async function downloadTo(url, path, timeoutMs) { fail(EXIT.unavailable, "Cannot download the result from the provider."); } if (!response.ok || !response.body) fail(EXIT.unavailable, `Cannot download the result (HTTP ${response.status}).`); - const target = freePath(resolve(path)); + const target = resolve(path); mkdirSync(dirname(target), { recursive: true }); - const partial = `${target}.part`; - const out = createWriteStream(partial); + const work = mkdtempSync(join(dirname(target), ".bootagent-output-")); + const partial = join(work, "result.part"); let size = 0; try { - for await (const chunk of response.body) { - size += chunk.length; - if (size > LIMITS.video) throw new Failure(EXIT.unavailable, "The result is larger than expected and was not saved."); - if (!out.write(chunk)) await new Promise((done) => out.once("drain", done)); - } - await new Promise((done, failed) => out.end((error) => (error ? failed(error) : done()))); - renameSync(partial, target); + await pipeline(Readable.fromWeb(response.body), async function* (chunks) { + for await (const chunk of chunks) { + size += chunk.length; + if (size > LIMITS.video) throw new Failure(EXIT.unavailable, "The result is larger than expected and was not saved."); + yield chunk; + } + }, createWriteStream(partial, { mode: 0o600, flags: "wx" })); + return publishOutput(partial, target); } catch (error) { - out.destroy(); - rmSync(partial, { force: true }); if (error instanceof Failure) throw error; fail(EXIT.unavailable, "The download was interrupted."); + } finally { + rmSync(work, { recursive: true, force: true }); } - return target; } // Each capability and adapter this script implements: the subcommand, its @@ -559,7 +580,7 @@ async function describeVideo(config, key, options, timeoutMs) { const text = body?.choices?.[0]?.message?.content; const answer = Array.isArray(text) ? text.map((part) => part?.text ?? "").join("") : text; if (typeof answer !== "string") fail(EXIT.unavailable, "The provider returned no text."); - process.stdout.write(answer.trim() + "\n"); + process.stdout.write(redact(answer.trim()) + "\n"); } finally { rmSync(work, { recursive: true, force: true }); } @@ -605,10 +626,10 @@ async function main() { } main().then( - () => process.exit(EXIT.ok), + () => { process.exitCode = EXIT.ok; }, (error) => { const code = error instanceof Failure ? error.code : EXIT.internal; process.stderr.write(redact(error instanceof Failure ? error.message : `Unexpected error: ${error?.message ?? error}`) + "\n"); - process.exit(code); + process.exitCode = code; }, ); diff --git a/internal/skillgen/templates/speech-to-text.md.tmpl b/internal/skillgen/templates/speech-to-text.md.tmpl index 036ac2c4..f89856c5 100644 --- a/internal/skillgen/templates/speech-to-text.md.tmpl +++ b/internal/skillgen/templates/speech-to-text.md.tmpl @@ -5,7 +5,7 @@ When the user asks for the text of an audio recording: transcribing a voice note ## Command -Run the script in this Skill's directory (the directory containing this `SKILL.md`): +Run the script from the user's working directory, using the absolute path to this Skill: ```bash node /scripts/run.mjs transcribe --audio [--language ] [--prompt ""] diff --git a/internal/skillgen/templates/text-to-speech.md.tmpl b/internal/skillgen/templates/text-to-speech.md.tmpl index 91846c7c..e1e69e67 100644 --- a/internal/skillgen/templates/text-to-speech.md.tmpl +++ b/internal/skillgen/templates/text-to-speech.md.tmpl @@ -5,7 +5,7 @@ When the user asks you to turn text into spoken audio: reading out a summary, pr ## Command -Run the script in this Skill's directory (the directory containing this `SKILL.md`): +Run the script from the user's working directory, using the absolute path to this Skill: {{if eq .Adapter "novita-audio"}} ```bash node /scripts/run.mjs speak --text "" [--voice ] [--format mp3|wav|flac|pcm] [--speed <0.5-2>] [--out ] @@ -15,7 +15,7 @@ node /scripts/run.mjs speak --text "" [--voice /scripts/run.mjs speak --text "" [--voice /scripts/run.mjs video --prompt ""{{if has .Options "duration"}} [--duration ]{{end}}{{if has .Options "size"}} [--size ]{{end}}{{if has .Options "aspect_ratio"}} [--aspect-ratio 16:9|9:16|1:1]{{end}}{{if has .Options "negative_prompt"}} [--negative-prompt ""]{{end}}{{if has .Options "sound"}} [--sound true|false]{{end}} [--out ] @@ -27,7 +27,7 @@ node /scripts/run.mjs video --prompt ""{{if {{- if has .Options "sound"}} - `--sound`: `true` to generate audio with the video. {{- end}} -- `--out`: directory for the file. Default `bootagent-output` in the current directory. +- `--out`: directory for the file. Default `~/.bootagent/output`, outside the Skill directory. - `--timeout`: seconds to wait for the result. Default 600. The script submits the task, prints `task: ` to stderr, waits for it, downloads the video before the Provider's link expires, and prints the saved file path to stdout. If it times out (exit code 7), the task may still finish: fetch it later with diff --git a/internal/skillgen/templates/video-understanding.md.tmpl b/internal/skillgen/templates/video-understanding.md.tmpl index 7522a69b..4285b136 100644 --- a/internal/skillgen/templates/video-understanding.md.tmpl +++ b/internal/skillgen/templates/video-understanding.md.tmpl @@ -9,7 +9,7 @@ When the user asks about the contents of a local video file: what happens in a s ## Command -Run the script in this Skill's directory (the directory containing this `SKILL.md`): +Run the script from the user's working directory, using the absolute path to this Skill: ```bash node /scripts/run.mjs describe-video --video [--prompt ""] [--frames <1-16>] [--max-tokens ] diff --git a/internal/skillgen/templates/vision.md.tmpl b/internal/skillgen/templates/vision.md.tmpl index 3dd4dd87..45afb5a2 100644 --- a/internal/skillgen/templates/vision.md.tmpl +++ b/internal/skillgen/templates/vision.md.tmpl @@ -5,7 +5,7 @@ When the user asks about the contents of an image file: reading a screenshot, fi ## Command -Run the script in this Skill's directory (the directory containing this `SKILL.md`): +Run the script from the user's working directory, using the absolute path to this Skill: ```bash node /scripts/run.mjs describe --image [--image ...] [--prompt ""] [--max-tokens ] diff --git a/internal/skillgen/testdata/golden/image-generation/SKILL.md b/internal/skillgen/testdata/golden/image-generation/SKILL.md index 9373c7bf..a1163983 100644 --- a/internal/skillgen/testdata/golden/image-generation/SKILL.md +++ b/internal/skillgen/testdata/golden/image-generation/SKILL.md @@ -13,7 +13,7 @@ When the user asks you to create an image: an illustration, an icon draft, a moc ## Command -Run the script in this Skill's directory (the directory containing this `SKILL.md`): +Run the script from the user's working directory, using the absolute path to this Skill: ```bash node /scripts/run.mjs generate --prompt "" [--size ] [--n ] [--out ] @@ -22,7 +22,7 @@ node /scripts/run.mjs generate --prompt "" - `--prompt`: a full description of the image to create. - `--size`: for example `1024x1024`. Leave it out unless the user asks; valid sizes depend on the model. - `--n`: how many images, 1 to 4. Default 1. -- `--out`: directory for the files. Default `bootagent-output` in the current directory. +- `--out`: directory for the files. Default `~/.bootagent/output`, outside the Skill directory. - `--timeout`: seconds to wait. Default 300. The saved file paths are printed to stdout, one per line. Existing files are never overwritten. diff --git a/internal/skillgen/testdata/golden/image-generation/config.json b/internal/skillgen/testdata/golden/image-generation/config.json index 1bd98ed2..40f9e9df 100644 --- a/internal/skillgen/testdata/golden/image-generation/config.json +++ b/internal/skillgen/testdata/golden/image-generation/config.json @@ -9,5 +9,5 @@ "base_url": "https://gw.example/v1", "key_env": "BOOTAGENT_MY_GATEWAY_API_KEY", "key_file": "~/.bootagent/skill-env/my-gateway.env", - "output_dir": "bootagent-output" + "output_dir": "~/.bootagent/output" } diff --git a/internal/skillgen/testdata/golden/speech-to-text/SKILL.md b/internal/skillgen/testdata/golden/speech-to-text/SKILL.md index 8e75c6f8..3e17966a 100644 --- a/internal/skillgen/testdata/golden/speech-to-text/SKILL.md +++ b/internal/skillgen/testdata/golden/speech-to-text/SKILL.md @@ -13,7 +13,7 @@ When the user asks for the text of an audio recording: transcribing a voice note ## Command -Run the script in this Skill's directory (the directory containing this `SKILL.md`): +Run the script from the user's working directory, using the absolute path to this Skill: ```bash node /scripts/run.mjs transcribe --audio [--language ] [--prompt ""] diff --git a/internal/skillgen/testdata/golden/speech-to-text/config.json b/internal/skillgen/testdata/golden/speech-to-text/config.json index db21a42a..e71ed773 100644 --- a/internal/skillgen/testdata/golden/speech-to-text/config.json +++ b/internal/skillgen/testdata/golden/speech-to-text/config.json @@ -9,5 +9,5 @@ "base_url": "https://api.ppio.com/openai/v1", "key_env": "BOOTAGENT_PPIO_API_KEY", "key_file": "~/.bootagent/skill-env/ppio.env", - "output_dir": "bootagent-output" + "output_dir": "~/.bootagent/output" } diff --git a/internal/skillgen/testdata/golden/text-to-speech-native/SKILL.md b/internal/skillgen/testdata/golden/text-to-speech-native/SKILL.md index 25911097..0fe9f138 100644 --- a/internal/skillgen/testdata/golden/text-to-speech-native/SKILL.md +++ b/internal/skillgen/testdata/golden/text-to-speech-native/SKILL.md @@ -13,7 +13,7 @@ When the user asks you to turn text into spoken audio: reading out a summary, pr ## Command -Run the script in this Skill's directory (the directory containing this `SKILL.md`): +Run the script from the user's working directory, using the absolute path to this Skill: ```bash node /scripts/run.mjs speak --text "" [--voice ] [--format mp3|wav|flac|pcm] [--speed <0.5-2>] [--out ] @@ -23,7 +23,7 @@ node /scripts/run.mjs speak --text "" [--voice /scripts/run.mjs speak --text "" [--voice ] [--format mp3|wav|opus|aac|flac|pcm] [--out ] @@ -22,7 +22,7 @@ node /scripts/run.mjs speak --text "" [--voice /scripts/run.mjs video --prompt "" [--duration ] [--size ] [--negative-prompt ""] [--out ] @@ -23,7 +23,7 @@ node /scripts/run.mjs video --prompt "" [-- - `--duration`: length in seconds. Billed by the second; ask the user before going above the default of 5. - `--size`: resolution as width*height, for example `1280*720` or `1920*1080`. Larger costs more. - `--negative-prompt`: what must not appear. -- `--out`: directory for the file. Default `bootagent-output` in the current directory. +- `--out`: directory for the file. Default `~/.bootagent/output`, outside the Skill directory. - `--timeout`: seconds to wait for the result. Default 600. The script submits the task, prints `task: ` to stderr, waits for it, downloads the video before the Provider's link expires, and prints the saved file path to stdout. If it times out (exit code 7), the task may still finish: fetch it later with diff --git a/internal/skillgen/testdata/golden/video-generation/config.json b/internal/skillgen/testdata/golden/video-generation/config.json index db262b54..c2c1fdf9 100644 --- a/internal/skillgen/testdata/golden/video-generation/config.json +++ b/internal/skillgen/testdata/golden/video-generation/config.json @@ -15,5 +15,5 @@ ], "key_env": "BOOTAGENT_JIEKOU_API_KEY", "key_file": "~/.bootagent/skill-env/jiekou.env", - "output_dir": "bootagent-output" + "output_dir": "~/.bootagent/output" } diff --git a/internal/skillgen/testdata/golden/video-understanding/SKILL.md b/internal/skillgen/testdata/golden/video-understanding/SKILL.md index b2f8ffa8..3b181921 100644 --- a/internal/skillgen/testdata/golden/video-understanding/SKILL.md +++ b/internal/skillgen/testdata/golden/video-understanding/SKILL.md @@ -17,7 +17,7 @@ When the user asks about the contents of a local video file: what happens in a s ## Command -Run the script in this Skill's directory (the directory containing this `SKILL.md`): +Run the script from the user's working directory, using the absolute path to this Skill: ```bash node /scripts/run.mjs describe-video --video [--prompt ""] [--frames <1-16>] [--max-tokens ] diff --git a/internal/skillgen/testdata/golden/video-understanding/config.json b/internal/skillgen/testdata/golden/video-understanding/config.json index 6ad6e96e..7122f36a 100644 --- a/internal/skillgen/testdata/golden/video-understanding/config.json +++ b/internal/skillgen/testdata/golden/video-understanding/config.json @@ -9,5 +9,5 @@ "base_url": "https://api.novita.ai/openai/v1", "key_env": "BOOTAGENT_NOVITA_API_KEY", "key_file": "~/.bootagent/skill-env/novita.env", - "output_dir": "bootagent-output" + "output_dir": "~/.bootagent/output" } diff --git a/internal/skillgen/testdata/golden/vision/SKILL.md b/internal/skillgen/testdata/golden/vision/SKILL.md index ebc27b2e..7a831f63 100644 --- a/internal/skillgen/testdata/golden/vision/SKILL.md +++ b/internal/skillgen/testdata/golden/vision/SKILL.md @@ -13,7 +13,7 @@ When the user asks about the contents of an image file: reading a screenshot, fi ## Command -Run the script in this Skill's directory (the directory containing this `SKILL.md`): +Run the script from the user's working directory, using the absolute path to this Skill: ```bash node /scripts/run.mjs describe --image [--image ...] [--prompt ""] [--max-tokens ] diff --git a/internal/skillgen/testdata/golden/vision/config.json b/internal/skillgen/testdata/golden/vision/config.json index a0901bba..e00ea90b 100644 --- a/internal/skillgen/testdata/golden/vision/config.json +++ b/internal/skillgen/testdata/golden/vision/config.json @@ -9,5 +9,5 @@ "base_url": "https://api.novita.ai/openai/v1", "key_env": "BOOTAGENT_NOVITA_API_KEY", "key_file": "~/.bootagent/skill-env/novita.env", - "output_dir": "bootagent-output" + "output_dir": "~/.bootagent/output" } From 6eb905ec0dff2a1309296ba331676da632a55f6a Mon Sep 17 00:00:00 2001 From: yujiezhang-ops Date: Sat, 10 Oct 2026 11:19:04 +0800 Subject: [PATCH 15/18] fix: treat malformed native speech responses as unverified A 200 response whose body could not be read, was not JSON, lacked base_resp.status_code or carried no decodable audio was reported as a supported speech route and cached. These now classify as a transient failure, so the capability stays unverified. Co-Authored-By: Claude Opus 5 (1M context) --- internal/app/multimodal_test.go | 3 +++ internal/provider/client.go | 27 ++++++++++++++++++++++----- internal/provider/client_test.go | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 5 deletions(-) diff --git a/internal/app/multimodal_test.go b/internal/app/multimodal_test.go index b5c7c6de..50c988a4 100644 --- a/internal/app/multimodal_test.go +++ b/internal/app/multimodal_test.go @@ -50,6 +50,9 @@ func (g *multimodalGateway) Do(request *http.Request) (*http.Response, error) { status = http.StatusOK } if status == http.StatusOK { + if strings.HasPrefix(request.URL.Path, "/v3/") { + return appProviderResponse(status, `{"data":{"audio":"4944"},"base_resp":{"status_code":0}}`), nil + } return appProviderResponse(status, `{}`), nil } return appProviderResponse(status, `{"error":{"message":"model `+body.Model+` does not support image input, key `+multimodalKey+`"}}`), nil diff --git a/internal/provider/client.go b/internal/provider/client.go index c3343e3f..3d01c410 100644 --- a/internal/provider/client.go +++ b/internal/provider/client.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/binary" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -277,24 +278,40 @@ func (c *Client) ProbeNativeSpeech(ctx context.Context, apiKey, model, nativeBas return result, nil } defer response.Body.Close() - body, _, _ := c.readBody(response.Body) + body, tooLarge, readErr := c.readBody(response.Body) + if readErr != nil { + result := transportResult(readErr) + result.Protocol = new(ProtocolOpenAI) + return result, nil + } if response.StatusCode < 200 || response.StatusCode >= 300 { result := classifyHTTPProbe(response.StatusCode, string(body), ProtocolOpenAI, model) result.Detail = upstreamDetail(body, apiKey) return result, nil } var parsed struct { + Data struct { + Audio string `json:"audio"` + } `json:"data"` BaseResp *struct { - StatusCode int `json:"status_code"` + StatusCode *int `json:"status_code"` StatusMsg string `json:"status_msg"` } `json:"base_resp"` } - _ = json.Unmarshal(body, &parsed) - if parsed.BaseResp == nil || parsed.BaseResp.StatusCode == 0 { + invalidResponse := func() (ProbeResult, error) { + return classifyHTTPProbe(http.StatusBadGateway, "", ProtocolOpenAI, model), nil + } + if tooLarge || json.Unmarshal(body, &parsed) != nil || parsed.BaseResp == nil || parsed.BaseResp.StatusCode == nil { + return invalidResponse() + } + if *parsed.BaseResp.StatusCode == 0 { + if audio, err := hex.DecodeString(parsed.Data.Audio); err != nil || len(audio) == 0 { + return invalidResponse() + } return ProbeResult{OK: true, Reachable: true, Status: response.StatusCode, Message: "Speech route accepted the request.", Protocol: new(ProtocolOpenAI)}, nil } status := http.StatusBadRequest - switch parsed.BaseResp.StatusCode { + switch *parsed.BaseResp.StatusCode { case 1004: status = http.StatusUnauthorized case 1002, 1039: diff --git a/internal/provider/client_test.go b/internal/provider/client_test.go index bb91d37c..fec6746f 100644 --- a/internal/provider/client_test.go +++ b/internal/provider/client_test.go @@ -522,3 +522,35 @@ func TestProbeNativeSpeechReadsTheVerdictInTheBody(t *testing.T) { } } } + +func TestNativeSpeechProbeRequiresACompleteSuccessResponse(t *testing.T) { + for name, body := range map[string]string{ + "HTML": "service unavailable", + "truncated": `{"data":{"audio":"4944"},"base_resp":`, + "no verdict": `{"data":{"audio":"4944"}}`, + "no status": `{"data":{"audio":"4944"},"base_resp":{}}`, + "no audio": `{"base_resp":{"status_code":0}}`, + "invalid audio": `{"data":{"audio":"not hex"},"base_resp":{"status_code":0}}`, + } { + t.Run(name, func(t *testing.T) { + client := NewClient(fakeDoer(func(*http.Request) (*http.Response, error) { return fakeResponse(http.StatusOK, body), nil })) + result, err := client.ProbeNativeSpeech(context.Background(), "key", "minimax-speech-2.8-turbo", "https://api.novita.ai") + if err != nil || result.OK || result.Status != http.StatusBadGateway { + t.Fatalf("invalid success body should be transient, got %#v, %v", result, err) + } + }) + } + client := NewClient(fakeDoer(func(*http.Request) (*http.Response, error) { + return &http.Response{StatusCode: http.StatusOK, Body: failingBody{err: io.ErrUnexpectedEOF}}, nil + })) + if result, err := client.ProbeNativeSpeech(context.Background(), "key", "m", "https://api.novita.ai"); err != nil || result.OK || result.Reachable { + t.Fatalf("body read failure should be unverified, got %#v, %v", result, err) + } + client = NewClient(fakeDoer(func(*http.Request) (*http.Response, error) { + return fakeResponse(http.StatusOK, `{"data":{"audio":"4944"},"base_resp":{"status_code":0}}`), nil + })) + client.maxBody = 20 + if result, err := client.ProbeNativeSpeech(context.Background(), "key", "m", "https://api.novita.ai"); err != nil || result.OK || result.Status != http.StatusBadGateway { + t.Fatalf("oversized body should be unverified, got %#v, %v", result, err) + } +} From 9b463e473fa74c8edc9d89c05aeccd3b6fa4ed25 Mon Sep 17 00:00:00 2001 From: yujiezhang-ops Date: Sat, 10 Oct 2026 11:19:04 +0800 Subject: [PATCH 16/18] fix: keep generated Skills consistent with Provider changes - InstallMultimodal re-reads the Provider after taking writeMu and refuses with a retryable error if it changed while models were being listed, so a Skill never pairs an old endpoint with a new key. - Key writes from Profile save, Agent activation and install now refresh generated Skills like SaveProvider does, and report refresh failures. - v2 transfer imports refresh generated Skills for every Provider they change, and roll those Skill trees and credential files back if the refresh fails. - Key reconciliation never hands a new key to a Skill that still points at a previous endpoint, and reports registry and catalog read failures instead of silently succeeding. Co-Authored-By: Claude Opus 5 (1M context) --- internal/app/agent.go | 2 +- internal/app/install.go | 2 +- internal/app/multimodal_consistency_test.go | 359 ++++++++++++++++++++ internal/app/multimodal_install.go | 112 +++++- internal/app/provider.go | 62 +++- internal/app/status.go | 2 +- internal/app/transfer.go | 63 +++- 7 files changed, 590 insertions(+), 12 deletions(-) create mode 100644 internal/app/multimodal_consistency_test.go diff --git a/internal/app/agent.go b/internal/app/agent.go index 3003d651..225454fa 100644 --- a/internal/app/agent.go +++ b/internal/app/agent.go @@ -98,7 +98,7 @@ func (u *UseCases) activateAgentLocked(ctx context.Context, options ActivateAgen if apiKey == "" { return ActivateAgentResult{}, oneerrors.New(oneerrors.InvalidRequest, "API key is required") } - if err := u.providers.SaveKey(ctx, providerID, apiKey); err != nil { + if err := u.saveProviderKeyLocked(ctx, providerID, apiKey); err != nil { return ActivateAgentResult{}, err } diff --git a/internal/app/install.go b/internal/app/install.go index b6c1b4a6..c1c2aa6a 100644 --- a/internal/app/install.go +++ b/internal/app/install.go @@ -211,7 +211,7 @@ func (u *UseCases) prepareInstallCredentials(ctx context.Context, options Instal if options.APIKey == "" { return "", "", oneerrors.New(oneerrors.InvalidRequest, "API key is required") } - if err := u.providers.SaveKey(ctx, options.Provider, options.APIKey); err != nil { + if err := u.saveProviderKeyLocked(ctx, options.Provider, options.APIKey); err != nil { return "", "", err } target, err := u.providers.Resolve(options.Provider, options.APIBaseURL) diff --git a/internal/app/multimodal_consistency_test.go b/internal/app/multimodal_consistency_test.go new file mode 100644 index 00000000..7eb5aefb --- /dev/null +++ b/internal/app/multimodal_consistency_test.go @@ -0,0 +1,359 @@ +package app + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/MaimoryLab/BootAgent/internal/modality" + "github.com/MaimoryLab/BootAgent/internal/platform" + "github.com/MaimoryLab/BootAgent/internal/provider" + "github.com/MaimoryLab/BootAgent/internal/skill" + "github.com/MaimoryLab/BootAgent/internal/skillgen" + "github.com/MaimoryLab/BootAgent/internal/transfer" +) + +type blockingMultimodalListing struct{ entered, release chan struct{} } + +func (g *blockingMultimodalListing) Do(request *http.Request) (*http.Response, error) { + close(g.entered) + <-g.release + return appProviderResponse(http.StatusOK, gatewayListing), nil +} + +func TestMultimodalInstallRejectsProviderChangedDuringListing(t *testing.T) { + gateway := &blockingMultimodalListing{make(chan struct{}), make(chan struct{})} + home := t.TempDir() + core := NewUseCasesWithProviderClient(StatusOptions{Home: home, Platform: platform.Info{OS: "darwin", Arch: "arm64"}, Lookup: func(command string) (string, bool) { return "/fake/" + command, true }}, provider.NewClient(gateway)) + ctx := context.Background() + if _, err := core.SaveProvider(ctx, provider.Entry{ID: "gw", Name: "Gateway", BaseURL: "https://old.example/v1", APIKey: multimodalKey}, true, false); err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { + _, err := core.InstallMultimodal(ctx, MultimodalInstallRequest{Provider: "gw", Model: "qwen/qwen3-vl-8b", Capability: modality.Vision, Agents: []string{"codex"}}) + done <- err + }() + <-gateway.entered + // This also proves the model listing does not hold the configuration lock. + _, err := core.SaveProvider(ctx, provider.Entry{ID: "gw", Name: "Gateway", BaseURL: "https://new.example/v1", APIKey: "sk-rotated-key"}, false, false) + close(gateway.release) + if err != nil { + t.Fatal(err) + } + if err := <-done; err == nil { + t.Fatal("an obsolete Provider snapshot was published") + } + if _, err := os.Stat(filepath.Join(home, ".codex", "skills", "bootagent-vision")); !os.IsNotExist(err) { + t.Fatal("a Skill was written after the Provider changed") + } +} + +func TestAllCredentialWritesRefreshGeneratedSkillKeys(t *testing.T) { + for _, operation := range []string{"profile", "activate", "install"} { + t.Run(operation, func(t *testing.T) { + core, home := multimodalInstallCore(t, &multimodalGateway{listing: gatewayListing}) + core.status.Platform = platform.For("macos", "arm64") + installSkill(t, core, "qwen/qwen3-vl-8b", modality.Vision, "codex", "opencode") + if err := writeMultimodalCredential(core, operation); err != nil { + t.Fatal(err) + } + if codexSet(t, home)["BOOTAGENT_GW_API_KEY"] != "sk-rotated-key" { + t.Fatal("the native Skill variable kept the previous credential") + } + data, err := os.ReadFile(filepath.Join(home, ".bootagent", "skill-env", "gw.env")) + if err != nil || !strings.Contains(string(data), "sk-rotated-key") { + t.Fatal("the fallback Skill variable kept the previous credential") + } + }) + } +} + +func writeMultimodalCredential(core *UseCases, operation string) error { + ctx := context.Background() + var err error + switch operation { + case "profile": + _, err = core.SaveProfile(ctx, SaveProfileOptions{ID: "p", Label: "P", Provider: "gw", APIKey: "sk-rotated-key", Model: "qwen/qwen3-vl-8b", ConfigMode: "provider"}) + case "activate": + _, err = core.ActivateAgent(ctx, ActivateAgentOptions{AgentID: "codex", Provider: "gw", APIKey: "sk-rotated-key", Model: "qwen/qwen3-vl-8b"}) + case "install": + _, err = core.InstallAgents(ctx, InstallAgentsOptions{Agents: []string{"codex"}, Provider: "gw", APIKey: "sk-rotated-key", Model: "qwen/qwen3-vl-8b", Configure: true, SkipTest: true, Timeout: time.Minute}) + } + return err +} + +func TestCredentialWritesReportSkillEnvironmentFailureAndRetryTheSameKey(t *testing.T) { + for _, operation := range []string{"profile", "activate", "install"} { + t.Run(operation, func(t *testing.T) { + core, home := multimodalInstallCore(t, &multimodalGateway{listing: gatewayListing}) + core.status.Platform = platform.For("macos", "arm64") + installSkill(t, core, "qwen/qwen3-vl-8b", modality.Vision, "codex") + path := filepath.Join(home, ".codex", "config.toml") + before, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte("invalid = ["), 0o600); err != nil { + t.Fatal(err) + } + if err := writeMultimodalCredential(core, operation); err == nil || !strings.Contains(err.Error(), "generated Skills could not be updated") { + t.Fatalf("failed environment update was hidden: %v", err) + } + entry, err := core.GetProvider(context.Background(), "gw") + if err != nil || entry.APIKey != "sk-rotated-key" { + t.Fatal("the saved Provider credential was lost") + } + if err := os.WriteFile(path, before, 0o600); err != nil { + t.Fatal(err) + } + if err := writeMultimodalCredential(core, operation); err != nil { + t.Fatal(err) + } + if codexSet(t, home)["BOOTAGENT_GW_API_KEY"] != "sk-rotated-key" { + t.Fatal("retrying the same saved key did not repair the Skill environment") + } + }) + } +} + +func TestNativeSkillKeyUsesTheNativeEndpointWhenOpenAIRefreshFails(t *testing.T) { + core, home := jiekouCore(t, &multimodalGateway{listing: `{"data":[]}`}) + ctx := context.Background() + if check, err := core.ProbeMultimodal(ctx, "jiekou", "minimax-speech-2.8-turbo", modality.TextToSpeech, modality.AdapterNovitaAudio, false); err != nil || check.Status != ModelCheckAvailable { + t.Fatalf("probe = %#v, %v", check, err) + } + if result, err := core.InstallMultimodal(ctx, MultimodalInstallRequest{Provider: "jiekou", Model: "minimax-speech-2.8-turbo", Capability: modality.TextToSpeech, Agents: []string{"codex"}}); err != nil || !result.Results[0].EnvWritten { + t.Fatalf("install = %#v, %v", result, err) + } + path := filepath.Join(home, ".codex", "skills", "bootagent-text-to-speech", "SKILL.md") + if err := os.WriteFile(path, []byte("user edited this Skill"), 0o600); err != nil { + t.Fatal(err) + } + result, err := core.SaveProvider(ctx, provider.Entry{ID: "jiekou", Name: "JieKou.AI", BaseURL: "https://new.example/v1", APIKey: "sk-rotated-key"}, false, false) + if err != nil || result.Failures["codex"] == "" { + t.Fatalf("expected the user-edited tree to retain its refresh failure: %v", err) + } + if codexSet(t, home)["BOOTAGENT_JIEKOU_API_KEY"] != "sk-rotated-key" { + t.Fatal("the key was blocked despite an unchanged native request endpoint") + } +} + +func TestUnchangedCredentialDoesNotRequireAReadableSkillRegistry(t *testing.T) { + for _, operation := range []string{"profile", "activate", "install"} { + t.Run(operation, func(t *testing.T) { + core, _ := multimodalInstallCore(t, &multimodalGateway{listing: gatewayListing}) + core.status.Platform = platform.For("macos", "arm64") + installSkill(t, core, "qwen/qwen3-vl-8b", modality.Vision, "codex") + // Make the unchanged-key request match the saved Provider. + if err := core.providers.SaveKey(context.Background(), "gw", "sk-rotated-key"); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(core.skillStore().RegistryPath(), []byte("invalid"), 0o600); err != nil { + t.Fatal(err) + } + if err := writeMultimodalCredential(core, operation); err != nil { + t.Fatalf("the unchanged credential required an unrelated Registry: %v", err) + } + }) + } +} + +func TestMultimodalGeneratedOutputPreservesReinstallAndUninstall(t *testing.T) { + node, err := exec.LookPath("node") + if err != nil { + t.Skip("node is not installed") + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]any{"data": []any{map[string]string{"b64_json": "aW1hZ2U="}}}) + })) + defer server.Close() + core, home := multimodalInstallCore(t, &multimodalGateway{listing: gatewayListing}) + ctx := context.Background() + if _, err := core.SaveProvider(ctx, provider.Entry{ID: "gw", Name: "Gateway", BaseURL: server.URL + "/v1", APIKey: multimodalKey}, false, false); err != nil { + t.Fatal(err) + } + if check, err := core.ProbeMultimodal(ctx, "gw", "gpt-image-1", modality.ImageGeneration, modality.AdapterImages, true); err != nil || check.Status != ModelCheckAvailable { + t.Fatalf("probe = %#v, %v", check, err) + } + result := installSkill(t, core, "gpt-image-1", modality.ImageGeneration, "codex") + if !result.Results[0].EnvWritten { + t.Fatalf("install = %#v", result) + } + dir := filepath.Join(home, ".codex", "skills", "bootagent-image-generation") + before, err := skill.HashTree(ctx, dir) + if err != nil { + t.Fatal(err) + } + runCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + command := exec.CommandContext(runCtx, node, filepath.Join(dir, "scripts", "run.mjs"), "generate", "--prompt", "a lighthouse") + command.Dir = dir + command.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + home, "BOOTAGENT_GW_API_KEY=" + multimodalKey} + output, err := command.CombinedOutput() + if err != nil { + t.Fatalf("generated script failed: %v", err) + } + if !strings.HasPrefix(strings.TrimSpace(string(output)), filepath.Join(home, ".bootagent", "output")+string(os.PathSeparator)) { + t.Fatal("generation did not write to the separate output directory") + } + after, err := skill.HashTree(ctx, dir) + if err != nil || before != after { + t.Fatal("generation changed the managed Skill tree") + } + if result := installSkill(t, core, "gpt-image-1", modality.ImageGeneration, "codex"); !result.Results[0].Installed || result.Results[0].Error != "" { + t.Fatalf("reinstall = %#v", result) + } + if result := core.UninstallSkill(ctx, "bootagent-image-generation"); result.Error != "" { + t.Fatalf("uninstall = %#v", result) + } + if _, err := os.Stat(dir); !os.IsNotExist(err) { + t.Fatal("the generated Skill was not removed") + } +} + +func providerTransfer(t *testing.T) []byte { + t.Helper() + config, err := json.Marshal(map[string]any{"providers": map[string]any{"providers": []provider.Entry{{ID: "gw", Name: "Gateway", BaseURL: "https://new.example/v1", APIKey: "sk-rotated-key"}}}}) + if err != nil { + t.Fatal(err) + } + bundle, err := transfer.Build(map[string][]byte{"config.json": config}) + if err != nil { + t.Fatal(err) + } + return bundle +} + +func TestTransferProviderChangeRefreshesInstalledGeneratedSkills(t *testing.T) { + core, home := multimodalInstallCore(t, &multimodalGateway{listing: gatewayListing}) + installSkill(t, core, "qwen/qwen3-vl-8b", modality.Vision, "codex", "opencode") + if err := core.ImportTransferV2(context.Background(), providerTransfer(t)); err != nil { + t.Fatal(err) + } + cfg, err := skillgen.ReadConfig(filepath.Join(home, ".codex", "skills", "bootagent-vision")) + if err != nil || cfg.BaseURL != "https://new.example/v1" { + t.Fatal("the imported Provider endpoint did not reach its Skill") + } + if codexSet(t, home)["BOOTAGENT_GW_API_KEY"] != "sk-rotated-key" { + t.Fatal("the imported Provider credential did not reach its Skill") + } +} + +func TestTransferRollsBackGeneratedSkillChangesWhenEnvironmentWriteFails(t *testing.T) { + core, home := multimodalInstallCore(t, &multimodalGateway{listing: gatewayListing}) + installSkill(t, core, "qwen/qwen3-vl-8b", modality.Vision, "claude-code", "codex") + broken := filepath.Join(home, ".codex", "config.toml") + if err := os.WriteFile(broken, []byte("invalid = ["), 0o600); err != nil { + t.Fatal(err) + } + registryBefore, err := os.ReadFile(core.skillStore().RegistryPath()) + if err != nil { + t.Fatal(err) + } + if err := core.ImportTransferV2(context.Background(), providerTransfer(t)); err == nil { + t.Fatal("import reported success despite a failed Skill environment write") + } + entry, err := core.GetProvider(context.Background(), "gw") + if err != nil || entry.BaseURL != "https://gw.example/v1" || entry.APIKey != multimodalKey { + t.Fatal("the Provider was not rolled back") + } + for _, target := range []string{".claude", ".codex"} { + cfg, err := skillgen.ReadConfig(filepath.Join(home, target, "skills", "bootagent-vision")) + if err != nil || cfg.BaseURL != "https://gw.example/v1" { + t.Fatal("a published Skill was not rolled back") + } + } + if claudeEnv(t, home)["BOOTAGENT_GW_API_KEY"] != multimodalKey { + t.Fatal("a successful environment update was not rolled back") + } + registryAfter, _ := os.ReadFile(core.skillStore().RegistryPath()) + if string(registryBefore) != string(registryAfter) { + t.Fatal("the generated variant registry was not rolled back") + } +} + +func TestTransferNormalizedProviderIDKeepsGeneratedRollbackSnapshots(t *testing.T) { + core, home := multimodalInstallCore(t, &multimodalGateway{listing: gatewayListing}) + installSkill(t, core, "qwen/qwen3-vl-8b", modality.Vision, "claude-code", "codex") + if err := os.WriteFile(filepath.Join(home, ".codex", "config.toml"), []byte("invalid = ["), 0o600); err != nil { + t.Fatal(err) + } + config, err := json.Marshal(map[string]any{"providers": map[string]any{"providers": []provider.Entry{{ID: " gw ", Name: "Gateway", BaseURL: "https://new.example/v1", APIKey: "sk-rotated-key"}}}}) + if err != nil { + t.Fatal(err) + } + bundle, err := transfer.Build(map[string][]byte{"config.json": config}) + if err != nil { + t.Fatal(err) + } + if err := core.ImportTransferV2(context.Background(), bundle); err == nil { + t.Fatal("import reported success despite failed Skill environment refresh") + } + cfg, err := skillgen.ReadConfig(filepath.Join(home, ".claude", "skills", "bootagent-vision")) + if err != nil || cfg.BaseURL != "https://gw.example/v1" || claudeEnv(t, home)["BOOTAGENT_GW_API_KEY"] != multimodalKey { + t.Fatal("normalizing the imported ID skipped the generated Skill rollback snapshot") + } +} + +func TestProviderRefreshIgnoresUnrelatedStaleSkillFailures(t *testing.T) { + core, home := jiekouCore(t, &multimodalGateway{listing: gatewayListing}) + ctx := context.Background() + // The two Providers share both the guide pseudo-target and the native Agent. + if result, err := core.InstallMultimodal(ctx, MultimodalInstallRequest{Provider: "gw", Model: "qwen/qwen3-vl-8b", Capability: modality.Vision, Agents: []string{"codex"}, Guide: true}); err != nil || !result.Results[0].EnvWritten { + t.Fatalf("install = %#v, %v", result, err) + } + if result, err := core.InstallMultimodal(ctx, MultimodalInstallRequest{Provider: "jiekou", Model: "kling-v3.0-4k-t2v", Capability: modality.VideoGeneration, Agents: []string{"codex"}, Guide: true}); err != nil || !result.Results[0].EnvWritten { + t.Fatalf("install = %#v, %v", result, err) + } + // Simulate an independently changed Provider without synchronizing its Skill. + if _, err := core.providers.Save(ctx, provider.Entry{ID: "gw", Name: "Gateway", BaseURL: "https://old.example/v1", APIKey: multimodalKey}); err != nil { + t.Fatal(err) + } + result, err := core.SaveProvider(ctx, provider.Entry{ID: "jiekou", Name: "JieKou.AI", BaseURL: "https://api.highwayapi.ai/openai", APIKey: "sk-rotated-key"}, false, false) + if err != nil || len(result.Failures) != 0 { + t.Fatalf("unrelated stale Skill blocked the selected Provider refresh: %#v, %v", result.Failures, err) + } + if codexSet(t, home)["BOOTAGENT_JIEKOU_API_KEY"] != "sk-rotated-key" { + t.Fatal("the selected Provider key was not updated") + } +} + +func TestMultimodalInstallReportsBlockedFallbackDirectory(t *testing.T) { + core, home := multimodalInstallCore(t, &multimodalGateway{listing: gatewayListing}) + if err := os.WriteFile(filepath.Join(home, ".bootagent", "skill-env"), []byte("blocked"), 0o600); err != nil { + t.Fatal(err) + } + result := installSkill(t, core, "qwen/qwen3-vl-8b", modality.Vision, "opencode") + if !result.Results[0].Installed || result.Results[0].EnvWritten || result.Results[0].Error == "" { + t.Fatalf("fallback write failure was hidden: %#v", result) + } +} + +func TestFailedEndpointRefreshNeverWritesNewKeyToOldSkill(t *testing.T) { + core, home := multimodalInstallCore(t, &multimodalGateway{listing: gatewayListing}) + installSkill(t, core, "qwen/qwen3-vl-8b", modality.Vision, "codex", "opencode") + for _, target := range []string{".codex", ".config/opencode"} { + path := filepath.Join(home, target, "skills", "bootagent-vision", "SKILL.md") + if err := os.WriteFile(path, []byte("user edited this Skill"), 0o600); err != nil { + t.Fatal(err) + } + } + result, err := core.SaveProvider(context.Background(), provider.Entry{ID: "gw", Name: "Gateway", BaseURL: "https://new.example/v1", APIKey: "sk-rotated-key"}, false, false) + if err != nil || len(result.Failures) == 0 { + t.Fatalf("expected per-target refresh failures, got %v", err) + } + if codexSet(t, home)["BOOTAGENT_GW_API_KEY"] == "sk-rotated-key" { + t.Fatal("the new credential was written for a Skill still using the old endpoint") + } + if data, _ := os.ReadFile(filepath.Join(home, ".bootagent", "skill-env", "gw.env")); strings.Contains(string(data), "sk-rotated-key") { + t.Fatal("the new credential was written to the fallback for an old endpoint") + } +} diff --git a/internal/app/multimodal_install.go b/internal/app/multimodal_install.go index 00d5b1ff..aa15a1d5 100644 --- a/internal/app/multimodal_install.go +++ b/internal/app/multimodal_install.go @@ -158,6 +158,13 @@ func (u *UseCases) InstallMultimodal(ctx context.Context, request MultimodalInst u.writeMu.Lock() defer u.writeMu.Unlock() + current, err := u.providers.Get(target.ID) + if err != nil { + return result, err + } + if provider.Fingerprint(current) != provider.Fingerprint(target) || current.Name != target.Name { + return result, oneerrors.New(oneerrors.InvalidRequest, "The Provider changed while its model was being checked; retry the installation", oneerrors.WithRetryable(true)) + } targets, _, err := u.multimodalInstallTargets() if err != nil { return result, err @@ -457,15 +464,28 @@ var fallbackEnvFile = regexp.MustCompile(`^([a-z0-9][a-z0-9-]{0,63})\.env$`) // Codex get them in their own configuration; any other Agent a generated Skill // is published to reads the private fallback file. It returns failures by // Agent and never fails the caller's own change. Callers hold writeMu. -func (u *UseCases) reconcileSkillEnvLocked(ctx context.Context) map[string]string { +func (u *UseCases) reconcileSkillEnvLocked(ctx context.Context, providerIDs ...string) map[string]string { failures := map[string]string{} + selectedProvider := func(providerID string) bool { + return len(providerIDs) == 0 || slices.Contains(providerIDs, providerID) + } + loadFailure := func(message string) map[string]string { + if len(providerIDs) == 0 { + failures[GuideTarget] = message + return failures + } + for _, providerID := range providerIDs { + failures[providerID] = message + } + return failures + } registry, err := u.skillStore().Load() if err != nil { - return failures + return loadFailure("Cannot read the Skill Registry") } manifest, err := catalog.LoadEmbedded() if err != nil { - return failures + return loadFailure("Cannot read the Agent catalog") } keys := map[string]string{} keyFor := func(providerID string) string { @@ -483,10 +503,57 @@ func (u *UseCases) reconcileSkillEnvLocked(ctx context.Context) map[string]strin // desired[agent][variable] = key; fallback[provider] = agents that need it. desired := map[string]map[string]string{} fallback := map[string][]string{} - for _, variant := range u.generatedVariants(registry) { + blockedNative := map[string]map[string]bool{} + blockedFallback := map[string]bool{} + selectedAgents := map[string]bool{} + variants := u.generatedVariants(registry) + for _, variant := range variants { + if slices.Contains(providerIDs, variant.config.ProviderID) { + for _, agentID := range variant.targets { + selectedAgents[agentID] = true + } + } + } + rules, err := modality.Load() + if err != nil { + return loadFailure("Cannot read modality routes") + } + for _, variant := range variants { + entry, entryErr := u.providers.Get(variant.config.ProviderID) + // An endpoint refresh may fail for a user-edited tree. Never hand the + // new endpoint's key to a variant that still sends to the old one. Native + // adapters send to native_base_url, regardless of the OpenAI base snapshot. + matches := entryErr == nil && provider.OpenAIBaseURL(entry.BaseURL) == variant.config.BaseURL + if modality.IsNative(variant.config.Adapter) { + matches = entryErr == nil && rules.NativeBaseURL(entry.ID) == variant.config.NativeBaseURL + } + if !matches { + if variant.guide { + blockedFallback[variant.config.ProviderID] = true + if selectedProvider(variant.config.ProviderID) { + failures[GuideTarget] = "Skill still uses a previous Provider endpoint; reinstall it before updating its key" + } + } + for _, agentID := range variant.targets { + if selectedProvider(variant.config.ProviderID) { + failures[agentID] = "Skill still uses a previous Provider endpoint; reinstall it before updating its key" + } + if nativeSkillEnvAdapters[manifest.Agents[agentID].ConfigAdapter] { + if blockedNative[agentID] == nil { + blockedNative[agentID] = map[string]bool{} + } + blockedNative[agentID][variant.config.KeyEnv] = true + } else { + blockedFallback[variant.config.ProviderID] = true + } + } + continue + } if variant.guide { if keyFor(variant.config.ProviderID) == "" { - failures[GuideTarget] = "Provider " + variant.config.ProviderID + " has no saved API key" + if selectedProvider(variant.config.ProviderID) { + failures[GuideTarget] = "Provider " + variant.config.ProviderID + " has no saved API key" + } } else { fallback[variant.config.ProviderID] = append(fallback[variant.config.ProviderID], GuideTarget) } @@ -498,7 +565,9 @@ func (u *UseCases) reconcileSkillEnvLocked(ctx context.Context) map[string]strin } key := keyFor(variant.config.ProviderID) if key == "" { - failures[agentID] = "Provider " + variant.config.ProviderID + " has no saved API key" + if selectedProvider(variant.config.ProviderID) { + failures[agentID] = "Provider " + variant.config.ProviderID + " has no saved API key" + } continue } if nativeSkillEnvAdapters[agent.ConfigAdapter] { @@ -511,8 +580,24 @@ func (u *UseCases) reconcileSkillEnvLocked(ctx context.Context) map[string]strin } } } + for agentID, variables := range blockedNative { + for variable := range variables { + delete(desired[agentID], variable) + } + } + for providerID := range blockedFallback { + if selectedProvider(providerID) { + for _, agentID := range fallback[providerID] { + failures[agentID] = "Another Skill using this key still points at a previous Provider endpoint" + } + } + delete(fallback, providerID) + } writer := configWriter.NewWriter(u.status.Home, u.status.Platform.OS, u.filesystem) for _, agentID := range slices.Sorted(maps.Keys(manifest.Agents)) { + if len(providerIDs) > 0 && !selectedAgents[agentID] { + continue + } agent := manifest.Agents[agentID] if !nativeSkillEnvAdapters[agent.ConfigAdapter] { continue @@ -533,6 +618,9 @@ func (u *UseCases) reconcileSkillEnvLocked(ctx context.Context) map[string]strin } } for providerID, agents := range fallback { + if len(providerIDs) > 0 && !slices.Contains(providerIDs, providerID) { + continue + } path := configWriter.SkillEnvFilePath(u.status.Home, providerID) if err := writer.WriteSkillEnvFile(ctx, path, configWriter.SkillEnvName(providerID), keyFor(providerID)); err != nil { for _, agentID := range agents { @@ -547,10 +635,17 @@ func (u *UseCases) reconcileSkillEnvLocked(ctx context.Context) map[string]strin if match == nil || !entry.Type().IsRegular() { continue } + if len(providerIDs) > 0 && !slices.Contains(providerIDs, match[1]) { + continue + } if _, needed := fallback[match[1]]; !needed { - _ = configWriter.RemoveSkillEnvFile(filepath.Join(directory, entry.Name())) + if err := configWriter.RemoveSkillEnvFile(filepath.Join(directory, entry.Name())); err != nil { + failures[match[1]] = oneerrors.As(err).Message + } } } + } else if !os.IsNotExist(err) { + return loadFailure("Cannot inspect the Skill environment directory") } return failures } @@ -617,6 +712,9 @@ func (u *UseCases) refreshGeneratedSkillsLocked(ctx context.Context, target prov for _, agentID := range variant.targets { failures[agentID] = oneerrors.As(err).Message } + if variant.guide { + failures[GuideTarget] = oneerrors.As(err).Message + } continue } for agentID, failure := range published { diff --git a/internal/app/provider.go b/internal/app/provider.go index b0c7a430..f2759258 100644 --- a/internal/app/provider.go +++ b/internal/app/provider.go @@ -203,6 +203,7 @@ func (u *UseCases) SaveProvider(ctx context.Context, entry provider.Entry, creat // Nothing an Agent config carries has changed, so there is nothing to rewrite. if before.BaseFor(provider.ProtocolAnthropic) == saved.BaseFor(provider.ProtocolAnthropic) && before.BaseURL == saved.BaseURL && before.APIKey == saved.APIKey { + u.reapplyGeneratedSkillsLocked(ctx, saved, &result) return result, nil } result.Reapplied, result.Failures, err = u.reapplyProviderLocked(ctx, saved) @@ -218,16 +219,26 @@ func (u *UseCases) SaveProvider(ctx context.Context, entry provider.Entry, creat // in every Agent they are published to. Agents rewritten this way join // Reapplied; failures join Failures, keyed by Agent like the rest. func (u *UseCases) reapplyGeneratedSkillsLocked(ctx context.Context, saved provider.Entry, result *SaveProviderResult) { + if _, err := u.skillStore().Load(); err != nil { + if result.Failures == nil { + result.Failures = map[string]string{} + } + result.Failures[saved.ID] = "Cannot read the Skill Registry" + return + } using := u.generatedSkillsUsing(saved.ID) if len(using) == 0 { return } failures := u.refreshGeneratedSkillsLocked(ctx, saved) - for agentID, failure := range u.reconcileSkillEnvLocked(ctx) { + for agentID, failure := range u.reconcileSkillEnvLocked(ctx, saved.ID) { failures[agentID] = failure } touched := map[string]bool{} for _, variant := range using { + if variant.guide { + touched[GuideTarget] = true + } for _, agentID := range variant.targets { touched[agentID] = true } @@ -244,9 +255,58 @@ func (u *UseCases) reapplyGeneratedSkillsLocked(ctx context.Context, saved provi result.Reapplied = append(result.Reapplied, agentID) } } + if failure, failed := failures[saved.ID]; failed { + if result.Failures == nil { + result.Failures = map[string]string{} + } + result.Failures[saved.ID] = failure + } sort.Strings(result.Reapplied) } +// saveProviderKeyLocked covers credential updates accepted by Profile, Agent +// activation and install calls, which do not go through the Provider editor. +// Callers hold writeMu. A failed refresh is explicit; the saved Provider remains +// authoritative and obsolete Skills never receive its new credential. +func (u *UseCases) saveProviderKeyLocked(ctx context.Context, providerID, key string) error { + if providerID == "custom" || key == "" { + return nil + } + before, err := u.providers.Get(providerID) + if err != nil { + return err + } + if err := u.providers.SaveKey(ctx, providerID, key); err != nil { + return err + } + // An unchanged credential does not make a corrupt, unrelated Skill Registry + // a new prerequisite for applying an otherwise valid Profile or Agent config. + // With a readable Registry we still refresh, allowing same-key repair retries. + if before.APIKey == key { + if _, err := u.skillStore().Load(); err != nil { + return nil + } + } + saved, err := u.providers.Get(providerID) + if err != nil { + return err + } + result := SaveProviderResult{} + u.reapplyGeneratedSkillsLocked(ctx, saved, &result) + return generatedSkillRefreshError(result.Failures) +} + +func generatedSkillRefreshError(failures map[string]string) error { + if len(failures) == 0 { + return nil + } + var messages []string + for _, target := range slices.Sorted(maps.Keys(failures)) { + messages = append(messages, target+": "+failures[target]) + } + return oneerrors.New(oneerrors.ConfigWriteFailed, "Provider saved, but generated Skills could not be updated: "+strings.Join(messages, "; ")) +} + // reapplyProviderLocked rewrites each configured Agent that points at this // Provider, keeping its own model and Profile reference. Callers must hold // writeMu. diff --git a/internal/app/status.go b/internal/app/status.go index 24ef0e34..adc86ebb 100644 --- a/internal/app/status.go +++ b/internal/app/status.go @@ -629,7 +629,7 @@ func (u *UseCases) SaveProfile(ctx context.Context, options SaveProfileOptions) providerKey = target.APIKey } if strings.TrimSpace(providerKey) != "" { - if err := u.providers.SaveKey(ctx, options.Provider, providerKey); err != nil { + if err := u.saveProviderKeyLocked(ctx, options.Provider, providerKey); err != nil { return SaveProfileResult{}, err } } diff --git a/internal/app/transfer.go b/internal/app/transfer.go index b78886a2..f72d9d34 100644 --- a/internal/app/transfer.go +++ b/internal/app/transfer.go @@ -7,12 +7,15 @@ import ( "encoding/json" "errors" "fmt" + "maps" "os" "path/filepath" "slices" "sort" "strings" + "github.com/MaimoryLab/BootAgent/internal/catalog" + configWriter "github.com/MaimoryLab/BootAgent/internal/config" "github.com/MaimoryLab/BootAgent/internal/mcp" profileStore "github.com/MaimoryLab/BootAgent/internal/profile" "github.com/MaimoryLab/BootAgent/internal/provider" @@ -275,6 +278,43 @@ func (u *UseCases) ImportTransferV2WithOptions(ctx context.Context, data []byte, return err } created := make([]string, 0) + var generatedSnapshots []transferFileSnapshot + snapshotPaths := map[string]bool{} + snapshotGenerated := func(providerID string) error { + manifest, err := catalog.LoadEmbedded() + if err != nil { + return err + } + var paths []string + for _, variant := range u.generatedSkillsUsing(providerID) { + // Keep rollback limited to generated trees and credential files this + // Provider can change; never snapshot an entire Agent home. + paths = append(paths, filepath.Join(store.SkillsRoot(), variant.id)) + if variant.guide { + paths = append(paths, u.guideDir(variant.config.Capability)) + } + for _, agentID := range variant.targets { + agent := manifest.Agents[agentID] + paths = append(paths, filepath.Join(skillPath(u.status.Home, u.status.Platform.OS, agent), variant.id)) + if nativeSkillEnvAdapters[agent.ConfigAdapter] { + paths = append(paths, configPath(u.status.Home, u.status.Platform.OS, agent)) + } + } + paths = append(paths, configWriter.SkillEnvFilePath(u.status.Home, providerID)) + } + for _, path := range paths { + if path == "" || snapshotPaths[path] { + continue + } + snapshot, err := snapshotTree(path) + if err != nil { + return err + } + generatedSnapshots = append(generatedSnapshots, snapshot) + snapshotPaths[path] = true + } + return nil + } rollback := func() error { for _, p := range created { if err := os.RemoveAll(p); err != nil { @@ -282,6 +322,11 @@ func (u *UseCases) ImportTransferV2WithOptions(ctx context.Context, data []byte, } } rollbackCtx := context.Background() + for _, snapshot := range generatedSnapshots { + if err := snapshot.restore(); err != nil { + return err + } + } if err := store.Save(rollbackCtx, rollbackRegistry); err != nil { return err } @@ -303,6 +348,7 @@ func (u *UseCases) ImportTransferV2WithOptions(ctx context.Context, data []byte, return cause } configRaw := pkg.Files["config.json"] + changedProviders := map[string]provider.Entry{} if len(configRaw) == 0 { legacy := map[string]json.RawMessage{} if raw := pkg.Files["providers.json"]; len(raw) > 0 { @@ -329,13 +375,19 @@ func (u *UseCases) ImportTransferV2WithOptions(ctx context.Context, data []byte, return fail(errors.New("invalid providers section")) } for _, entry := range providers.Providers { + entry.ID = strings.TrimSpace(entry.ID) + if err := snapshotGenerated(entry.ID); err != nil { + return fail(err) + } existing, _ := u.providers.Get(entry.ID) if entry.APIKey == "" { entry.APIKey = existing.APIKey } - if _, e := u.providers.Save(ctx, entry); e != nil { + saved, e := u.providers.Save(ctx, entry) + if e != nil { return fail(e) } + changedProviders[saved.ID] = saved } } var profiles []ProfileSummary @@ -428,6 +480,15 @@ func (u *UseCases) ImportTransferV2WithOptions(ctx context.Context, data []byte, if err := store.Save(ctx, before); err != nil { return fail(err) } + // All package sections are valid before any installed Skill is refreshed. + // A partial refresh is rolled back along with the imported Provider records. + for _, providerID := range slices.Sorted(maps.Keys(changedProviders)) { + result := SaveProviderResult{} + u.reapplyGeneratedSkillsLocked(ctx, changedProviders[providerID], &result) + if err := generatedSkillRefreshError(result.Failures); err != nil { + return fail(err) + } + } return nil } From 09dba0be35d923ab4a6027903a6ee6624dd47110 Mon Sep 17 00:00:00 2001 From: yujiezhang-ops Date: Sat, 10 Oct 2026 11:19:04 +0800 Subject: [PATCH 17/18] fix: discard stale Profile model checks after a save A model check that resolved after a later save could still offer to switch back, and reverting overwrote the newer model. Every save now invalidates in-flight checks, and reverting first confirms the Profile still holds the checked model. Co-Authored-By: Claude Opus 5 (1M context) --- frontend/src/pages/AgentProfilePage.test.tsx | 61 +++++++++++++++++++- frontend/src/pages/AgentProfilePage.tsx | 37 ++++++++---- frontend/src/pages/ProfilesPage.test.tsx | 50 +++++++++++++++- frontend/src/pages/ProfilesPage.tsx | 17 ++++-- 4 files changed, 146 insertions(+), 19 deletions(-) diff --git a/frontend/src/pages/AgentProfilePage.test.tsx b/frontend/src/pages/AgentProfilePage.test.tsx index 0d6a23fc..dcf48717 100644 --- a/frontend/src/pages/AgentProfilePage.test.tsx +++ b/frontend/src/pages/AgentProfilePage.test.tsx @@ -1,4 +1,4 @@ -import { fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { act, fireEvent, render, screen, waitFor } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { MemoryRouter, Route, Routes } from "react-router-dom"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; @@ -67,7 +67,7 @@ function statusWith(profiles: ProfileSummary[]): StatusResponse { function renderPage(profiles: ProfileSummary[]) { mockState = { status: statusWith(profiles), statusState: "success" }; - render( + return render( } /> @@ -210,7 +210,10 @@ describe("AgentProfilePage", () => { // to, checked for this Agent only. it("checks on request after an inline edit and switches back", async () => { const save = vi.spyOn(api, "saveProfile") - .mockResolvedValueOnce({ ...saved({ model: "claude-opus-5-5" }), reapplied: ["codex"] }) + .mockImplementationOnce(async () => { + mockState.status!.profiles = [profile({ model: "claude-opus-5-5" })]; + return { ...saved({ model: "claude-opus-5-5" }), reapplied: ["codex"] }; + }) .mockResolvedValueOnce({ ...saved(), reapplied: ["codex"] }); const check = vi.spyOn(api, "checkProfileModel").mockResolvedValue(refusedCheck); renderPage([profile()]); @@ -340,6 +343,58 @@ describe("AgentProfilePage", () => { await waitFor(() => expect(button).not.toBeDisabled()); }); + it.each(["result", "error"])("discards a late model check %s after another Profile save", async (outcome) => { + let finish: (value: unknown) => void = () => undefined; + let reject: (error: Error) => void = () => undefined; + vi.spyOn(api, "checkProfileModel").mockImplementation(() => new Promise((resolve, fail) => { finish = resolve; reject = fail; }) as never); + vi.spyOn(api, "saveProfile").mockImplementation((request) => { + mockState.status!.profiles = [profile({ model: request.model })]; + return Promise.resolve(saved({ model: request.model })) as never; + }); + renderPage([profile({ model: "model-B" })]); + fireEvent.click(screen.getByRole("button", { name: "检测模型" })); + fireEvent.click(screen.getByRole("button", { name: "编辑 团队 PPIO" })); + fireEvent.change(screen.getByLabelText("模型"), { target: { value: "model-C" } }); + fireEvent.click(screen.getByRole("button", { name: "保存配置模版" })); + await waitFor(() => expect(screen.queryByRole("button", { name: "保存配置模版" })).toBeNull()); + await act(async () => { + if (outcome === "result") finish({ profileId: "team-ppio", model: "model-B", status: "unavailable", reason: "refused", agents: ["codex"] }); + else reject(new Error("late check failure")); + }); + expect(screen.queryByText("模型「model-B」不可用")).toBeNull(); + expect(screen.queryByText("late check failure")).toBeNull(); + expect(screen.getByRole("button", { name: "检测模型" })).not.toBeDisabled(); + }); + + it("does not start a model check after leaving while its inline save is pending", async () => { + let finish: (value: SaveProfileResult) => void = () => undefined; + vi.spyOn(api, "saveProfile").mockImplementation(() => new Promise((resolve) => { finish = resolve; }) as never); + const check = vi.spyOn(api, "checkProfileModel"); + const page = renderPage([profile()]); + await waitFor(() => expect(screen.getByLabelText("模型")).toHaveAttribute("placeholder", "搜索模型")); + fireEvent.change(screen.getByLabelText("模型"), { target: { value: "model-B" } }); + fireEvent.click(screen.getByRole("button", { name: "检测模型" })); + page.unmount(); + await act(async () => finish(saved({ model: "model-B" }))); + expect(check).not.toHaveBeenCalled(); + }); + + it("does not revert a Profile changed outside this editor after the check", async () => { + const save = vi.spyOn(api, "saveProfile").mockImplementation((request) => { + mockState.status!.profiles = [profile({ model: request.model })]; + return Promise.resolve(saved({ model: request.model })) as never; + }); + vi.spyOn(api, "checkProfileModel").mockResolvedValue(refusedCheck); + renderPage([profile()]); + await waitFor(() => expect(screen.getByLabelText("模型")).toHaveAttribute("placeholder", "搜索模型")); + fireEvent.change(screen.getByLabelText("模型"), { target: { value: "claude-opus-5-5" } }); + fireEvent.click(screen.getByRole("button", { name: "检测模型" })); + await screen.findByRole("alert"); + mockState.status!.profiles = [profile({ model: "model-C" })]; + fireEvent.click(screen.getByRole("button", { name: "改回 deepseek/deepseek-v3" })); + expect(save).toHaveBeenCalledOnce(); + }); + it("does not write when the model is unchanged or blank", async () => { const save = vi.spyOn(api, "saveProfile").mockResolvedValue(saved()); renderPage([profile()]); diff --git a/frontend/src/pages/AgentProfilePage.tsx b/frontend/src/pages/AgentProfilePage.tsx index e8fd979d..4da30611 100644 --- a/frontend/src/pages/AgentProfilePage.tsx +++ b/frontend/src/pages/AgentProfilePage.tsx @@ -1,5 +1,5 @@ import { Check, Pencil, Plus, Save, X } from "lucide-react"; -import { useEffect, useMemo, useState, type FormEvent } from "react"; +import { useEffect, useMemo, useRef, useState, type FormEvent } from "react"; import { useNavigate, useParams } from "react-router-dom"; import { api, describeFailure } from "../backend/api"; @@ -84,6 +84,14 @@ export function AgentProfilePage() { const [checked, setChecked] = useState(null); const [checking, setChecking] = useState(false); const [lastEdit, setLastEdit] = useState(null); + const checkGeneration = useRef(0); + const statusRef = useRef(status); + statusRef.current = status; + useEffect(() => { + setChecked(null); + setChecking(false); + return () => { checkGeneration.current += 1; }; + }, [agentId]); const profiles = useMemo(() => { if (!status) return []; @@ -157,7 +165,12 @@ export function AgentProfilePage() { * the Profile showed nothing at all. Nothing is sent to the model: that is * what 检测模型 is for. */ - const persistProfile = async (request: SaveRequest) => { + const persistProfile = async (request: SaveRequest, invalidateCheck = true) => { + if (invalidateCheck) { + checkGeneration.current += 1; + setChecking(false); + } + if (checked?.profileId === request.id) setChecked(null); const before = status.profiles.find((profile) => profile.id === request.id); const result = await api.saveProfile(request); const failures = Object.entries(result.failures ?? {}); @@ -167,8 +180,6 @@ export function AgentProfilePage() { message: failures[0][1] ?? "", })); } - // A result for this Profile describes the model it held before this save. - if (checked?.profileId === request.id) setChecked(null); const edit = before && before.provider === request.provider && before.model && before.model !== request.model ? { profileId: request.id, previousModel: before.model, request } : null; @@ -182,6 +193,8 @@ export function AgentProfilePage() { // 检测模型 once more if they want the old one confirmed. const revertModel = async () => { if (!checked?.previousModel || lastEdit?.profileId !== checked.profileId) return; + const current = statusRef.current?.profiles.find((profile) => profile.id === checked.profileId); + if (current?.model !== checked.model || current.provider !== lastEdit.request.provider) return; const request = { ...lastEdit.request, model: checked.previousModel }; setBusy(true); setFailure(""); @@ -261,7 +274,7 @@ export function AgentProfilePage() { * write a Profile per letter typed. Everything else about the Profile is passed * through unchanged, so this cannot quietly reset a field the editor owns. */ - const commitModel = async (profile: ProfileSummary, next: string): Promise => { + const commitModel = async (profile: ProfileSummary, next: string, invalidateCheck = true): Promise => { const model = next.trim(); setModelDraft(null); if (!model || model === (profile.model || "")) return { kind: "unchanged" }; @@ -279,7 +292,7 @@ export function AgentProfilePage() { context1M: Boolean(profile.context1M), configMode: "provider", protocol: profile.protocol || (app ? desktopProtocol(app) : catalog?.protocol || ""), - }); + }, invalidateCheck); await refreshStatus(); return { kind: "saved", model, edit }; } catch (error) { @@ -297,18 +310,22 @@ export function AgentProfilePage() { */ const checkModel = async () => { if (!selected || checking || busy || savingModel) return; + const generation = ++checkGeneration.current; setChecking(true); setFailure(""); try { - const flushed: CommitOutcome = modelDraft !== null ? await commitModel(selected, modelDraft) : { kind: "unchanged" }; - if (flushed.kind === "failed") return; + // This save belongs to the check; other saves and leaving the page + // invalidate its generation, including while this flush is awaiting. + const flushed: CommitOutcome = modelDraft !== null ? await commitModel(selected, modelDraft, false) : { kind: "unchanged" }; + if (flushed.kind === "failed" || checkGeneration.current !== generation) return; const check = await api.checkProfileModel(selected.id, [app?.id || owner]); + if (checkGeneration.current !== generation) return; const edit = flushed.kind === "saved" ? flushed.edit : lastEdit; setChecked({ ...check, previousModel: edit?.profileId === selected.id && edit.request.model === check.model ? edit.previousModel : undefined }); } catch (error) { - setFailure(describeFailure(error, t("无法检测模型"), t).message); + if (checkGeneration.current === generation) setFailure(describeFailure(error, t("无法检测模型"), t).message); } finally { - setChecking(false); + if (checkGeneration.current === generation) setChecking(false); } }; diff --git a/frontend/src/pages/ProfilesPage.test.tsx b/frontend/src/pages/ProfilesPage.test.tsx index 7ab6e98c..dac35d9c 100644 --- a/frontend/src/pages/ProfilesPage.test.tsx +++ b/frontend/src/pages/ProfilesPage.test.tsx @@ -1,4 +1,4 @@ -import { fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { act, fireEvent, render, screen, waitFor } from "@testing-library/react"; import { MemoryRouter, Route, Routes } from "react-router-dom"; import { afterEach, describe, expect, it, vi } from "vitest"; @@ -315,7 +315,10 @@ describe("ProfilesPage", () => { // Profile with only the model changed. it("names a refused model and switches back to the one the edit replaced", async () => { const save = vi.spyOn(api, "saveProfile") - .mockResolvedValueOnce({ profile: profile({ model: "claude-opus-5-5" }), reapplied: ["codex"], failures: null }) + .mockImplementationOnce(async () => { + mockState.status!.profiles = [profile({ model: "claude-opus-5-5" })]; + return { profile: profile({ model: "claude-opus-5-5" }), reapplied: ["codex"], failures: null }; + }) .mockResolvedValueOnce({ profile: profile(), reapplied: ["codex"], failures: null }); vi.spyOn(api, "checkProfileModel").mockResolvedValue({ profileId: "team-ppio", model: "claude-opus-5-5", status: "unavailable", reason: "refused", protocol: "responses", httpStatus: 404, @@ -403,6 +406,49 @@ describe("ProfilesPage", () => { expect(screen.getByRole("button", { name: "检查模型服务的 API Key" })).toBeTruthy(); }); + it.each(["result", "error"])("discards a late model check %s after another save", async (outcome) => { + let finish: (value: unknown) => void = () => undefined; + let reject: (error: Error) => void = () => undefined; + vi.spyOn(api, "checkProfileModel").mockImplementation(() => new Promise((resolve, fail) => { finish = resolve; reject = fail; }) as never); + vi.spyOn(api, "saveProfile").mockImplementation((request) => { + mockState.status!.profiles = [profile({ model: request.model })]; + return Promise.resolve(saved({ model: request.model })) as never; + }); + vi.spyOn(api, "models").mockResolvedValue({ ok: true, reachable: true, status: 200, error_code: "", retryable: false, models: [], message: "" }); + renderPage([profile({ model: "model-B" })]); + fireEvent.click(screen.getByRole("button", { name: "检测 团队 PPIO 的模型" })); + fireEvent.click(screen.getByRole("button", { name: "编辑 团队 PPIO" })); + fireEvent.change(screen.getByLabelText("模型"), { target: { value: "model-C" } }); + fireEvent.click(screen.getByRole("button", { name: "保存配置模版" })); + await waitFor(() => expect(screen.queryByRole("button", { name: "保存配置模版" })).toBeNull()); + await act(async () => { + if (outcome === "result") finish({ profileId: "team-ppio", model: "model-B", status: "unavailable", reason: "refused", agents: ["codex"] }); + else reject(new Error("late check failure")); + }); + expect(screen.queryByRole("alert")).toBeNull(); + expect(screen.queryByText("late check failure")).toBeNull(); + expect(screen.getByRole("button", { name: "检测 团队 PPIO 的模型" })).not.toBeDisabled(); + }); + + it("does not revert a Profile changed outside this editor after the check", async () => { + const save = vi.spyOn(api, "saveProfile").mockImplementation((request) => { + mockState.status!.profiles = [profile({ model: request.model })]; + return Promise.resolve(saved({ model: request.model })) as never; + }); + vi.spyOn(api, "checkProfileModel").mockResolvedValue({ profileId: "team-ppio", model: "model-B", status: "unavailable", reason: "refused", agents: ["codex"] }); + vi.spyOn(api, "models").mockResolvedValue({ ok: true, reachable: true, status: 200, error_code: "", retryable: false, models: [], message: "" }); + renderPage([profile()]); + fireEvent.click(screen.getByRole("button", { name: "编辑 团队 PPIO" })); + fireEvent.change(screen.getByLabelText("模型"), { target: { value: "model-B" } }); + fireEvent.click(screen.getByRole("button", { name: "保存配置模版" })); + await waitFor(() => expect(screen.queryByRole("button", { name: "保存配置模版" })).toBeNull()); + fireEvent.click(screen.getByRole("button", { name: "检测 团队 PPIO 的模型" })); + await screen.findByRole("alert"); + mockState.status!.profiles = [profile({ model: "model-C" })]; + fireEvent.click(screen.getByRole("button", { name: "改回 deepseek/deepseek-v3" })); + expect(save).toHaveBeenCalledOnce(); + }); + it("saves a Profile whose name was left blank", async () => { // The backend fills an empty label in from the existing value or the ID // (internal/profile/write.go:71-74), so requiring one here was stricter than diff --git a/frontend/src/pages/ProfilesPage.tsx b/frontend/src/pages/ProfilesPage.tsx index 53a45955..c03d72de 100644 --- a/frontend/src/pages/ProfilesPage.tsx +++ b/frontend/src/pages/ProfilesPage.tsx @@ -66,6 +66,9 @@ export function ProfilesPage() { const [checked, setChecked] = useState(null); const [checkingId, setCheckingId] = useState(""); const [lastEdit, setLastEdit] = useState(null); + const checkGeneration = useRef(0); + const statusRef = useRef(status); + statusRef.current = status; const editorRef = useRef(null); const editing = editor ? editor.originalId || "new" : ""; @@ -74,6 +77,7 @@ export function ProfilesPage() { useEffect(() => { if (editing) editorRef.current?.scrollIntoView?.({ block: "start" }); }, [editing]); + useEffect(() => () => { checkGeneration.current += 1; }, []); if (!status) { return ( @@ -172,10 +176,11 @@ export function ProfilesPage() { // the Agent's own config file moves with it. Shared by the editor's save and // the check result's "switch back", which is the same save with the old model. const persist = async (request: SaveRequest) => { + checkGeneration.current += 1; + setCheckingId(""); + if (checked?.profileId === request.id) setChecked(null); const before = status.profiles.find((profile) => profile.id === request.id); const result = await api.saveProfile(request); - // A result for this Profile describes the model it held before this save. - if (checked?.profileId === request.id) setChecked(null); setLastEdit(before && before.provider === request.provider && before.model && before.model !== request.model ? { profileId: request.id, previousModel: before.model, request } : null); @@ -222,16 +227,18 @@ export function ProfilesPage() { }; const checkModel = async (profileId: string) => { + const generation = ++checkGeneration.current; setCheckingId(profileId); setFailure(""); try { const check = await api.checkProfileModel(profileId); + if (checkGeneration.current !== generation) return; const edit = lastEdit?.profileId === profileId && lastEdit.request.model === check.model ? lastEdit : null; setChecked({ ...check, previousModel: edit?.previousModel }); } catch (error) { - setFailure(describeFailure(error, t("无法检测模型"), t).message); + if (checkGeneration.current === generation) setFailure(describeFailure(error, t("无法检测模型"), t).message); } finally { - setCheckingId(""); + if (checkGeneration.current === generation) setCheckingId(""); } }; @@ -239,6 +246,8 @@ export function ProfilesPage() { // 检测模型 once more if they want the old one confirmed. const revertModel = async () => { if (!checked?.previousModel || lastEdit?.profileId !== checked.profileId) return; + const current = statusRef.current?.profiles.find((profile) => profile.id === checked.profileId); + if (current?.model !== checked.model || current.provider !== lastEdit.request.provider) return; const request = { ...lastEdit.request, model: checked.previousModel }; setBusy(true); setFailure(""); From 2b6db887aca58c67a3d996c19471ccb26a3bf4e5 Mon Sep 17 00:00:00 2001 From: yujiezhang-ops Date: Sat, 10 Oct 2026 11:19:04 +0800 Subject: [PATCH 18/18] fix: keep multimodal install and probe results visible - The install button opens the dialog even with no install targets, so users can still generate the guide files for other Agents. - A manual probe's result or error is shown unless the same model's result is already shown in its capability row. - The install dialog cannot be closed while an installation is running, so its result and the follow-up refresh are never lost. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/components/MultimodalSection.test.tsx | 58 +++++++++++++++++-- frontend/src/components/MultimodalSection.tsx | 32 ++++++---- 2 files changed, 74 insertions(+), 16 deletions(-) diff --git a/frontend/src/components/MultimodalSection.test.tsx b/frontend/src/components/MultimodalSection.test.tsx index 133048f7..0be2e3c5 100644 --- a/frontend/src/components/MultimodalSection.test.tsx +++ b/frontend/src/components/MultimodalSection.test.tsx @@ -1,4 +1,4 @@ -import { fireEvent, render, screen, waitFor, within } from "@testing-library/react"; +import { act, fireEvent, render, screen, waitFor, within } from "@testing-library/react"; import { MemoryRouter } from "react-router-dom"; import { beforeEach, describe, expect, it, vi } from "vitest"; @@ -236,11 +236,61 @@ describe("MultimodalSection", () => { await waitFor(() => expect(changed).toHaveBeenCalled()); }); - it("offers no install when no Agent can take the Skill", async () => { + it("allows guide-only installation when no Agent can take the Skill", async () => { detectMultimodal.mockResolvedValue(detection({ targets: [] })); + installMultimodal.mockResolvedValue({ skillId: "bootagent-vision", results: [{ agent: "guide", installed: true, envWritten: true }] }); renderSection(); - const button = await screen.findByRole("button", { name: /^安装$/ }); - expect((button as HTMLButtonElement).disabled).toBe(true); + fireEvent.click(await screen.findByRole("button", { name: /^安装$/ })); + const dialog = screen.getByRole("dialog"); + expect(within(dialog).getByRole("button", { name: "安装" })).toBeDisabled(); + fireEvent.click(within(dialog).getByLabelText("其他 Agent（生成说明文件）")); + fireEvent.click(within(dialog).getByRole("button", { name: "安装" })); + await waitFor(() => expect(installMultimodal).toHaveBeenCalledWith({ provider: "novita", model: "qwen/qwen3-vl-8b", capability: "vision", agents: [], guide: true })); + }); + + it.each(["network", "key", "transient"])("shows a manual model's %s result despite an existing capability row", async (reason) => { + detectMultimodal.mockResolvedValue(detection({ providers: [{ ...gateway, models: [{ id: "known-whisper", capabilities: [{ id: "speech-to-text", source: "catalog", adapter: "openai-audio-transcriptions", supported: true, installable: true, probe: "minimal" }] }] }] })); + probeMultimodal.mockResolvedValue({ providerId: "novita", model: "my-whisper", capability: "speech-to-text", adapter: "openai-audio-transcriptions", status: reason === "key" ? "unavailable" : "unverified", reason }); + renderSection(); + await screen.findByText("Novita"); + fireEvent.change(screen.getByLabelText("能力"), { target: { value: "speech-to-text" } }); + fireEvent.change(screen.getByLabelText("模型 ID"), { target: { value: "my-whisper" } }); + const manual = screen.getByText("手动填写模型 ID").closest("details") as HTMLElement; + fireEvent.click(within(manual).getByRole("button", { name: /^实测$/ })); + expect(await within(manual).findByText(reason === "key" ? "「my-whisper」不支持语音识别" : "无法验证「my-whisper」的语音识别")).toBeTruthy(); + }); + + it("shows a rejected manual probe even when no capability row exists", async () => { + detectMultimodal.mockResolvedValue(detection({ providers: [{ ...gateway, models: [] }] })); + probeMultimodal.mockRejectedValue(new Error("manual request failed")); + renderSection(); + await screen.findByText("Novita"); + fireEvent.change(screen.getByLabelText("能力"), { target: { value: "speech-to-text" } }); + fireEvent.change(screen.getByLabelText("模型 ID"), { target: { value: "my-whisper" } }); + const manual = screen.getByText("手动填写模型 ID").closest("details") as HTMLElement; + fireEvent.click(within(manual).getByRole("button", { name: /^实测$/ })); + expect(await within(manual).findByRole("alert")).toHaveTextContent("manual request failed"); + }); + + it("keeps the install dialog open through cancel, close and Esc while installing", async () => { + let finish: (value: unknown) => void = () => undefined; + detectMultimodal.mockResolvedValue(detection()); + installMultimodal.mockImplementation(() => new Promise((resolve) => { finish = resolve; })); + const changed = renderSection(); + fireEvent.click(await screen.findByRole("button", { name: /^安装$/ })); + const dialog = screen.getByRole("dialog"); + fireEvent.click(within(dialog).getByRole("button", { name: "安装" })); + expect(within(dialog).getByRole("button", { name: "取消" })).toBeDisabled(); + expect(within(dialog).getByTitle("关闭")).toBeDisabled(); + fireEvent.click(within(dialog).getByRole("button", { name: "取消" })); + fireEvent.click(within(dialog).getByTitle("关闭")); + fireEvent(dialog, new Event("cancel", { bubbles: true, cancelable: true })); + expect(screen.getByRole("dialog")).toBe(dialog); + await act(async () => finish({ skillId: "bootagent-vision", results: [{ agent: "codex", installed: true, envWritten: true }] })); + expect(within(dialog).getByText("已安装")).toBeTruthy(); + fireEvent.click(within(dialog).getByRole("button", { name: "完成" })); + await waitFor(() => expect(detectMultimodal).toHaveBeenCalledTimes(2)); + expect(changed).toHaveBeenCalledOnce(); }); it("offers no probe for a capability with no live check", async () => { diff --git a/frontend/src/components/MultimodalSection.tsx b/frontend/src/components/MultimodalSection.tsx index cc2bce83..6173e4e7 100644 --- a/frontend/src/components/MultimodalSection.tsx +++ b/frontend/src/components/MultimodalSection.tsx @@ -159,7 +159,7 @@ export function groupInstalled(installed: InstalledMultimodal[]): Array<{ skillI .map(([skillId, variants]) => ({ skillId, capability, variants, agents: [...new Set(variants.flatMap((variant) => variant.agents))].sort() }))); } -type ProbeState = { checking: boolean; result?: MultimodalProbe; error?: string }; +type ProbeState = { model: string; checking: boolean; result?: MultimodalProbe; error?: string }; type ProbeTarget = { provider: MultimodalProvider; capability: MultimodalCapabilityId; model: string; adapter: string; kind: MultimodalProbeKind }; type InstallTarget = { provider: MultimodalProvider; capability: MultimodalCapabilityId; model: string }; @@ -205,17 +205,17 @@ export function MultimodalSection({ agentLabels = {}, onSkillsChanged }: Multimo const runProbe = (target: ProbeTarget, confirmed: boolean) => { const key = rowKey(target.provider.id, target.capability); - setProbes((current) => ({ ...current, [key]: { checking: true } })); + setProbes((current) => ({ ...current, [key]: { model: target.model, checking: true } })); void api.probeMultimodal(target.provider.id, target.model, target.capability, target.adapter, confirmed) .then((result) => { - setProbes((current) => ({ ...current, [key]: { checking: false, result } })); + setProbes((current) => ({ ...current, [key]: { model: target.model, checking: false, result } })); // Show the probed model in its row; a conclusive verdict is cached by // the backend and outranks every other source once re-read. setSelected((current) => ({ ...current, [key]: target.model })); return detect(); }) .catch((reason: unknown) => { - setProbes((current) => ({ ...current, [key]: { checking: false, error: describeError(reason, t("无法向模型服务发送验证请求。")).message } })); + setProbes((current) => ({ ...current, [key]: { model: target.model, checking: false, error: describeError(reason, t("无法向模型服务发送验证请求。")).message } })); }); }; const probe = (target: ProbeTarget) => (target.kind === "confirm" ? setConfirmProbe(target) : runProbe(target, false)); @@ -292,6 +292,10 @@ export function MultimodalSection({ agentLabels = {}, onSkillsChanged }: Multimo const manualState = manual[entry.id]; const manualRoutes = detection?.manual ?? []; const manualRoute = manualRoutes.find((route) => route.capability === (manualState?.capability ?? manualRoutes[0]?.capability)); + const manualProbeState = manualRoute ? probes[rowKey(entry.id, manualRoute.capability)] : undefined; + const manualGroup = groups.find((group) => group.id === manualRoute?.capability); + const manualChosen = manualGroup?.options.find((option) => option.model === selected[rowKey(entry.id, manualRoute?.capability ?? "")]) ?? manualGroup?.options[0]; + const manualResultInRow = manualChosen?.model === manualProbeState?.model; return (
    @@ -333,7 +337,7 @@ export function MultimodalSection({ agentLabels = {}, onSkillsChanged }: Multimo /> ) : null} {install.ready ? ( - @@ -342,7 +346,7 @@ export function MultimodalSection({ agentLabels = {}, onSkillsChanged }: Multimo

    {t("视频理解要先用 ffmpeg 抽帧。BootAgent 没有在 PATH 上找到 ffmpeg；请在 Agent 运行的环境里安装它。")}

    ) : null} {verdict ? : null} - {state?.error ?

    {state.error}

    : null} + {state?.error && state.model === chosen.model ?

    {state.error}

    : null} ); })} @@ -367,9 +371,10 @@ export function MultimodalSection({ agentLabels = {}, onSkillsChanged }: Multimo /> ) : null}
    - {manualRoute && probes[rowKey(entry.id, manualRoute.capability)]?.result && !groups.some((group) => group.id === manualRoute.capability) - ? + {manualProbeState?.result && !manualResultInRow + ? : null} + {manualProbeState?.error && !manualResultInRow ?

    {manualProbeState.error}

    : null} ) : null} @@ -541,9 +546,12 @@ function InstallDialog({ target, targets, fallback, current, providerName, nameO }; const title = t("安装{capability} Skill", { capability }); + const close = () => { + if (!busy) (result ? onDone : onCancel)(); + }; return ( - -

    {title}

    + +

    {title}

    {result ? ( <>
      @@ -566,7 +574,7 @@ function InstallDialog({ target, targets, fallback, current, providerName, nameO ) : ( <>

      {t("用「{model}」（{provider}）为 Agent 提供{capability}。", { model: target.model, provider: providerName(target.provider.id), capability })}

      -
      +
      {t("安装到")} {targets.map((agent) => (