diff --git a/cmd/launcher/internal/config.go b/cmd/launcher/internal/config.go new file mode 100644 index 000000000000..a6e886dc09ec --- /dev/null +++ b/cmd/launcher/internal/config.go @@ -0,0 +1,162 @@ +// SPDX-License-Identifier: MIT +package launcher + +import ( + "bytes" + "encoding/json" + "fmt" +) + +// Config retains the standalone Go fields for existing launcher callers. The +// JSON representation is exclusively the versioned, separated profile contract. +type Config struct { + SchemaVersion int + Mode string + ModelsPath string + BackendsPath string + Address string + AutoStart *bool + StartOnBoot bool // Deprecated: inert compatibility field. + LogLevel string + EnvironmentVars map[string]string + ShowWelcome *bool + Legacy LegacyConfig + Worker WorkerConfig +} + +type LegacyConfig struct { + StartOnBoot *bool `json:"start_on_boot,omitempty"` +} +type SharedConfig struct { + ModelsPath string `json:"models_path"` + BackendsPath string `json:"backends_path"` + LogLevel string `json:"log_level"` + AutoStart *bool `json:"auto_start_server"` + ShowWelcome *bool `json:"show_welcome"` +} +type StandaloneConfig struct { + Address string `json:"address"` + EnvironmentVars map[string]string `json:"environment_vars"` +} +type WorkerConfig struct { + Connection WorkerConnectionConfig `json:"connection"` + NATS WorkerNATSConfig `json:"nats"` + EnvironmentVars map[string]string `json:"environment_vars"` +} +type WorkerConnectionConfig struct { + FrontendURL string `json:"frontend_url"` + RegistrationToken string `json:"registration_token"` + AdvertisedAddress string `json:"advertised_address"` + NodeName string `json:"node_name"` + GRPCMaxPort *int `json:"grpc_max_port,omitempty"` +} +type WorkerNATSConfig struct { + URL string `json:"url"` + TLSCA string `json:"tls_ca"` + TLSCert string `json:"tls_cert"` + TLSKey string `json:"tls_key"` + JWT string `json:"jwt"` + UserSeed string `json:"user_seed"` +} +type configDocument struct { + SchemaVersion int `json:"schema_version"` + Mode string `json:"mode"` + Legacy LegacyConfig `json:"legacy"` + Shared SharedConfig `json:"shared"` + Standalone StandaloneConfig `json:"standalone"` + Worker WorkerConfig `json:"worker"` +} + +func (c Config) MarshalJSON() ([]byte, error) { + if c.SchemaVersion != 0 && c.SchemaVersion != 2 { + return nil, fmt.Errorf("unsupported launcher settings version; upgrade the launcher") + } + mode := c.Mode + if mode == "" { + mode = "standalone" + } + if mode != "standalone" && mode != "worker" { + return nil, fmt.Errorf("unknown launcher mode; select a supported mode in settings") + } + legacy := c.Legacy + if legacy.StartOnBoot == nil && c.StartOnBoot { + value := true + legacy.StartOnBoot = &value + } + return json.Marshal(configDocument{2, mode, legacy, + SharedConfig{c.ModelsPath, c.BackendsPath, c.LogLevel, c.AutoStart, c.ShowWelcome}, + StandaloneConfig{c.Address, c.EnvironmentVars}, c.Worker}) +} + +func (c *Config) UnmarshalJSON(data []byte) error { + var d configDocument + if err := json.Unmarshal(data, &d); err != nil { + return fmt.Errorf("invalid launcher settings JSON; repair launcher.json or restore a backup") + } + if bytes.Equal(bytes.TrimSpace(data), []byte("null")) { + return fmt.Errorf("launcher settings must be a JSON object") + } + if d.SchemaVersion != 0 && d.SchemaVersion != 1 && d.SchemaVersion != 2 { + return fmt.Errorf("unsupported launcher settings version; upgrade the launcher") + } + if d.Mode != "" && d.Mode != "standalone" && d.Mode != "worker" { + return fmt.Errorf("unknown launcher mode; select a supported mode in settings") + } + if d.SchemaVersion < 2 { + var legacy struct { + SharedConfig + StandaloneConfig + StartOnBoot *bool `json:"start_on_boot"` + } + if err := json.Unmarshal(data, &legacy); err != nil { + return fmt.Errorf("invalid legacy launcher settings; repair launcher.json or restore a backup") + } + d.Shared, d.Standalone = legacy.SharedConfig, legacy.StandaloneConfig + d.Legacy.StartOnBoot = legacy.StartOnBoot + } + if d.Mode == "" { + d.Mode = "standalone" + } + next := Config{SchemaVersion: 2, Mode: d.Mode, Legacy: d.Legacy, Worker: d.Worker, + ModelsPath: d.Shared.ModelsPath, BackendsPath: d.Shared.BackendsPath, LogLevel: d.Shared.LogLevel, + AutoStart: d.Shared.AutoStart, ShowWelcome: d.Shared.ShowWelcome, + Address: d.Standalone.Address, EnvironmentVars: d.Standalone.EnvironmentVars} + if d.Legacy.StartOnBoot != nil { + next.StartOnBoot = *d.Legacy.StartOnBoot + } + *c = next + return nil +} + +func (c *Config) clone() *Config { + if c == nil { + return nil + } + next := *c + copyBool := func(p *bool) *bool { + if p == nil { + return nil + } + v := *p + return &v + } + copyMap := func(m map[string]string) map[string]string { + if m == nil { + return nil + } + n := make(map[string]string, len(m)) + for k, v := range m { + n[k] = v + } + return n + } + next.AutoStart, next.ShowWelcome = copyBool(c.AutoStart), copyBool(c.ShowWelcome) + next.Legacy.StartOnBoot = copyBool(c.Legacy.StartOnBoot) + next.EnvironmentVars = copyMap(c.EnvironmentVars) + next.Worker.EnvironmentVars = copyMap(c.Worker.EnvironmentVars) + if c.Worker.Connection.GRPCMaxPort != nil { + v := *c.Worker.Connection.GRPCMaxPort + next.Worker.Connection.GRPCMaxPort = &v + } + return &next +} diff --git a/cmd/launcher/internal/config_store.go b/cmd/launcher/internal/config_store.go new file mode 100644 index 000000000000..b6299a201df5 --- /dev/null +++ b/cmd/launcher/internal/config_store.go @@ -0,0 +1,182 @@ +// SPDX-License-Identifier: MIT +package launcher + +import ( + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + + "golang.org/x/sys/unix" +) + +// Never include filesystem paths or JSON decoder payloads in these errors: +// both may contain credentials supplied in settings. +func settingsIOError(action string) error { + return fmt.Errorf("cannot %s launcher.json securely; ensure ~/.localai is an owned, non-symlink directory writable only by you and launcher.json is an owned regular file (0600); check disk space and permissions", action) +} + +func configDirectory() (*os.File, error) { + home, err := os.UserHomeDir() + if err != nil { + return nil, settingsIOError("locate") + } + path := filepath.Join(home, ".localai") + if err := os.Mkdir(path, 0700); err != nil && !os.IsExist(err) { + return nil, settingsIOError("create") + } + fd, err := unix.Open(path, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0) + if err != nil { + return nil, settingsIOError("open directory for") + } + dir := os.NewFile(uintptr(fd), "launcher settings directory") + var st unix.Stat_t + if err := unix.Fstat(fd, &st); err != nil || int64(st.Uid) != int64(os.Geteuid()) || st.Mode&0022 != 0 { + dir.Close() + return nil, settingsIOError("trust directory for") + } + return dir, nil +} + +// The directory descriptor pins the parent and O_NOFOLLOW prevents following a +// replaced destination. Hard links are refused before chmod to avoid changing +// another file. No permissions on model/backend directories are changed. +func openConfig(dir *os.File) (*os.File, error) { + fd, err := unix.Openat(int(dir.Fd()), "launcher.json", unix.O_RDONLY|unix.O_NOFOLLOW|unix.O_NONBLOCK|unix.O_CLOEXEC, 0) + if err != nil { + return nil, err + } + f := os.NewFile(uintptr(fd), "launcher.json") + var st unix.Stat_t + if err = unix.Fstat(fd, &st); err != nil || st.Mode&unix.S_IFMT != unix.S_IFREG || int64(st.Uid) != int64(os.Geteuid()) || st.Nlink != 1 { + f.Close() + return nil, settingsIOError("trust") + } + if err = f.Chmod(0600); err != nil { + f.Close() + return nil, settingsIOError("protect") + } + if err = unix.Fstat(fd, &st); err != nil || st.Mode&07777 != 0600 { + f.Close() + return nil, settingsIOError("verify permissions for") + } + return f, nil +} + +func (l *Launcher) loadConfig() (err error) { + defer func() { l.configError = err }() + dir, err := configDirectory() + if err != nil { + return err + } + defer dir.Close() + f, err := openConfig(dir) + if errors.Is(err, unix.ENOENT) { + return l.saveConfig() + } + if err != nil { + return settingsIOError("read") + } + defer f.Close() + data, err := io.ReadAll(f) + if err != nil { + return settingsIOError("read") + } + var candidate Config + if err := json.Unmarshal(data, &candidate); err != nil { + return fmt.Errorf("cannot load launcher settings; repair launcher.json or restore a backup using a supported schema and mode") + } + // Migration must be durable before making this profile eligible for startup. + if err := l.persistConfig(&candidate); err != nil { + return err + } + l.config = &candidate + return nil +} + +func (l *Launcher) saveConfig() error { + err := l.persistConfig(l.config) + l.configError = err + return err +} + +func (l *Launcher) persistConfig(candidate *Config) error { + if candidate == nil { + return fmt.Errorf("launcher settings are missing") + } + data, err := json.MarshalIndent(candidate, "", " ") + if err != nil { + return fmt.Errorf("cannot save launcher settings: %w", err) + } + dir, err := configDirectory() + if err != nil { + return err + } + defer dir.Close() + existing, err := openConfig(dir) + if err != nil && !errors.Is(err, unix.ENOENT) { + return settingsIOError("replace") + } + if existing != nil { + existing.Close() + } + // A random name with exclusive creation keeps temporary files private; + // openat keeps all operations + // relative to the verified parent, including on Linux and macOS. + name, err := temporaryConfigName() + if err != nil { + return settingsIOError("create temporary") + } + fd, err := unix.Openat(int(dir.Fd()), name, unix.O_WRONLY|unix.O_CREAT|unix.O_EXCL|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0600) + if err != nil { + return settingsIOError("create temporary") + } + defer unix.Unlinkat(int(dir.Fd()), name, 0) + f := os.NewFile(uintptr(fd), "temporary launcher settings") + defer f.Close() + if err := f.Chmod(0600); err != nil { + return settingsIOError("protect temporary") + } + info, err := f.Stat() + if err != nil || info.Mode().Perm() != 0600 { + return settingsIOError("verify temporary permissions for") + } + if _, err := f.Write(data); err != nil { + return settingsIOError("write") + } + if err := f.Sync(); err != nil { + return settingsIOError("flush") + } + if err := f.Close(); err != nil { + return settingsIOError("close temporary") + } + // Recheck immediately before rename. The parent disallows other-user writes; + // protection against the same user or root is not part of this boundary. + existing, err = openConfig(dir) + if err != nil && !errors.Is(err, unix.ENOENT) { + return settingsIOError("replace") + } + if existing != nil { + existing.Close() + } + if err := unix.Renameat(int(dir.Fd()), name, int(dir.Fd()), "launcher.json"); err != nil { + return settingsIOError("replace") + } + candidate.SchemaVersion = 2 + if candidate.Mode == "" { + candidate.Mode = "standalone" + } + return nil +} + +func temporaryConfigName() (string, error) { + var random [16]byte + if _, err := rand.Read(random[:]); err != nil { + return "", err + } + return ".launcher-" + hex.EncodeToString(random[:]) + ".tmp", nil +} diff --git a/cmd/launcher/internal/config_test.go b/cmd/launcher/internal/config_test.go new file mode 100644 index 000000000000..490c3805ef66 --- /dev/null +++ b/cmd/launcher/internal/config_test.go @@ -0,0 +1,261 @@ +// SPDX-License-Identifier: MIT +package launcher_test + +import ( + "bytes" + "encoding/json" + "io" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + + "fyne.io/fyne/v2" + "fyne.io/fyne/v2/app" + launcher "github.com/mudler/LocalAI/cmd/launcher/internal" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Secure launcher configuration", func() { + var home, path string + var l *launcher.Launcher + var configApp fyne.App + BeforeEach(func() { + // Keep Fyne's asynchronous settings watcher outside per-spec HOME. + if configApp == nil { + configApp = app.NewWithID("config.test") + } + originalHome, err := os.UserHomeDir() + Expect(err).NotTo(HaveOccurred()) + GinkgoT().Setenv("XDG_CONFIG_HOME", filepath.Join(originalHome, ".config")) + home = GinkgoT().TempDir() + GinkgoT().Setenv("HOME", home) + path = filepath.Join(home, ".localai", "launcher.json") + Expect(os.MkdirAll(filepath.Dir(path), 0755)).To(Succeed()) + l = launcher.NewLauncher(nil, nil, configApp) + }) + It("migrates all legacy settings to separate schema v2 profiles without logging values", func() { + input := `{"models_path":"MODELS","backends_path":"BACKENDS","address":"localhost:8123","log_level":"debug","auto_start_server":false,"show_welcome":false,"start_on_boot":true,"environment_vars":{"API_KEY":"environment-secret"}}` + input = strings.ReplaceAll(input, "MODELS", filepath.Join(home, "models-secret")) + input = strings.ReplaceAll(input, "BACKENDS", filepath.Join(home, "backends-secret")) + Expect(os.WriteFile(path, []byte(input), 0644)).To(Succeed()) + var logs bytes.Buffer + original := log.Writer() + log.SetOutput(&logs) + defer log.SetOutput(original) + Expect(l.Initialize()).To(Succeed()) + Expect(l.ShouldAutoStartServer()).To(BeFalse()) + data, err := os.ReadFile(path) + Expect(err).NotTo(HaveOccurred()) + var saved map[string]any + Expect(json.Unmarshal(data, &saved)).To(Succeed()) + Expect(saved["schema_version"]).To(Equal(float64(2))) + Expect(saved["mode"]).To(Equal("standalone")) + Expect(saved["shared"]).To(HaveKeyWithValue("models_path", filepath.Join(home, "models-secret"))) + Expect(saved["shared"]).To(HaveKeyWithValue("auto_start_server", false)) + Expect(saved["shared"]).To(HaveKeyWithValue("show_welcome", false)) + Expect(saved["shared"]).To(HaveKeyWithValue("log_level", "debug")) + Expect(l.GetConfig().Worker).To(Equal(launcher.WorkerConfig{})) + Expect(l.BuildRunArgs()).To(ContainElements("run", "--address", "localhost:8123")) + Expect(saved["legacy"]).To(HaveKeyWithValue("start_on_boot", true)) + Expect(saved["standalone"]).To(HaveKeyWithValue("environment_vars", map[string]any{"API_KEY": "environment-secret"})) + Expect(saved).NotTo(HaveKey("environment_vars")) + for _, secret := range []string{filepath.Join(home, "models-secret"), filepath.Join(home, "backends-secret"), "localhost:8123", "environment-secret"} { + Expect(logs.String()).NotTo(ContainSubstring(secret)) + } + info, err := os.Stat(path) + Expect(err).NotTo(HaveOccurred()) + Expect(info.Mode().Perm()).To(Equal(os.FileMode(0600))) + }) + It("retains startup tri-state and ignores obsolete auto_start", func() { + for _, input := range []string{`{}`, `{"auto_start":false}`, `{"auto_start_server":false}`, `{"auto_start_server":true}`} { + var c launcher.Config + Expect(json.Unmarshal([]byte(input), &c)).To(Succeed()) + Expect(l.SetConfig(&c)).To(Succeed()) + Expect(l.Initialize()).To(Succeed()) + Expect(l.ShouldAutoStartServer()).To(Equal(c.AutoStart == nil || *c.AutoStart)) + data, err := os.ReadFile(path) + Expect(err).NotTo(HaveOccurred()) + var saved map[string]any + Expect(json.Unmarshal(data, &saved)).To(Succeed()) + Expect(saved).To(HaveKey("shared")) + shared := saved["shared"].(map[string]any) + if c.AutoStart == nil { + Expect(shared["auto_start_server"]).To(BeNil()) + } else { + Expect(shared["auto_start_server"]).To(Equal(*c.AutoStart)) + } + } + }) + It("rejects a symlink destination without changing the target or active settings", func() { + target := filepath.Join(home, "target") + Expect(os.WriteFile(target, []byte("unchanged"), 0644)).To(Succeed()) + Expect(os.Symlink(target, path)).To(Succeed()) + previous := l.GetConfig().Address + c := l.GetConfig() + c.Address = "changed" + Expect(l.SetConfig(c)).NotTo(Succeed()) + Expect(l.GetConfig().Address).To(Equal(previous)) + Expect(l.ShouldAutoStartServer()).To(BeFalse()) + data, err := os.ReadFile(target) + Expect(err).NotTo(HaveOccurred()) + Expect(string(data)).To(Equal("unchanged")) + Expect(l.Initialize()).NotTo(Succeed()) + }) + It("rejects unknown schema and mode without exposing their content", func() { + for _, input := range []string{`{"schema_version":99,"mode":"standalone"}`, `{"schema_version":2,"mode":"secret-mode"}`} { + Expect(os.WriteFile(path, []byte(input), 0600)).To(Succeed()) + err := l.Initialize() + Expect(err).To(HaveOccurred()) + Expect(err.Error()).NotTo(ContainSubstring("secret-mode")) + Expect(l.ShouldAutoStartServer()).To(BeFalse()) + } + }) + It("round-trips worker secrets and independent environments without logging or fallback", func() { + input := `{"schema_version":2,"mode":"worker","shared":{"auto_start_server":false},"standalone":{"address":"localhost:8123","environment_vars":{"STANDALONE":"standalone-secret"}},"worker":{"connection":{"frontend_url":"https://example.test","registration_token":"token-secret","advertised_address":"worker.test:50052","node_name":"node","grpc_max_port":50100},"nats":{"url":"tls://nats.test:4222","tls_ca":"ca.pem","tls_cert":"cert.pem","tls_key":"key.pem","jwt":"jwt-secret","user_seed":"seed-secret"},"environment_vars":{"WORKER":"worker-secret"}}}` + var c launcher.Config + Expect(json.Unmarshal([]byte(input), &c)).To(Succeed()) + var logs bytes.Buffer + original := log.Writer() + log.SetOutput(&logs) + defer log.SetOutput(original) + Expect(l.SetConfig(&c)).To(Succeed()) + Expect(l.Initialize()).To(Succeed()) + Expect(l.ShouldAutoStartServer()).To(BeFalse()) + Expect(l.StartLocalAI()).NotTo(Succeed(), "no runtime is installed here, so nothing may start") + Expect(l.IsRunning()).To(BeFalse()) + worker := l.GetConfig().Worker + for _, mode := range []string{"standalone", "worker", "standalone"} { + draft := l.GetConfig() + draft.Mode = mode + Expect(l.SetConfig(draft)).To(Succeed()) + Expect(l.GetConfig().Worker).To(Equal(worker)) + Expect(l.GetConfig().EnvironmentVars).To(Equal(map[string]string{"STANDALONE": "standalone-secret"})) + } + for _, secret := range []string{"token-secret", "jwt-secret", "seed-secret", "worker-secret", "standalone-secret"} { + Expect(logs.String()).NotTo(ContainSubstring(secret)) + } + draft := l.GetConfig() + draft.Worker.Connection.RegistrationToken = "" + draft.Worker.NATS.JWT = "" + draft.Worker.NATS.UserSeed = "" + Expect(l.SetConfig(draft)).To(Succeed()) + Expect(l.GetConfig().Worker.Connection.RegistrationToken).To(BeEmpty()) + }) + It("returns isolated drafts and atomically replaces the old inode", func() { + c := l.GetConfig() + c.EnvironmentVars = map[string]string{"KEY": "original"} + Expect(l.SetConfig(c)).To(Succeed()) + c.EnvironmentVars["KEY"] = "external mutation" + draft := l.GetConfig() + draft.EnvironmentVars["KEY"] = "cancelled" + Expect(l.GetConfig().EnvironmentVars["KEY"]).To(Equal("original")) + old, err := os.Open(path) + Expect(err).NotTo(HaveOccurred()) + defer old.Close() + before, err := os.ReadFile(path) + Expect(err).NotTo(HaveOccurred()) + Expect(l.SetConfig(draft)).To(Succeed()) + oldData, err := io.ReadAll(old) + Expect(err).NotTo(HaveOccurred()) + Expect(oldData).To(Equal(before)) + after, err := os.ReadFile(path) + Expect(err).NotTo(HaveOccurred()) + Expect(after).NotTo(Equal(before)) + files, err := filepath.Glob(filepath.Join(filepath.Dir(path), ".launcher-*.tmp")) + Expect(err).NotTo(HaveOccurred()) + Expect(files).To(BeEmpty()) + }) + It("rejects unsafe parent permissions, directories and hard links", func() { + Expect(os.Chmod(filepath.Dir(path), 0777)).To(Succeed()) + Expect(l.SetConfig(l.GetConfig())).NotTo(Succeed()) + Expect(os.Chmod(filepath.Dir(path), 0755)).To(Succeed()) + Expect(os.Mkdir(path, 0700)).To(Succeed()) + Expect(l.SetConfig(l.GetConfig())).NotTo(Succeed()) + Expect(os.Remove(path)).To(Succeed()) + target := filepath.Join(home, "target") + Expect(os.WriteFile(target, []byte("untouched"), 0644)).To(Succeed()) + Expect(os.Link(target, path)).To(Succeed()) + Expect(l.Initialize()).NotTo(Succeed()) + info, err := os.Stat(target) + Expect(err).NotTo(HaveOccurred()) + Expect(info.Mode().Perm()).To(Equal(os.FileMode(0644))) + }) + It("does not follow a symlink configuration directory", func() { + Expect(os.Remove(filepath.Dir(path))).To(Succeed()) + target := filepath.Join(home, "elsewhere") + Expect(os.Mkdir(target, 0700)).To(Succeed()) + Expect(os.Symlink(target, filepath.Dir(path))).To(Succeed()) + Expect(l.SetConfig(l.GetConfig())).NotTo(Succeed()) + entries, err := os.ReadDir(target) + Expect(err).NotTo(HaveOccurred()) + Expect(entries).To(BeEmpty()) + }) + It("hardens malformed files before decoding and never includes their payload in errors", func() { + for _, input := range []string{`{"secret-error":`, `{"schema_version":"secret-error"}`, ` null `} { + Expect(os.WriteFile(path, []byte(input), 0644)).To(Succeed()) + Expect(os.Chmod(path, 0644)).To(Succeed()) + err := l.Initialize() + Expect(err).To(HaveOccurred()) + Expect(err.Error()).NotTo(ContainSubstring("secret-error")) + info, err := os.Stat(path) + Expect(err).NotTo(HaveOccurred()) + Expect(info.Mode().Perm()).To(Equal(os.FileMode(0600))) + data, err := os.ReadFile(path) + Expect(err).NotTo(HaveOccurred()) + Expect(string(data)).To(Equal(input)) + } + }) + + It("refuses foreign ownership and real permission failures", func() { + if os.Getenv("LAUNCHER_CONFIG_PERMISSION_CHILD") == "1" { + // The parent supplies an inaccessible file and an unwritable directory. + foreign := os.Getenv("LAUNCHER_CONFIG_FOREIGN_HOME") + GinkgoT().Setenv("HOME", foreign) + err := l.SetConfig(l.GetConfig()) + Expect(err).To(HaveOccurred()) + Expect(l.ShouldAutoStartServer()).To(BeFalse()) + GinkgoT().Setenv("HOME", os.Getenv("LAUNCHER_CONFIG_READONLY_HOME")) + Expect(l.SetConfig(l.GetConfig())).NotTo(Succeed()) + return + } + if os.Geteuid() != 0 { + Expect(os.Chmod(filepath.Dir(path), 0500)).To(Succeed()) + defer os.Chmod(filepath.Dir(path), 0700) + Expect(l.SetConfig(l.GetConfig())).NotTo(Succeed()) + return + } + Expect(os.WriteFile(path, []byte(`{}`), 0600)).To(Succeed()) + Expect(os.Chown(path, 65534, 65534)).To(Succeed()) + Expect(l.Initialize()).NotTo(Succeed()) + Expect(os.Chown(path, 0, 0)).To(Succeed()) + // Run the same assertion without root's DAC override in the isolated runner. + scratch, err := os.MkdirTemp("", "config-permission-") + Expect(err).NotTo(HaveOccurred()) + defer os.RemoveAll(scratch) + Expect(os.Chmod(scratch, 0755)).To(Succeed()) + foreign := filepath.Join(scratch, "foreign") + readonly := filepath.Join(scratch, "readonly") + for _, h := range []string{foreign, readonly} { + Expect(os.MkdirAll(filepath.Join(h, ".localai"), 0755)).To(Succeed()) + } + Expect(os.Chown(readonly, 65534, 65534)).To(Succeed()) + Expect(os.Chown(filepath.Join(readonly, ".localai"), 65534, 65534)).To(Succeed()) + Expect(os.Chmod(filepath.Join(readonly, ".localai"), 0500)).To(Succeed()) + executable, err := os.Executable() + Expect(err).NotTo(HaveOccurred()) + binary, err := os.ReadFile(executable) + Expect(err).NotTo(HaveOccurred()) + helper := filepath.Join(scratch, "config.test") + Expect(os.WriteFile(helper, binary, 0755)).To(Succeed()) + cmd := exec.Command(helper, "-test.run=TestLauncher", "-ginkgo.focus=refuses foreign ownership and real permission failures", "-ginkgo.no-color") + cmd.Env = append(os.Environ(), "LAUNCHER_CONFIG_PERMISSION_CHILD=1", "LAUNCHER_CONFIG_FOREIGN_HOME="+foreign, "LAUNCHER_CONFIG_READONLY_HOME="+readonly) + cmd.SysProcAttr = &syscall.SysProcAttr{Credential: &syscall.Credential{Uid: 65534, Gid: 65534}} + output, err := cmd.CombinedOutput() + Expect(err).NotTo(HaveOccurred(), string(output)) + }) + +}) diff --git a/cmd/launcher/internal/launch.go b/cmd/launcher/internal/launch.go new file mode 100644 index 000000000000..3362a07ddaa3 --- /dev/null +++ b/cmd/launcher/internal/launch.go @@ -0,0 +1,279 @@ +// SPDX-License-Identifier: MIT +package launcher + +import ( + "fmt" + "net" + "net/url" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "unicode" + + "github.com/nats-io/jwt/v2" + "github.com/nats-io/nkeys" +) + +// launchDescription is a value snapshot: accessors never return its backing slices. +// It is not permission to start a worker: runtime/ownership gates are still absent. +type launchDescription struct { + args, env, secrets []string + dir, credentials string + // role is roleStandalone or roleWorker. + role string + // ledger is the backend process ledger a worker writes; empty otherwise. + ledger string +} + +const ( + roleStandalone = "standalone" + roleWorker = "worker" +) + +func (d launchDescription) Args() []string { return append([]string(nil), d.args...) } +func (d launchDescription) Env() []string { return append([]string(nil), d.env...) } +func (d launchDescription) Dir() string { return d.dir } +func (d launchDescription) CredentialMode() string { return d.credentials } + +// Prevent accidental diagnostic formatting of the child environment. +func (d launchDescription) String() string { return "launcher description (redacted)" } +func (d launchDescription) GoString() string { return d.String() } + +// Includes worker.Config aliases, global CLI path/role aliases and the reserved +// launcher observability namespace. Non-reserved accelerator options survive. +const reservedLaunchKeys = `LOCALAI_ADDR LOCALAI_SERVE_ADDR LOCALAI_HTTP_ADDR LOCALAI_ADVERTISE_ADDR LOCALAI_ADVERTISE_HTTP_ADDR LOCALAI_GRPC_MAX_PORT LOCALAI_REGISTER_TO LOCALAI_NODE_NAME LOCALAI_REGISTRATION_TOKEN LOCALAI_REGISTRATION_REQUIRE_AUTH LOCALAI_DISTRIBUTED_REQUIRE_AUTH LOCALAI_NATS_REQUIRE_AUTH LOCALAI_NATS_URL LOCALAI_NATS_JWT LOCALAI_NATS_USER_SEED LOCALAI_NATS_TLS_CA LOCALAI_NATS_TLS_CERT LOCALAI_NATS_TLS_KEY LOCALAI_MODELS_PATH MODELS_PATH LOCALAI_BACKENDS_PATH BACKENDS_PATH LOCALAI_BACKENDS_SYSTEM_PATH LOCALAI_ADDRESS ADDRESS LOCALAI_DATA_PATH DATA_PATH LOCALAI_CONFIG_DIR LOCALAI_GENERATED_CONTENT_PATH GENERATED_CONTENT_PATH LOCALAI_UPLOAD_PATH UPLOAD_PATH LOCALAI_P2P_TOKEN P2P_TOKEN LOCALAI_P2P_NETWORK_ID LOCALAI_P2P LOCALAI_WORKER LOCALAI_ROLE LOCALAI_HEARTBEAT_INTERVAL LOCALAI_LOG_LEVEL LOG_LEVEL LOCALAI_DEBUG DEBUG LOCALAI_CONFIG_PATH CONFIG_PATH LOCALAI_LOCALAI_CONFIG_DIR LOCALAI_P2P_DISABLE_DHT LOCALAI_TOKEN TOKEN BACKEND_SYSTEM_PATH LOCALAI_DOWNLOAD_STAGING_PATH DOWNLOAD_STAGING_PATH P2P P2P_NETWORK_ID P2P_DHT_INTERVAL P2P_OTP_INTERVAL LOCALAI_P2P_DHT_INTERVAL LOCALAI_P2P_OTP_INTERVAL` + +func reservedLaunchKey(k string) bool { + return strings.HasPrefix(k, "LOCALAI_LAUNCHER_") || strings.Contains(" "+reservedLaunchKeys+" ", " "+k+" ") +} +func controls(s string) bool { return strings.ContainsFunc(s, unicode.IsControl) } +func decimalPort(s string) (int, error) { + if s == "" || strings.Trim(s, "0123456789") != "" { + return 0, fmt.Errorf("invalid decimal port") + } + n, e := strconv.Atoi(s) + if e != nil || n < 1 || n > 65535 { + return 0, fmt.Errorf("port outside 1..65535") + } + return n, nil +} +func validHost(s string) bool { + if ip := net.ParseIP(s); ip != nil { + return !ip.IsUnspecified() + } + if len(s) > 253 || s == "" { + return false + } + for _, label := range strings.Split(strings.TrimSuffix(s, "."), ".") { + if label == "" || len(label) > 63 || label[0] == '-' || label[len(label)-1] == '-' { + return false + } + for _, r := range label { + if !(r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-') { + return false + } + } + } + return true +} +func launchURL(raw string, nats bool) (string, error) { + bad := fmt.Errorf("invalid connection URL; check scheme, host and port; credentials, query and fragment are forbidden") + if strings.ContainsAny(raw, "?#") || strings.ContainsFunc(raw, unicode.IsSpace) || controls(raw) { + return "", bad + } + u, e := url.Parse(raw) + if e != nil || u.Opaque != "" || u.User != nil || !validHost(u.Hostname()) || strings.ContainsFunc(u.Path, unicode.IsSpace) || controls(u.Path) || strings.Contains(u.Host, "%") { + return "", bad + } + if nats { + if (u.Scheme != "tls" && u.Scheme != "nats") || (u.Path != "" && u.Path != "/") { + return "", bad + } + } else if u.Scheme != "https" && u.Scheme != "http" { + return "", bad + } + if strings.HasSuffix(u.Host, ":") { + return "", bad + } + if u.Port() != "" { + if _, e = decimalPort(u.Port()); e != nil { + return "", bad + } + } + u.Path = strings.TrimRight(u.Path, "/") + u.RawPath = "" + return u.String(), nil +} +func buildLaunch(input *Config, dataPath string, inherited []string, plaintextAcknowledged bool) (launchDescription, error) { + d := launchDescription{} + fail := func(s string) (launchDescription, error) { return launchDescription{}, fmt.Errorf("%s", s) } + c := input.clone() + if c == nil { + return fail("launcher settings missing") + } + if c.SchemaVersion != 0 && c.SchemaVersion != 2 { + return fail("unsupported launcher settings version") + } + worker := c.Mode == "worker" + if !worker && c.Mode != "" && c.Mode != "standalone" { + return fail("unknown launcher mode") + } + profile := c.EnvironmentVars + if worker { + profile = c.Worker.EnvironmentVars + } + env := map[string]string{} + for _, entry := range inherited { + k, v, ok := strings.Cut(entry, "=") + if ok && (!worker || !reservedLaunchKey(k)) { + env[k] = v + } + } + for k, v := range profile { + if k == "" || strings.ContainsAny(k, "=\x00") || strings.ContainsRune(v, 0) { + return fail("invalid advanced environment entry") + } + if worker && reservedLaunchKey(k) { + return fail("reserved worker environment override; use connection settings") + } + env[k] = v + } + d.role = roleStandalone + if !worker { + d.args = standaloneArgs(c, dataPath) + } else { + d.role = roleWorker + // The worker keeps its ledger next to its models directory. + d.ledger = filepath.Join(filepath.Clean(filepath.Join(c.ModelsPath, "..")), "data", "worker-processes.json") + w := c.Worker + a := w.Connection + n := w.NATS + front, e := launchURL(a.FrontendURL, false) + if e != nil { + return d, e + } + broker, e := launchURL(n.URL, true) + if e != nil { + return d, e + } + if !plaintextAcknowledged && (strings.HasPrefix(front, "http:") || strings.HasPrefix(broker, "nats:")) { + return fail("plaintext connections require protected-network acknowledgement") + } + host, p, e := net.SplitHostPort(a.AdvertisedAddress) + if e != nil || !validHost(host) { + return fail("invalid advertised host:port") + } + base, e := decimalPort(p) + if e != nil || base < 2 { + return fail("worker base port must be 2..65535") + } + if a.GRPCMaxPort != nil && (*a.GRPCMaxPort < base || *a.GRPCMaxPort > 65535) { + return fail("maximum gRPC port must be base..65535") + } + if strings.TrimSpace(a.RegistrationToken) == "" || controls(a.RegistrationToken) || controls(a.NodeName) || controls(n.JWT) || controls(n.UserSeed) { + return fail("registration token required; names and credentials must not contain controls") + } + if (n.JWT == "") != (n.UserSeed == "") { + return fail("static NATS credentials require both JWT and user seed") + } + d.credentials = "registration-issued" + if n.JWT != "" { + claims, e := jwt.DecodeUserClaims(n.JWT) + if e != nil { + return fail("invalid static NATS user JWT") + } + kp, e := nkeys.FromSeed([]byte(n.UserSeed)) + if e != nil { + return fail("invalid static NATS user seed") + } + defer kp.Wipe() + pub, e := kp.PublicKey() + if e != nil || !nkeys.IsValidPublicUserKey(pub) || claims.Subject != pub { + return fail("static NATS JWT and user seed do not match") + } + vr := jwt.CreateValidationResults() + claims.Validate(vr) + if vr.IsBlocking(true) { + return fail("static NATS user JWT is expired or invalid") + } + d.credentials = "static (no automatic renewal)" + env["LOCALAI_NATS_JWT"] = n.JWT + env["LOCALAI_NATS_USER_SEED"] = n.UserSeed + } + if (n.TLSCert == "") != (n.TLSKey == "") { + return fail("NATS client certificate and key must be paired") + } + if (n.TLSCA != "" || n.TLSCert != "") && !strings.HasPrefix(broker, "tls:") { + return fail("NATS TLS files require a tls:// endpoint") + } + for _, p := range []string{c.ModelsPath, c.BackendsPath, dataPath} { + if strings.TrimSpace(p) == "" || controls(p) { + return fail("worker storage paths must be nonempty and contain no controls") + } + } + if c.LogLevel != "" && c.LogLevel != "error" && c.LogLevel != "warn" && c.LogLevel != "info" && c.LogLevel != "debug" && c.LogLevel != "trace" { + return fail("invalid worker log level") + } + d.dir = dataPath + d.args = []string{"worker", "--register-to", front, "--nats-url", broker, "--addr", a.AdvertisedAddress, "--models-path", c.ModelsPath, "--backends-path", c.BackendsPath, "--distributed-require-auth"} + if c.LogLevel != "" { + d.args = append(d.args, "--log-level", c.LogLevel) + } + if a.NodeName != "" { + d.args = append(d.args, "--node-name", a.NodeName) + } + if a.GRPCMaxPort != nil { + d.args = append(d.args, "--grpc-max-port", strconv.Itoa(*a.GRPCMaxPort)) + } + for _, f := range []struct{ flag, path string }{{"--nats-tls-ca", n.TLSCA}, {"--nats-tls-cert", n.TLSCert}, {"--nats-tls-key", n.TLSKey}} { + if f.path == "" { + continue + } + if controls(f.path) { + return fail("invalid NATS TLS file") + } + // Resolve against the launcher CWD before validation: the child uses + // dataPath as its CWD and must open the same path we checked. + absolutePath, e := filepath.Abs(f.path) + if e != nil { + return fail("invalid NATS TLS file") + } + f.path = absolutePath + st, e := os.Stat(f.path) + if e != nil || !st.Mode().IsRegular() { + return fail("NATS TLS file must be readable and regular") + } + file, e := os.Open(f.path) + if e != nil { + return fail("NATS TLS file must be readable and regular") + } + st, e = file.Stat() + file.Close() + if e != nil || !st.Mode().IsRegular() { + return fail("NATS TLS file must be readable and regular") + } + d.args = append(d.args, f.flag, f.path) + } + env["LOCALAI_REGISTRATION_TOKEN"] = a.RegistrationToken + } + // Snapshot values for redaction, including non-reserved secrets supplied by users. + d.secrets = []string{c.Worker.Connection.RegistrationToken, c.Worker.NATS.JWT, c.Worker.NATS.UserSeed} + keys := make([]string, 0, len(env)) + for k, v := range env { + keys = append(keys, k) + key := strings.ToUpper(k) + if v != "" && (strings.Contains(key, "TOKEN") || strings.Contains(key, "SECRET") || strings.Contains(key, "PASSWORD") || strings.Contains(key, "JWT") || strings.Contains(key, "SEED") || strings.Contains(key, "API_KEY")) { + d.secrets = append(d.secrets, v) + } + } + sort.Strings(keys) + for _, k := range keys { + d.env = append(d.env, k+"="+env[k]) + } + return d, nil +} +func standaloneArgs(c *Config, dataPath string) []string { + return []string{"run", "--models-path", c.ModelsPath, "--backends-path", c.BackendsPath, "--address", c.Address, "--log-level", c.LogLevel, "--data-path", filepath.Join(dataPath, "data"), "--localai-config-dir", filepath.Join(dataPath, "configuration"), "--generated-content-path", filepath.Join(dataPath, "generated"), "--upload-path", filepath.Join(dataPath, "uploads")} +} diff --git a/cmd/launcher/internal/launch_test.go b/cmd/launcher/internal/launch_test.go new file mode 100644 index 000000000000..bdd8251264da --- /dev/null +++ b/cmd/launcher/internal/launch_test.go @@ -0,0 +1,262 @@ +// SPDX-License-Identifier: MIT +package launcher + +import ( + "fmt" + "github.com/alecthomas/kong" + cliContext "github.com/mudler/LocalAI/core/cli/context" + "github.com/mudler/LocalAI/core/services/worker" + "github.com/nats-io/jwt/v2" + "github.com/nats-io/nkeys" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "os" + "path/filepath" + "strings" +) + +var _ = Describe("Secure launcher launch", func() { + var c *Config + BeforeEach(func() { + c = &Config{Mode: "worker", ModelsPath: "/models", BackendsPath: "/backends", Worker: WorkerConfig{Connection: WorkerConnectionConfig{FrontendURL: "https://frontend.example/prefix/", RegistrationToken: "registration-secret", AdvertisedAddress: "[2001:db8::1]:50051"}, NATS: WorkerNATSConfig{URL: "tls://nats.example:4222"}}} + }) + It("parses the actual generic worker Config and global context CLI tags", func() { + c.LogLevel = "info" + d, e := buildLaunch(c, "/data", nil, false) + Expect(e).NotTo(HaveOccurred()) + var cli struct { + cliContext.Context `embed:""` + Worker worker.Config `cmd:""` + } + parser, e := kong.New(&cli, kong.Vars{"basepath": "/data", "backends": "[]", "galleries": "[]"}) + Expect(e).NotTo(HaveOccurred()) + GinkgoT().Setenv("LOCALAI_REGISTRATION_TOKEN", c.Worker.Connection.RegistrationToken) + _, e = parser.Parse(d.Args()) + Expect(e).NotTo(HaveOccurred()) + Expect(cli.Worker.NatsAuthRequired()).To(BeTrue()) + Expect(cli.Worker.RegistrationAuthRequired()).To(BeTrue()) + Expect(cli.Worker.RegistrationToken).To(Equal(c.Worker.Connection.RegistrationToken)) + Expect(cli.Worker.RegisterTo).To(Equal("https://frontend.example/prefix")) + Expect(cli.Worker.Addr).To(Equal(c.Worker.Connection.AdvertisedAddress)) + Expect(cli.Worker.GRPCMaxPort).To(BeZero()) + }) + It("accepts a matching static user JWT and seed only in the child environment", func() { + account, e := nkeys.CreateAccount() + Expect(e).NotTo(HaveOccurred()) + defer account.Wipe() + user, e := nkeys.CreateUser() + Expect(e).NotTo(HaveOccurred()) + defer user.Wipe() + pub, e := user.PublicKey() + Expect(e).NotTo(HaveOccurred()) + claims := jwt.NewUserClaims(pub) + token, e := claims.Encode(account) + Expect(e).NotTo(HaveOccurred()) + seed, e := user.Seed() + Expect(e).NotTo(HaveOccurred()) + c.Worker.NATS.JWT = token + c.Worker.NATS.UserSeed = string(seed) + d, e := buildLaunch(c, "/data", nil, false) + Expect(e).NotTo(HaveOccurred()) + Expect(d.CredentialMode()).To(ContainSubstring("no automatic renewal")) + Expect(d.Env()).To(ContainElement("LOCALAI_NATS_JWT=" + token)) + Expect(strings.Join(d.Args(), " ") + fmt.Sprintf("%+v %#v", d, d)).NotTo(ContainSubstring(token)) + }) + It("creates unique owner-only logs and refuses unsafe directories", func() { + dir := GinkgoT().TempDir() + f, e := secureLogFile(dir) + Expect(e).NotTo(HaveOccurred()) + defer f.Close() + st, e := f.Stat() + Expect(e).NotTo(HaveOccurred()) + Expect(st.Mode().Perm()).To(Equal(os.FileMode(0600))) + other := GinkgoT().TempDir() + Expect(os.Symlink(dir, filepath.Join(other, "logs"))).To(Succeed()) + _, e = secureLogFile(other) + Expect(e).To(HaveOccurred()) + }) + + It("builds an isolated authenticated generic worker description", func() { + d, err := buildLaunch(c, "/data", []string{"LOCALAI_NATS_JWT=hostile", "MODELS_PATH=/evil", "CUDA_VISIBLE_DEVICES=0"}, false) + Expect(err).NotTo(HaveOccurred()) + Expect(d.Args()).To(ContainElements("worker", "--distributed-require-auth", "https://frontend.example/prefix", "--addr", "[2001:db8::1]:50051")) + Expect(strings.Join(d.Args(), " ")).NotTo(ContainSubstring("registration-secret")) + Expect(d.Env()).To(ContainElements("LOCALAI_REGISTRATION_TOKEN=registration-secret", "CUDA_VISIBLE_DEVICES=0")) + Expect(strings.Join(d.Env(), " ")).NotTo(ContainSubstring("hostile")) + Expect(d.CredentialMode()).To(Equal("registration-issued")) + args := d.Args() + args[0] = "run" + c.Worker.Connection.RegistrationToken = "changed" + Expect(d.Args()[0]).To(Equal("worker")) + Expect(d.Env()).To(ContainElement("LOCALAI_REGISTRATION_TOKEN=registration-secret")) + }) + It("preserves standalone arguments and environment precedence", func() { + c.Mode = "standalone" + c.Address = ":8080" + c.EnvironmentVars = map[string]string{"CUDA_VISIBLE_DEVICES": "2"} + l := &Launcher{config: c} + d, e := buildLaunch(c, l.GetDataPath(), []string{"CUDA_VISIBLE_DEVICES=0"}, false) + Expect(e).NotTo(HaveOccurred()) + Expect(d.Args()).To(Equal(l.BuildRunArgs())) + Expect(d.Env()).To(Equal([]string{"CUDA_VISIBLE_DEVICES=2"})) + Expect(d.Dir()).To(BeEmpty()) + }) + It("does not start a worker without an installed, verified runtime", func() { + l := &Launcher{config: c, releaseManager: &ReleaseManager{BinaryPath: GinkgoT().TempDir(), ChecksumsPath: GinkgoT().TempDir(), MetadataPath: GinkgoT().TempDir()}} + Expect(l.StartLocalAI()).NotTo(Succeed()) + Expect(l.IsRunning()).To(BeFalse()) + }) + It("requires explicit plaintext acknowledgement", func() { + c.Worker.Connection.FrontendURL = "http://frontend.example" + _, e := buildLaunch(c, "/data", nil, false) + Expect(e).To(HaveOccurred()) + _, e = buildLaunch(c, "/data", nil, true) + Expect(e).NotTo(HaveOccurred()) + }) + It("rejects ambiguous URLs without echoing input", func() { + for _, v := range []string{"https://user:secret@host", "https://host?q=secret", "https://host/#secret", "https://host:", "https://host:65536", "https://host/a b", "https://host/%0a"} { + c.Worker.Connection.FrontendURL = v + _, e := buildLaunch(c, "/data", nil, false) + Expect(e).To(HaveOccurred(), v) + Expect(e.Error()).NotTo(ContainSubstring(v)) + } + }) + It("rejects unsupported NATS forms", func() { + for _, v := range []string{"wss://host", "tls://host/path", "tls://a,tls://b", "tls://u:p@host", "tls://host?token=x"} { + c.Worker.NATS.URL = v + _, e := buildLaunch(c, "/data", nil, false) + Expect(e).To(HaveOccurred(), v) + } + }) + It("validates address and maximum port boundaries", func() { + for _, v := range []string{"host:1", "host:65536", ":50051", "0.0.0.0:5", "[::]:5", "http://host:5", "host:+5"} { + c.Worker.Connection.AdvertisedAddress = v + _, e := buildLaunch(c, "/data", nil, false) + Expect(e).To(HaveOccurred(), v) + } + c.Worker.Connection.AdvertisedAddress = "host:65535" + max := 65535 + c.Worker.Connection.GRPCMaxPort = &max + _, e := buildLaunch(c, "/data", nil, false) + Expect(e).NotTo(HaveOccurred()) + max = 65534 + _, e = buildLaunch(c, "/data", nil, false) + Expect(e).To(HaveOccurred()) + }) + It("rejects empty credentials, controls and incomplete static pairs", func() { + for _, v := range []string{"", " ", "secret\n"} { + c.Worker.Connection.RegistrationToken = v + _, e := buildLaunch(c, "/data", nil, false) + Expect(e).To(HaveOccurred()) + } + c.Worker.Connection.RegistrationToken = "safe" + c.Worker.NATS.JWT = "jwt" + _, e := buildLaunch(c, "/data", nil, false) + Expect(e).To(HaveOccurred()) + c.Worker.NATS.UserSeed = "seed" + _, e = buildLaunch(c, "/data", nil, false) + Expect(e).To(HaveOccurred()) + }) + It("rejects every reserved advanced override and strips inherited aliases", func() { + for _, key := range strings.Fields(reservedLaunchKeys) { + c.Worker.EnvironmentVars = map[string]string{key: "false"} + _, e := buildLaunch(c, "/data", nil, false) + Expect(e).To(HaveOccurred(), key) + c.Worker.EnvironmentVars = nil + d, e := buildLaunch(c, "/data", []string{key + "=evil"}, false) + Expect(e).NotTo(HaveOccurred()) + Expect(d.Env()).NotTo(ContainElement(key + "=evil")) + } + }) + It("checks TLS files and pairing independently", func() { + c.Worker.NATS.TLSCert = "missing" + _, e := buildLaunch(c, "/data", nil, false) + Expect(e).To(HaveOccurred()) + c.Worker.NATS.TLSCert = "" + c.Worker.NATS.TLSCA = GinkgoT().TempDir() + _, e = buildLaunch(c, "/data", nil, false) + Expect(e).To(HaveOccurred()) + p := filepath.Join(GinkgoT().TempDir(), "ca") + Expect(os.WriteFile(p, []byte("test"), 0600)).To(Succeed()) + c.Worker.NATS.TLSCA = p + _, e = buildLaunch(c, "/data", nil, false) + Expect(e).NotTo(HaveOccurred()) + }) + It("snapshots absolute TLS paths from the launcher directory without changing settings", func() { + launcherDir, childDir := GinkgoT().TempDir(), GinkgoT().TempDir() + GinkgoT().Chdir(launcherDir) + c.Worker.NATS.TLSCA = "ca.pem" + c.Worker.NATS.TLSCert = "cert.pem" + c.Worker.NATS.TLSKey = "key.pem" + before := c.clone() + paths := []string{"ca.pem", "cert.pem", "key.pem"} + flags := []string{"--nats-tls-ca", "--nats-tls-cert", "--nats-tls-key"} + for _, path := range paths { + Expect(os.WriteFile(filepath.Join(launcherDir, path), []byte("launcher TLS material"), 0600)).To(Succeed()) + } + // Neither absent child files nor conflicting same-named files may change resolution. + for _, conflict := range []bool{false, true} { + if conflict { + for _, path := range paths { + Expect(os.WriteFile(filepath.Join(childDir, path), []byte("different child TLS material"), 0600)).To(Succeed()) + } + } + d, err := buildLaunch(c, childDir, nil, false) + Expect(err).NotTo(HaveOccurred()) + Expect(d.Dir()).To(Equal(childDir)) + Expect(c).To(Equal(before)) + args := d.Args() + for i, flag := range flags { + index := -1 + for j, arg := range args { + if arg == flag { + index = j + break + } + } + Expect(index).To(BeNumerically(">=", 0), flag) + Expect(args[index+1]).To(Equal(filepath.Join(launcherDir, paths[i]))) + } + // The snapshot survives config, accessor and working-directory changes. + c.Worker.NATS.TLSCA, c.Worker.NATS.TLSCert, c.Worker.NATS.TLSKey = "changed-ca", "changed-cert", "changed-key" + copyArgs := d.Args() + for i := range copyArgs { + copyArgs[i] = "changed" + } + Expect(os.Chdir(childDir)).To(Succeed()) + Expect(d.Args()).To(Equal(args)) + Expect(os.Chdir(launcherDir)).To(Succeed()) + c = before.clone() + } + }) + It("rejects each missing launcher TLS file even when the child directory has it", func() { + launcherDir, childDir := GinkgoT().TempDir(), GinkgoT().TempDir() + GinkgoT().Chdir(launcherDir) + c.Worker.NATS.TLSCA, c.Worker.NATS.TLSCert, c.Worker.NATS.TLSKey = "ca.pem", "cert.pem", "key.pem" + before := c.clone() + paths := []string{"ca.pem", "cert.pem", "key.pem"} + for _, path := range paths { + Expect(os.WriteFile(filepath.Join(childDir, path), []byte("child TLS material"), 0600)).To(Succeed()) + Expect(os.WriteFile(filepath.Join(launcherDir, path), []byte("launcher TLS material"), 0600)).To(Succeed()) + } + for _, path := range paths { + Expect(os.Remove(filepath.Join(launcherDir, path))).To(Succeed()) + d, err := buildLaunch(c, childDir, nil, false) + Expect(err).To(MatchError("NATS TLS file must be readable and regular"), path) + Expect(d).To(Equal(launchDescription{})) + Expect(c).To(Equal(before)) + Expect(os.WriteFile(filepath.Join(launcherDir, path), []byte("launcher TLS material"), 0600)).To(Succeed()) + } + }) + It("redacts configured and issued credentials before emitting bounded complete lines", func() { + var lines []string + r := newLogFilter([]string{"my-secret"}) + r.consume(strings.NewReader("my-secret\nAuthorization: Bearer new-secret\ntls://u:p@host\neyJhbGciOi.test.signature\nSU"+strings.Repeat("A", 56)+"\n"+strings.Repeat("x", 20000)+"\nokay\npartial-secret"), func(s string) { lines = append(lines, s) }) + out := strings.Join(lines, "\n") + for _, v := range []string{"my-secret", "new-secret", "u:p", "eyJhbGciOi", "SUAAAA", "partial-secret"} { + Expect(out).NotTo(ContainSubstring(v)) + } + Expect(out).To(ContainSubstring("okay")) + Expect(len(out)).To(BeNumerically("<", 2000)) + }) +}) diff --git a/cmd/launcher/internal/launcher.go b/cmd/launcher/internal/launcher.go index 63478d4f43d1..c38ceb2ea4ce 100644 --- a/cmd/launcher/internal/launcher.go +++ b/cmd/launcher/internal/launcher.go @@ -1,9 +1,8 @@ package launcher import ( - "bufio" "context" - "encoding/json" + "errors" "fmt" "io" "log" @@ -11,40 +10,25 @@ import ( "os" "os/exec" "path/filepath" + "reflect" "strings" "sync" - "syscall" + "sync/atomic" "time" "fyne.io/fyne/v2" "fyne.io/fyne/v2/container" "fyne.io/fyne/v2/dialog" "fyne.io/fyne/v2/widget" -) -// Config represents the launcher configuration -type Config struct { - ModelsPath string `json:"models_path"` - BackendsPath string `json:"backends_path"` - Address string `json:"address"` - // AutoStart controls whether the launcher starts the LocalAI server as - // soon as the launcher itself opens (and right after a fresh install). - // Unset means enabled: launching the app must yield a serving endpoint, - // which is what the quickstart docs promise. The JSON key is deliberately - // not the legacy "auto_start": that field was never honored nor exposed - // in any UI, so every existing launcher.json carries an unintentional - // false that would keep auto-start permanently off (#11673). - AutoStart *bool `json:"auto_start_server"` - StartOnBoot bool `json:"start_on_boot"` - LogLevel string `json:"log_level"` - EnvironmentVars map[string]string `json:"environment_vars"` - ShowWelcome *bool `json:"show_welcome"` -} + "github.com/mudler/LocalAI/pkg/workerstatus" +) // Launcher represents the main launcher application type Launcher struct { // Core components releaseManager *ReleaseManager + configError error config *Config ui *LauncherUI systray *SystrayManager @@ -52,16 +36,36 @@ type Launcher struct { window fyne.Window app fyne.App - // Process management - localaiCmd *exec.Cmd - isRunning bool - logBuffer *strings.Builder - logMutex sync.RWMutex - statusChannel chan string + // Process management: one serialized lifecycle owns the child. + life *lifecycle + lifeOnce sync.Once + lock *userLock + lockDir string + lockMu sync.Mutex + // applying is set while a stop-and-apply runs; start, save and update are + // refused for that time. + applying atomic.Bool + // plaintextAck is the owner's per-session acknowledgement that plaintext + // connections run inside a protected network. It is never saved. + plaintextAck atomic.Bool + statusMu sync.Mutex + statusMsg string + // statusPath and statusGeneration identify the current worker's private + // status snapshot; tracker measures its freshness. + statusPath string + statusGeneration string + statusTracker *workerstatus.Tracker + // statusFromLogs is true only for a standalone runtime: a worker's state + // comes from its status snapshot, never from log text. + statusFromLogs bool + logBuffer *strings.Builder + logMutex sync.RWMutex + statusChannel chan string // Logging - logFile *os.File - logPath string + logFile *os.File + logPath string + logBytes int // UI state lastUpdateCheck time.Time @@ -83,21 +87,11 @@ func NewLauncher(ui *LauncherUI, window fyne.Window, app fyne.App) *Launcher { // setupLogging sets up log file for LocalAI process output func (l *Launcher) setupLogging() error { - // Create logs directory in data folder - dataPath := l.GetDataPath() - logsDir := filepath.Join(dataPath, "logs") - if err := os.MkdirAll(logsDir, 0755); err != nil { - return fmt.Errorf("failed to create logs directory: %w", err) - } - - // Create log file with timestamp - timestamp := time.Now().Format("2006-01-02_15-04-05") - l.logPath = filepath.Join(logsDir, fmt.Sprintf("localai_%s.log", timestamp)) - - logFile, err := os.Create(l.logPath) + logFile, err := secureLogFile(l.GetDataPath()) if err != nil { - return fmt.Errorf("failed to create log file: %w", err) + return err } + l.logPath = logFile.Name() l.logFile = logFile return nil @@ -110,18 +104,13 @@ func (l *Launcher) Initialize() error { } log.Printf("Initializing launcher...") - // Setup logging - if err := l.setupLogging(); err != nil { - return fmt.Errorf("failed to setup logging: %w", err) - } - - // Load configuration - log.Printf("Loading configuration...") + // Verify settings storage before touching files beneath its directory. if err := l.loadConfig(); err != nil { return fmt.Errorf("failed to load config: %w", err) } - log.Printf("Configuration loaded, current state: ModelsPath=%s, BackendsPath=%s, Address=%s, LogLevel=%s", - l.config.ModelsPath, l.config.BackendsPath, l.config.Address, l.config.LogLevel) + if err := l.setupLogging(); err != nil { + return fmt.Errorf("failed to setup logging: %w", err) + } // Clean up any partial downloads log.Printf("Cleaning up partial downloads...") @@ -133,20 +122,16 @@ func (l *Launcher) Initialize() error { if l.config.ModelsPath == "" { homeDir, _ := os.UserHomeDir() l.config.ModelsPath = filepath.Join(homeDir, ".localai", "models") - log.Printf("Setting default ModelsPath: %s", l.config.ModelsPath) } if l.config.BackendsPath == "" { homeDir, _ := os.UserHomeDir() l.config.BackendsPath = filepath.Join(homeDir, ".localai", "backends") - log.Printf("Setting default BackendsPath: %s", l.config.BackendsPath) } if l.config.Address == "" { l.config.Address = "127.0.0.1:8080" - log.Printf("Setting default Address: %s", l.config.Address) } if l.config.LogLevel == "" { l.config.LogLevel = "info" - log.Printf("Setting default LogLevel: %s", l.config.LogLevel) } if l.config.EnvironmentVars == nil { l.config.EnvironmentVars = make(map[string]string) @@ -160,19 +145,13 @@ func (l *Launcher) Initialize() error { log.Printf("Setting default ShowWelcome: true") } - if l.config.AutoStart == nil { - enabled := true - l.config.AutoStart = &enabled - log.Printf("Setting default AutoStart: true") - } - // Create directories os.MkdirAll(l.config.ModelsPath, 0755) os.MkdirAll(l.config.BackendsPath, 0755) // Save the configuration with default values if err := l.saveConfig(); err != nil { - log.Printf("Warning: failed to save default configuration: %v", err) + return err } // System tray is now handled in main.go using Fyne's built-in approach @@ -202,14 +181,17 @@ func (l *Launcher) Initialize() error { // ShouldAutoStartServer reports whether the launcher should start the server // without user interaction: at launcher startup and right after a fresh -// install. Defaults to enabled; StartOnBoot forces a start even when -// auto-start was explicitly disabled, preserving its historical behavior. +// install. Unset means enabled; legacy StartOnBoot is inert. func (l *Launcher) ShouldAutoStartServer() bool { - if l.config == nil { + if l.config == nil || l.configError != nil { return false } - if l.config.StartOnBoot { - return true + if l.config.Mode == roleWorker { + // Joining a deployment is never a default: the owner must have chosen it. + return l.config.AutoStart != nil && *l.config.AutoStart + } + if l.config.Mode != "" && l.config.Mode != roleStandalone { + return false } return l.config.AutoStart == nil || *l.config.AutoStart } @@ -229,9 +211,18 @@ func (l *Launcher) autoStartServer() { }() } -// StartLocalAI starts the LocalAI server +// StartLocalAI starts the LocalAI runtime for the selected mode. func (l *Launcher) StartLocalAI() error { - if l.isRunning { + if l.configError != nil { + return l.configError + } + if l.config == nil { + return fmt.Errorf("launcher settings are missing") + } + if l.Applying() { + return errLifecycleBusy + } + if l.IsRunning() { return fmt.Errorf("LocalAI is already running") } @@ -250,120 +241,282 @@ func (l *Launcher) StartLocalAI() error { return fmt.Errorf("LocalAI binary not found. Please download a release first") } - // Build command arguments - args := l.BuildRunArgs() + description, err := l.describeLaunch() + if err != nil { + return err + } + return l.launchRuntime(binaryPath, description) +} + +// describeLaunch validates the active profile. A plaintext connection needs the +// owner's acknowledgement for this launcher session. +func (l *Launcher) describeLaunch() (launchDescription, error) { + return buildLaunch(l.config, l.GetDataPath(), os.Environ(), l.plaintextAck.Load()) +} + +// SetPlaintextAcknowledged records that the owner accepts plaintext frontend or +// message-bus connections inside a protected network, for this session only. +func (l *Launcher) SetPlaintextAcknowledged(v bool) { l.plaintextAck.Store(v) } + +// PlaintextAcknowledged reports the session acknowledgement. +func (l *Launcher) PlaintextAcknowledged() bool { return l.plaintextAck.Load() } + +// Applying reports whether a stop-and-apply transition is under way. +func (l *Launcher) Applying() bool { return l.applying.Load() } - l.localaiCmd = exec.CommandContext(l.ctx, binaryPath, args...) +func (l *Launcher) runtimeContext() context.Context { + if l.ctx == nil { + return context.Background() + } + return l.ctx +} - // Apply environment variables - if len(l.config.EnvironmentVars) > 0 { - env := os.Environ() - for key, value := range l.config.EnvironmentVars { - env = append(env, fmt.Sprintf("%s=%s", key, value)) +func (l *Launcher) lifecycleController() *lifecycle { + l.lifeOnce.Do(func() { + if l.life == nil { + l.life = newLifecycle() } - l.localaiCmd.Env = env + }) + return l.life +} + +// runUI runs f on the UI thread when an application exists. +func (l *Launcher) runUI(f func()) { + if l.app == nil { + f() + return } + fyne.Do(f) +} - // Setup logging - stdout, err := l.localaiCmd.StdoutPipe() +// ensureLock takes the per-user launcher lock once and keeps it until +// Shutdown completes. A second launcher cannot start an overlapping role. +func (l *Launcher) ensureLock() error { + l.lockMu.Lock() + defer l.lockMu.Unlock() + if l.lock != nil { + return nil + } + dir := l.lockDir + if dir == "" { + d, err := configDirectory() + if err != nil { + return err + } + d.Close() + home, err := os.UserHomeDir() + if err != nil { + return err + } + dir = filepath.Join(home, ".localai") + } + lock, err := acquireUserLock(dir) if err != nil { - return fmt.Errorf("failed to create stdout pipe: %w", err) + return err } + l.lock = lock + return nil +} - stderr, err := l.localaiCmd.StderrPipe() - if err != nil { - return fmt.Errorf("failed to create stderr pipe: %w", err) +func (l *Launcher) releaseLock() { + l.lockMu.Lock() + defer l.lockMu.Unlock() + if l.lock != nil { + _ = l.lock.Release() + l.lock = nil } +} - // Start the process - if err := l.localaiCmd.Start(); err != nil { - return fmt.Errorf("failed to start LocalAI: %w", err) +// launchRuntime starts one validated launch. A worker additionally needs a +// runtime that speaks the status contract and a private status channel; an +// older runtime is an upgrade error and never a log-parsing fallback. +func (l *Launcher) launchRuntime(binary string, d launchDescription) error { + if err := l.ensureLock(); err != nil { + return err + } + worker := d.role == roleWorker + env := d.Env() + var statusDir string + if worker { + if err := checkWorkerRuntime(l.runtimeContext(), binary); err != nil { + return err + } + parent := filepath.Join(d.Dir(), "launcher-run") + if err := os.MkdirAll(parent, 0o700); err != nil { + return fmt.Errorf("cannot create the worker status directory: %w", err) + } + path, generation, err := workerstatus.NewSession(parent) + if err != nil { + return err + } + statusDir = filepath.Dir(path) + env = append(env, workerstatus.PathEnv+"="+path, workerstatus.GenerationEnv+"="+generation) + l.statusMu.Lock() + l.statusPath, l.statusGeneration = path, generation + l.statusMu.Unlock() } - l.isRunning = true + filter := newLogFilter(d.secrets) + spec := launchSpec{ + Binary: binary, + Args: d.Args(), + Env: env, + Dir: d.Dir(), + LedgerPath: d.ledger, + Attach: func(cmd *exec.Cmd) error { + stdout, err := cmd.StdoutPipe() + if err != nil { + return fmt.Errorf("failed to create stdout pipe: %w", err) + } + stderr, err := cmd.StderrPipe() + if err != nil { + return fmt.Errorf("failed to create stderr pipe: %w", err) + } + go l.monitorFilteredLogs(stdout, "STDOUT", filter) + go l.monitorFilteredLogs(stderr, "STDERR", newLogFilter(d.secrets)) + return nil + }, + OnExit: func(generation uint64, err error) { l.runtimeExited(generation, err, statusDir) }, + } + l.statusMu.Lock() + l.statusFromLogs = !worker + l.statusMu.Unlock() + generation, err := l.lifecycleController().Start(spec) + if err != nil { + if statusDir != "" { + _ = os.RemoveAll(statusDir) + } + return err + } - fyne.Do(func() { - l.updateStatus("LocalAI is starting...") + if worker { + l.statusMu.Lock() + l.statusTracker = &workerstatus.Tracker{Generation: l.statusGeneration, ExpectedPID: l.life.PID(), Started: time.Now()} + l.statusMu.Unlock() + go l.pollWorkerStatus(generation) + } + l.runUI(func() { + if worker { + l.updateStatus("Starting worker...") + } else { + l.updateStatus("LocalAI is starting...") + } l.updateRunningState(true) }) + return nil +} - // Start log monitoring - go l.monitorLogs(stdout, "STDOUT") - go l.monitorLogs(stderr, "STDERR") +// runtimeExited is called once per run after cleanup finished or failed. +func (l *Launcher) runtimeExited(generation uint64, err error, statusDir string) { + if errors.Is(err, errShutdownIncomplete) { + l.runUI(func() { l.updateStatus("Failed: shutdown incomplete. Stop again to retry.") }) + return // keep the status directory: ownership is not resolved + } + if statusDir != "" { + _ = os.RemoveAll(statusDir) + } + l.runUI(func() { + l.updateRunningState(false) + if err != nil { + l.updateStatus(fmt.Sprintf("LocalAI stopped with error: %v", err)) + } else { + l.updateStatus("LocalAI stopped") + } + }) +} - // Monitor process with startup timeout - go func() { - // Wait for process to start or fail - err := l.localaiCmd.Wait() - l.isRunning = false - fyne.Do(func() { - l.updateRunningState(false) - if err != nil { - l.updateStatus(fmt.Sprintf("LocalAI stopped with error: %v", err)) - } else { - l.updateStatus("LocalAI stopped") +// pollWorkerStatus shows the worker's evidence-backed state every two seconds +// while this generation is the live run. Results of older runs are dropped. +func (l *Launcher) pollWorkerStatus(generation uint64) { + ticker := time.NewTicker(workerstatus.PollInterval) + defer ticker.Stop() + for range ticker.C { + if !l.lifecycleController().Running(generation) { + return + } + text := l.WorkerStatus() + l.runUI(func() { + if l.lifecycleController().Running(generation) { + l.updateStatus(text) } }) - }() + } +} - // Add startup timeout detection - go func() { - time.Sleep(10 * time.Second) // Wait 10 seconds for startup - if l.isRunning { - // Check if process is still alive - if l.localaiCmd.Process != nil { - if err := l.localaiCmd.Process.Signal(syscall.Signal(0)); err != nil { - // Process is dead, mark as not running - l.isRunning = false - fyne.Do(func() { - l.updateRunningState(false) - l.updateStatus("LocalAI failed to start properly") - }) - } - } - } - }() +// WorkerStatus classifies the current worker snapshot into display text. +func (l *Launcher) WorkerStatus() string { + l.statusMu.Lock() + defer l.statusMu.Unlock() + if l.statusTracker == nil || l.statusPath == "" { + return "Starting worker..." + } + s, err := workerstatus.Read(l.statusPath, l.statusGeneration) + if err != nil { + return workerStateText("status_unavailable") + } + return workerStateText(l.statusTracker.Classify(s, time.Now())) +} - return nil +func workerStateText(state string) string { + switch state { + case "ready": + return "Ready - waiting for work" + case "starting": + return "Starting worker..." + case "pending": + return "Waiting for approval" + case "stopping": + return "Stopping..." + case "registration": + return "Connection problem: registration" + case "credentials": + return "Connection problem: credentials" + case "nats": + return "Connection problem: message bus" + case "subscriptions": + return "Connection problem: lifecycle subscriptions" + case "readiness": + return "Connection problem: local readiness checks" + case "heartbeat": + return "Connection problem: heartbeat" + default: + return "Connection problem: status unavailable" + } } -// StopLocalAI stops the LocalAI server +// StopLocalAI runs the bounded stop sequence. It returns only after the +// runtime and its owned backend groups are confirmed gone, or reports that +// shutdown is incomplete and keeps further starts blocked. func (l *Launcher) StopLocalAI() error { - if !l.isRunning || l.localaiCmd == nil { - return fmt.Errorf("LocalAI is not running") + if l.lifecycleController().State() == lifecycleStopped { + return fmt.Errorf("LocalAI is not running: %w", errNotRunning) } - - // Gracefully terminate the process - if err := l.localaiCmd.Process.Signal(os.Interrupt); err != nil { - // If graceful termination fails, force kill - if killErr := l.localaiCmd.Process.Kill(); killErr != nil { - return fmt.Errorf("failed to kill LocalAI process: %w", killErr) - } + l.runUI(func() { l.updateStatus("Stopping...") }) + err := l.lifecycleController().Stop() + if err != nil { + l.runUI(func() { l.updateStatus("Failed: shutdown incomplete. Stop again to retry.") }) } - - l.isRunning = false - fyne.Do(func() { - l.updateRunningState(false) - l.updateStatus("LocalAI stopped") - }) - return nil + return err } -// IsRunning returns whether LocalAI is currently running +// IsRunning reports whether a runtime, or unresolved ownership of one, exists. +// Stop stays available until shutdown is confirmed. func (l *Launcher) IsRunning() bool { - return l.isRunning + return l.lifecycleController().State() != lifecycleStopped } -// Shutdown performs cleanup when the application is closing +// Shutdown stops the runtime before the launcher exits. It does not claim +// success, and keeps the lock, when shutdown cannot be confirmed. func (l *Launcher) Shutdown() error { log.Printf("Launcher shutting down, stopping LocalAI...") - // Stop LocalAI if it's running - if l.isRunning { - if err := l.StopLocalAI(); err != nil { + if l.IsRunning() { + if err := l.StopLocalAI(); err != nil && !errors.Is(err, errNotRunning) { log.Printf("Error stopping LocalAI during shutdown: %v", err) + return err } } + l.releaseLock() // Close log file if open if l.logFile != nil { @@ -402,13 +555,22 @@ func (l *Launcher) GetRecentLogs() string { // GetConfig returns the current configuration func (l *Launcher) GetConfig() *Config { - return l.config + return l.config.clone() } // SetConfig updates the configuration func (l *Launcher) SetConfig(config *Config) error { - l.config = config - return l.saveConfig() + candidate := config.clone() + if candidate == nil { + return fmt.Errorf("launcher settings are missing") + } + if err := l.persistConfig(candidate); err != nil { + l.configError = err + return err + } + l.config = candidate + l.configError = nil + return nil } func (l *Launcher) GetUI() *LauncherUI { @@ -426,6 +588,11 @@ func (l *Launcher) GetReleaseManager() *ReleaseManager { // GetWebUIURL returns the URL for the WebUI func (l *Launcher) GetWebUIURL() string { + if l.config != nil && l.config.Mode == roleWorker { + // A worker hosts no UI: the browser goes to the deployment. + u, _ := l.DeploymentURL() + return u + } address := l.config.Address if strings.HasPrefix(address, ":") { address = "localhost" + address @@ -448,18 +615,7 @@ func (l *Launcher) GetWebUIURL() string { // path under the per-user data directory avoids both the misplacement (#10610) // and the cross-user /tmp collision. func (l *Launcher) BuildRunArgs() []string { - dataPath := l.GetDataPath() - return []string{ - "run", - "--models-path", l.config.ModelsPath, - "--backends-path", l.config.BackendsPath, - "--address", l.config.Address, - "--log-level", l.config.LogLevel, - "--data-path", filepath.Join(dataPath, "data"), - "--localai-config-dir", filepath.Join(dataPath, "configuration"), - "--generated-content-path", filepath.Join(dataPath, "generated"), - "--upload-path", filepath.Join(dataPath, "uploads"), - } + return standaloneArgs(l.config, l.GetDataPath()) } // GetDataPath returns the path where LocalAI data and logs are stored @@ -494,6 +650,9 @@ func (l *Launcher) CheckForUpdates() (bool, string, error) { // DownloadUpdate downloads the latest version func (l *Launcher) DownloadUpdate(version string, progressCallback func(downloaded, total int64)) error { + if l.Applying() || l.lifecycleController().State() == lifecycleStopping { + return errLifecycleBusy + } return l.releaseManager.DownloadRelease(version, progressCallback) } @@ -502,13 +661,20 @@ func (l *Launcher) GetCurrentVersion() string { return l.releaseManager.GetInstalledVersion() } +// StatusText is the last status shown, without consuming it. +func (l *Launcher) StatusText() string { + l.statusMu.Lock() + defer l.statusMu.Unlock() + return l.statusMsg +} + // GetCurrentStatus returns the current status func (l *Launcher) GetCurrentStatus() string { select { case status := <-l.statusChannel: return status default: - if l.isRunning { + if l.IsRunning() { return "LocalAI is running" } return "Ready" @@ -517,7 +683,7 @@ func (l *Launcher) GetCurrentStatus() string { // GetLastStatus returns the last known status without consuming from channel func (l *Launcher) GetLastStatus() string { - if l.isRunning { + if l.IsRunning() { return "LocalAI is running" } @@ -701,7 +867,7 @@ func (l *Launcher) showDownloadProgress(version, title string) { } // A fresh install should end with a running server, not with // the user hunting for a start button in the tray (#11673). - if l.ShouldAutoStartServer() && !l.isRunning { + if l.ShouldAutoStartServer() && !l.IsRunning() { l.autoStartServer() } }, win) @@ -823,9 +989,11 @@ func formatBytes(b int64) string { // monitorLogs monitors the output of LocalAI and adds it to the log buffer func (l *Launcher) monitorLogs(reader io.Reader, prefix string) { - scanner := bufio.NewScanner(reader) - for scanner.Scan() { - line := scanner.Text() + l.monitorFilteredLogs(reader, prefix, newLogFilter(nil)) +} + +func (l *Launcher) monitorFilteredLogs(reader io.Reader, prefix string, filter logFilter) { + filter.consume(reader, func(line string) { timestamp := time.Now().Format("15:04:05") logLine := fmt.Sprintf("[%s] %s: %s\n", timestamp, prefix, line) @@ -840,14 +1008,13 @@ func (l *Launcher) monitorLogs(reader io.Reader, prefix string) { l.logBuffer.WriteString(content[len(content)-50000:]) } } - l.logMutex.Unlock() // Write to log file if available - if l.logFile != nil { - if _, err := l.logFile.WriteString(logLine); err != nil { - log.Printf("Failed to write to log file: %v", err) - } + if l.logFile != nil && l.logBytes+len(logLine) <= maxLogFile { + n, _ := l.logFile.WriteString(logLine) + l.logBytes += n } + l.logMutex.Unlock() fyne.Do(func() { // Notify UI of new log content @@ -855,16 +1022,23 @@ func (l *Launcher) monitorLogs(reader io.Reader, prefix string) { l.ui.OnLogUpdate(logLine) } - // Check for startup completion - if strings.Contains(line, "API server listening") { + // Check for startup completion. A worker never takes its state from + // log text. + l.statusMu.Lock() + fromLogs := l.statusFromLogs + l.statusMu.Unlock() + if fromLogs && strings.Contains(line, "API server listening") { l.updateStatus("LocalAI is running") } }) - } + }) } // updateStatus updates the status and notifies UI func (l *Launcher) updateStatus(status string) { + l.statusMu.Lock() + l.statusMsg = status + l.statusMu.Unlock() select { case l.statusChannel <- status: default: @@ -917,71 +1091,59 @@ func (l *Launcher) periodicUpdateCheck() { } } -// loadConfig loads configuration from file -func (l *Launcher) loadConfig() error { - homeDir, err := os.UserHomeDir() - if err != nil { - return fmt.Errorf("failed to get home directory: %w", err) +// DeploymentURL is the saved frontend URL, validated, without credentials or a +// guessed internal route. The browser handles its own login. +func (l *Launcher) DeploymentURL() (string, error) { + if l.config == nil { + return "", fmt.Errorf("launcher settings are missing") } + return launchURL(l.config.Worker.Connection.FrontendURL, false) +} - configPath := filepath.Join(homeDir, ".localai", "launcher.json") - log.Printf("Loading config from: %s", configPath) +// errStopRequired means the change needs the running role stopped first. +var errStopRequired = errors.New("the running role must be stopped to apply this change") - if _, err := os.Stat(configPath); os.IsNotExist(err) { - log.Printf("Config file not found, creating default config") - // Create default config - return l.saveConfig() +// activeProfileChanged reports whether a change affects what the running role +// was launched with: a role switch, or any shared or active-profile setting. +func activeProfileChanged(old, next *Config) bool { + role := func(c *Config) string { + if c.Mode == "" { + return roleStandalone + } + return c.Mode } - - // Load existing config - configData, err := os.ReadFile(configPath) - if err != nil { - return fmt.Errorf("failed to read config file: %w", err) + if role(old) != role(next) { + return true } - - log.Printf("Config file content: %s", string(configData)) - - log.Printf("loadConfig: about to unmarshal JSON data") - if err := json.Unmarshal(configData, l.config); err != nil { - return fmt.Errorf("failed to parse config file: %w", err) + if old.ModelsPath != next.ModelsPath || old.BackendsPath != next.BackendsPath || old.LogLevel != next.LogLevel { + return true } - log.Printf("loadConfig: JSON unmarshaled successfully") - - log.Printf("Loaded config: ModelsPath=%s, BackendsPath=%s, Address=%s, LogLevel=%s", - l.config.ModelsPath, l.config.BackendsPath, l.config.Address, l.config.LogLevel) - log.Printf("Environment vars: %v", l.config.EnvironmentVars) - - return nil -} - -// saveConfig saves configuration to file -func (l *Launcher) saveConfig() error { - homeDir, err := os.UserHomeDir() - if err != nil { - return fmt.Errorf("failed to get home directory: %w", err) + if role(old) == roleWorker { + return !reflect.DeepEqual(old.Worker, next.Worker) } + return old.Address != next.Address || !reflect.DeepEqual(old.EnvironmentVars, next.EnvironmentVars) +} - configDir := filepath.Join(homeDir, ".localai") - if err := os.MkdirAll(configDir, 0755); err != nil { - return fmt.Errorf("failed to create config directory: %w", err) +// ApplySettings commits candidate. When a runtime exists and the change affects +// it, nothing is committed until the caller confirms a stop (stop=true) and the +// stop is verified. It never starts the replacement. Cancelling, a failed stop +// and an incomplete shutdown leave the active configuration unchanged. +func (l *Launcher) ApplySettings(candidate *Config, stop bool) error { + if !l.applying.CompareAndSwap(false, true) { + return errLifecycleBusy } - - // Marshal config to JSON - log.Printf("saveConfig: marshaling config with EnvironmentVars: %v", l.config.EnvironmentVars) - configData, err := json.MarshalIndent(l.config, "", " ") - if err != nil { - return fmt.Errorf("failed to marshal config: %w", err) + defer l.applying.Store(false) + state := l.lifecycleController().State() + if state == lifecycleStarting || state == lifecycleStopping { + return errLifecycleBusy } - log.Printf("saveConfig: JSON marshaled successfully, length: %d", len(configData)) - - configPath := filepath.Join(configDir, "launcher.json") - log.Printf("Saving config to: %s", configPath) - log.Printf("Config content: %s", string(configData)) - - if err := os.WriteFile(configPath, configData, 0644); err != nil { - return fmt.Errorf("failed to write config file: %w", err) + if state != lifecycleStopped && l.config != nil && activeProfileChanged(l.config, candidate) { + if !stop { + return errStopRequired + } + if err := l.StopLocalAI(); err != nil && !errors.Is(err, errNotRunning) { + return err + } } - - log.Printf("Config saved successfully") - return nil + return l.SetConfig(candidate) } diff --git a/cmd/launcher/internal/launcher_runtime_test.go b/cmd/launcher/internal/launcher_runtime_test.go new file mode 100644 index 000000000000..f9fe90fbb671 --- /dev/null +++ b/cmd/launcher/internal/launcher_runtime_test.go @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: MIT +package launcher + +import ( + "errors" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "time" + + "github.com/mudler/LocalAI/pkg/workerledger" + "github.com/mudler/LocalAI/pkg/workerstatus" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// The launcher owns one runtime at a time. These specs drive Launcher with a +// real child process standing in for the LocalAI binary. +var _ = Describe("Launcher runtime ownership", func() { + var ( + dir string + runtime string + l *Launcher + ) + + fakeRuntime := func(capability string) { + body := "#!/bin/sh\n" + + "if [ \"$1\" = worker-status-capability ]; then echo " + capability + "; exit 0; fi\n" + + "trap 'exit 0' TERM\n" + + "env > \"$(dirname \"$0\")/child.env\"\n" + + ": > \"$(dirname \"$0\")/ready\"\n" + + "while :; do sleep 0.05; done\n" + Expect(os.WriteFile(runtime, []byte(body), 0o700)).To(Succeed()) + } + waitReady := func() { + Eventually(func() error { _, err := os.Stat(filepath.Join(dir, "ready")); return err }, 5*time.Second, 10*time.Millisecond).Should(Succeed()) + } + workerDescription := func() launchDescription { + return launchDescription{ + role: roleWorker, + args: []string{"worker"}, + env: []string{"PATH=/usr/bin:/bin", "LOCALAI_REGISTRATION_TOKEN=token-secret"}, + dir: dir, + ledger: filepath.Join(dir, "data", "worker-processes.json"), + } + } + newLauncher := func() *Launcher { + n := &Launcher{config: &Config{ModelsPath: filepath.Join(dir, "models")}} + n.lockDir = dir + n.life = newLifecycle() + n.life.termGrace = 500 * time.Millisecond + n.life.killGrace = 2 * time.Second + return n + } + + BeforeEach(func() { + dir = GinkgoT().TempDir() + runtime = filepath.Join(dir, "local-ai") + l = newLauncher() + DeferCleanup(func() { _ = l.Shutdown() }) + }) + + It("runs a standalone runtime and stops it with confirmation", func() { + fakeRuntime("irrelevant") + d := launchDescription{role: roleStandalone, args: []string{"run"}, env: []string{"PATH=/usr/bin:/bin"}, dir: dir} + Expect(l.launchRuntime(runtime, d)).To(Succeed()) + waitReady() + Expect(l.IsRunning()).To(BeTrue()) + pid := l.life.PID() + Expect(l.StopLocalAI()).To(Succeed()) + Expect(l.IsRunning()).To(BeFalse()) + Expect(workerledger.GroupAlive(pid)).To(BeFalse()) + Expect(errors.Is(l.StopLocalAI(), errNotRunning)).To(BeTrue()) + }) + + It("does not start a worker whose runtime lacks the status contract", func() { + fakeRuntime("old-version") + Expect(l.launchRuntime(runtime, workerDescription())).To(MatchError(errWorkerUpgrade)) + Expect(l.IsRunning()).To(BeFalse()) + _, err := os.Stat(filepath.Join(dir, "ready")) + Expect(err).To(HaveOccurred(), "the unsupported runtime must never run") + }) + + It("gives a worker only the private status channel and never reports ready without evidence", func() { + fakeRuntime(workerstatus.Capability) + Expect(l.launchRuntime(runtime, workerDescription())).To(Succeed()) + waitReady() + env, err := os.ReadFile(filepath.Join(dir, "child.env")) + Expect(err).ToNot(HaveOccurred()) + Expect(string(env)).To(ContainSubstring(workerstatus.PathEnv + "=")) + Expect(string(env)).To(ContainSubstring(workerstatus.GenerationEnv + "=")) + // Process liveness alone is not readiness. + Expect(l.WorkerStatus()).ToNot(ContainSubstring("Ready")) + Expect(l.WorkerStatus()).To(ContainSubstring("Connection problem")) + }) + + It("reports ready only from a current, complete snapshot of the running child", func() { + fakeRuntime(workerstatus.Capability) + Expect(l.launchRuntime(runtime, workerDescription())).To(Succeed()) + waitReady() + path, generation := l.statusPath, l.statusGeneration + Expect(path).ToNot(BeEmpty()) + now := time.Now() + snapshot := workerstatus.Snapshot{ + PID: l.life.PID(), Schema: workerstatus.Schema, Generation: generation, Sequence: 1, Time: now, + Registration: "healthy", RegistrationTime: now, Credentials: true, Authenticated: true, Connected: true, + Subscriptions: true, Armed: true, LocalReady: true, Heartbeat: now, HeartbeatInterval: 10 * time.Second, + } + Expect(workerstatus.Write(path, snapshot)).To(Succeed()) + Expect(l.WorkerStatus()).To(ContainSubstring("Ready")) + + // A snapshot written by some other process id is not evidence about this child. + snapshot.PID, snapshot.Sequence = l.life.PID()+1, 2 + Expect(workerstatus.Write(path, snapshot)).To(Succeed()) + Expect(l.WorkerStatus()).ToNot(ContainSubstring("Ready")) + }) + + It("refuses a second launcher for the same user and keeps the first running", func() { + fakeRuntime("irrelevant") + d := launchDescription{role: roleStandalone, args: []string{"run"}, env: []string{"PATH=/usr/bin:/bin"}, dir: dir} + Expect(l.launchRuntime(runtime, d)).To(Succeed()) + other := newLauncher() + DeferCleanup(func() { _ = other.Shutdown() }) + Expect(other.launchRuntime(runtime, d)).To(MatchError(errLauncherLocked)) + Expect(l.IsRunning()).To(BeTrue()) + }) + + It("lets a new launcher start after the first one shut down", func() { + fakeRuntime("irrelevant") + d := launchDescription{role: roleStandalone, args: []string{"run"}, env: []string{"PATH=/usr/bin:/bin"}, dir: dir} + Expect(l.launchRuntime(runtime, d)).To(Succeed()) + Expect(l.Shutdown()).To(Succeed()) + other := newLauncher() + DeferCleanup(func() { _ = other.Shutdown() }) + Expect(other.launchRuntime(runtime, d)).To(Succeed()) + }) + + It("does not report stopped or release the lock when shutdown is incomplete", func() { + fakeRuntime(workerstatus.Capability) + d := workerDescription() + Expect(l.launchRuntime(runtime, d)).To(Succeed()) + waitReady() + child := exec.Command("/bin/sleep", "300") + child.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + Expect(child.Start()).To(Succeed()) + go func() { _ = child.Wait() }() + DeferCleanup(func() { _ = syscall.Kill(-child.Process.Pid, syscall.SIGKILL) }) + Expect(workerledger.WriteFile(d.ledger, []workerledger.Entry{{Key: "m#0", PID: child.Process.Pid, StartTime: workerledger.ReadStartTime(child.Process.Pid)}})).To(Succeed()) + + realKill, realReap := workerledger.KillGroup, workerledger.ReapTimeout + workerledger.KillGroup = func(int) error { return nil } + workerledger.ReapTimeout = 200 * time.Millisecond + DeferCleanup(func() { workerledger.KillGroup, workerledger.ReapTimeout = realKill, realReap }) + + err := l.StopLocalAI() + Expect(errors.Is(err, errShutdownIncomplete)).To(BeTrue()) + Expect(l.IsRunning()).To(BeTrue(), "stop stays available for a retry") + Expect(l.Shutdown()).To(MatchError(errShutdownIncomplete)) + Expect(strings.ToLower(l.StatusText())).To(ContainSubstring("shutdown incomplete")) + Expect(newLauncher().launchRuntime(runtime, d)).To(MatchError(errLauncherLocked), "no overlapping role while ownership is unresolved") + + workerledger.KillGroup = realKill + Expect(l.StopLocalAI()).To(Succeed()) + Expect(l.IsRunning()).To(BeFalse()) + }) +}) diff --git a/cmd/launcher/internal/launcher_test.go b/cmd/launcher/internal/launcher_test.go index a73dcaaf551e..d59806756918 100644 --- a/cmd/launcher/internal/launcher_test.go +++ b/cmd/launcher/internal/launcher_test.go @@ -202,16 +202,18 @@ var _ = Describe("Launcher", func() { config := launcherInstance.GetConfig() err := json.Unmarshal([]byte(`{"auto_start_server": false}`), config) Expect(err).ToNot(HaveOccurred()) + Expect(launcherInstance.SetConfig(config)).To(Succeed()) Expect(launcherInstance.ShouldAutoStartServer()).To(BeFalse()) }) - It("should still auto-start when StartOnBoot is set even if auto-start is off", func() { + It("should keep legacy StartOnBoot inert when auto-start is off", func() { config := launcherInstance.GetConfig() err := json.Unmarshal([]byte(`{"auto_start_server": false, "start_on_boot": true}`), config) Expect(err).ToNot(HaveOccurred()) + Expect(launcherInstance.SetConfig(config)).To(Succeed()) - Expect(launcherInstance.ShouldAutoStartServer()).To(BeTrue()) + Expect(launcherInstance.ShouldAutoStartServer()).To(BeFalse()) }) It("should ignore the legacy auto_start key older launchers persisted as false", func() { @@ -221,6 +223,7 @@ var _ = Describe("Launcher", func() { config := launcherInstance.GetConfig() err := json.Unmarshal([]byte(`{"auto_start": false}`), config) Expect(err).ToNot(HaveOccurred()) + Expect(launcherInstance.SetConfig(config)).To(Succeed()) Expect(launcherInstance.ShouldAutoStartServer()).To(BeTrue()) }) diff --git a/cmd/launcher/internal/lifecycle.go b/cmd/launcher/internal/lifecycle.go new file mode 100644 index 000000000000..d17180177908 --- /dev/null +++ b/cmd/launcher/internal/lifecycle.go @@ -0,0 +1,339 @@ +// SPDX-License-Identifier: MIT +package launcher + +import ( + "errors" + "fmt" + "os/exec" + "strings" + "sync" + "syscall" + "time" + + "github.com/mudler/LocalAI/pkg/workerledger" +) + +type lifecycleState string + +const ( + lifecycleStopped lifecycleState = "stopped" + lifecycleStarting lifecycleState = "starting" + lifecycleRunning lifecycleState = "running" + lifecycleStopping lifecycleState = "stopping" + lifecycleIncomplete lifecycleState = "shutdown incomplete" +) + +var ( + errLifecycleBusy = errors.New("a runtime operation is already in progress or a runtime is running") + errNotRunning = errors.New("no runtime is running") + errShutdownIncomplete = errors.New("shutdown incomplete: a process owned by the earlier run may still be alive") +) + +// Spec section "Lifecycle and ownership": 30 s to drain, then force, then 5 s to +// reap and verify. +const ( + defaultTermGrace = 30 * time.Second + defaultKillGrace = 5 * time.Second +) + +// launchSpec is everything the controller needs to run one child. It carries +// no policy: validation happened when the launch description was built. +type launchSpec struct { + Binary string + Args []string + Env []string + Dir string + // LedgerPath is the backend process ledger the runtime writes. Empty for a + // role that records no backend ownership. + LedgerPath string + // Attach runs before the child starts, to connect its output. + Attach func(*exec.Cmd) error + // OnExit is called once per run, with the run's generation, after the + // runtime and every owned group are confirmed gone (or cleanup failed). + OnExit func(generation uint64, err error) +} + +// lifecycle serializes the single child the launcher may own. All state changes +// happen under mu; blocking waits happen outside it, behind the Stopping and +// Starting states, which reject every other operation. +type lifecycle struct { + mu sync.Mutex + state lifecycleState + generation uint64 + cmd *exec.Cmd + exited chan struct{} + spec launchSpec + lastGen uint64 + lastErr error + + termGrace time.Duration + killGrace time.Duration +} + +func newLifecycle() *lifecycle { + return &lifecycle{state: lifecycleStopped, termGrace: defaultTermGrace, killGrace: defaultKillGrace} +} + +func (l *lifecycle) State() lifecycleState { + l.mu.Lock() + defer l.mu.Unlock() + return l.state +} + +// Running reports whether generation is the current, live run. Asynchronous +// work tagged with an older generation must be discarded. +func (l *lifecycle) Running(generation uint64) bool { + l.mu.Lock() + defer l.mu.Unlock() + return l.state == lifecycleRunning && l.generation == generation +} + +// PID is the runtime's process id, or 0 when none runs. +func (l *lifecycle) PID() int { + l.mu.Lock() + defer l.mu.Unlock() + if l.cmd == nil || l.cmd.Process == nil { + return 0 + } + return l.cmd.Process.Pid +} + +// LastExit is the generation and result of the most recent finished run. +func (l *lifecycle) LastExit() (uint64, error) { + l.mu.Lock() + defer l.mu.Unlock() + return l.lastGen, l.lastErr +} + +// busy returns the error that explains why no new operation may begin. +func (l *lifecycle) busyLocked() error { + if l.state == lifecycleIncomplete { + return errShutdownIncomplete + } + return errLifecycleBusy +} + +// Start reconciles ownership left by earlier runs, then runs spec. +func (l *lifecycle) Start(spec launchSpec) (uint64, error) { + l.mu.Lock() + if l.state != lifecycleStopped { + err := l.busyLocked() + l.mu.Unlock() + return 0, err + } + l.state = lifecycleStarting + l.mu.Unlock() + + fail := func(state lifecycleState, err error) (uint64, error) { + l.mu.Lock() + l.state = state + l.mu.Unlock() + return 0, err + } + + // Starting requires confirmed absence of an owned process tree. Uncertain + // ownership blocks the start and nothing unverified is signalled. + if spec.LedgerPath != "" { + if _, err := sweepLedger(spec.LedgerPath); err != nil { + return fail(lifecycleStopped, err) + } + } + + cmd := exec.Command(spec.Binary, spec.Args...) + cmd.Env = spec.Env + cmd.Dir = spec.Dir + // Own process group: the runtime can be signalled as a unit, and a + // terminal interrupt aimed at the launcher does not reach it. + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + if spec.Attach != nil { + if err := spec.Attach(cmd); err != nil { + return fail(lifecycleStopped, err) + } + } + if err := cmd.Start(); err != nil { + return fail(lifecycleStopped, fmt.Errorf("failed to start the runtime: %w", err)) + } + + exited := make(chan struct{}) + l.mu.Lock() + l.generation++ + gen := l.generation + l.cmd, l.exited, l.spec = cmd, exited, spec + l.state = lifecycleRunning + l.mu.Unlock() + + go l.watch(gen, cmd, exited) + return gen, nil +} + +// watch notices an exit nobody asked for. Everything the runtime owned must +// still be cleaned up: a parent exit does not prove its descendants stopped. +func (l *lifecycle) watch(gen uint64, cmd *exec.Cmd, exited chan struct{}) { + waitErr := cmd.Wait() + close(exited) + + l.mu.Lock() + if l.generation != gen || l.state != lifecycleRunning { + l.mu.Unlock() // Stop owns the finish + return + } + l.state = lifecycleStopping + spec := l.spec + l.mu.Unlock() + + cleanupErr := l.reapOwned(cmd.Process.Pid, spec) + if waitErr == nil { + waitErr = errors.New("the runtime exited without being asked to stop") + } + l.finish(gen, spec, waitErr, cleanupErr) +} + +// finish records the end of a run. A failed cleanup leaves the controller +// blocked in the incomplete state. +func (l *lifecycle) finish(gen uint64, spec launchSpec, runErr, cleanupErr error) { + l.mu.Lock() + if cleanupErr != nil { + l.state = lifecycleIncomplete + l.lastErr = fmt.Errorf("%w: %v", errShutdownIncomplete, cleanupErr) + } else { + l.state = lifecycleStopped + l.lastErr = runErr + l.cmd, l.exited = nil, nil + } + l.lastGen = gen + reported := l.lastErr + l.mu.Unlock() + if spec.OnExit != nil { + spec.OnExit(gen, reported) + } +} + +// Stop runs the bounded stop sequence and returns only once the runtime and +// every owned group are confirmed gone. Calling it again after +// errShutdownIncomplete retries the cleanup. +func (l *lifecycle) Stop() error { + l.mu.Lock() + switch l.state { + case lifecycleStopped: + l.mu.Unlock() + return errNotRunning + case lifecycleStarting, lifecycleStopping: + l.mu.Unlock() + return errLifecycleBusy + } + retry := l.state == lifecycleIncomplete + l.state = lifecycleStopping + cmd, exited, spec, gen := l.cmd, l.exited, l.spec, l.generation + l.mu.Unlock() + + var stopErr error + if !retry { + stopErr = l.stopRuntime(cmd, exited) + } + pid := 0 + if cmd != nil && cmd.Process != nil { + pid = cmd.Process.Pid + } + cleanupErr := l.reapOwned(pid, spec) + if stopErr != nil && cleanupErr == nil { + cleanupErr = stopErr + } + l.finish(gen, spec, nil, cleanupErr) + if cleanupErr != nil { + return fmt.Errorf("%w: %v", errShutdownIncomplete, cleanupErr) + } + return nil +} + +// stopRuntime asks for a graceful exit, then forces the runtime's group. +func (l *lifecycle) stopRuntime(cmd *exec.Cmd, exited chan struct{}) error { + select { + case <-exited: + return nil + default: + } + _ = cmd.Process.Signal(syscall.SIGTERM) + timer := time.NewTimer(l.termGrace) + defer timer.Stop() + select { + case <-exited: + return nil + case <-timer.C: + } + _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + _ = cmd.Process.Kill() + reap := time.NewTimer(l.killGrace) + defer reap.Stop() + select { + case <-exited: + return nil + case <-reap.C: + return errors.New("the runtime did not exit after being killed") + } +} + +// reapOwned empties the runtime's own process group and every recorded backend +// group. It reports failure when any group cannot be proven empty. +func (l *lifecycle) reapOwned(runtimePID int, spec launchSpec) error { + var problems []string + if runtimePID > 0 && workerledger.GroupAlive(runtimePID) { + _ = syscall.Kill(-runtimePID, syscall.SIGKILL) + deadline := time.Now().Add(l.killGrace) + for workerledger.GroupAlive(runtimePID) { + if time.Now().After(deadline) { + problems = append(problems, fmt.Sprintf("runtime process group %d still has live members", runtimePID)) + break + } + time.Sleep(10 * time.Millisecond) + } + } + if spec.LedgerPath != "" { + if _, err := sweepLedger(spec.LedgerPath); err != nil { + problems = append(problems, err.Error()) + } + } + if len(problems) > 0 { + return errors.New(strings.Join(problems, "; ")) + } + return nil +} + +// sweepLedger kills the backend groups a ledger records and drops records whose +// group is verified gone. A record it cannot resolve (identity unverifiable, +// group that survives a kill, unreadable ledger) is kept, nothing unverified is +// signalled, and an error is returned: the caller must not start another role. +// The identity rules are those the worker uses to write the ledger. +func sweepLedger(path string) (int, error) { + entries, err := workerledger.ReadFile(path) + if err != nil { + return 0, err + } + if len(entries) == 0 { + return 0, nil + } + killed := 0 + var remaining []workerledger.Entry + var problems []string + for _, e := range entries { + switch workerledger.Classify(e) { + case workerledger.Gone, workerledger.Foreign: + case workerledger.Uncertain: + remaining = append(remaining, e) + problems = append(problems, fmt.Sprintf("%s (group %d: ownership cannot be verified)", e.Key, e.PID)) + case workerledger.Owned: + if err := workerledger.Reap(e); err != nil { + remaining = append(remaining, e) + problems = append(problems, fmt.Sprintf("%s (%v)", e.Key, err)) + continue + } + killed++ + } + } + if err := workerledger.WriteFile(path, remaining); err != nil { + problems = append(problems, fmt.Sprintf("cannot update the process ledger: %v", err)) + } + if len(problems) > 0 { + return killed, fmt.Errorf("backend ownership is unresolved: %s", strings.Join(problems, "; ")) + } + return killed, nil +} diff --git a/cmd/launcher/internal/lifecycle_lock.go b/cmd/launcher/internal/lifecycle_lock.go new file mode 100644 index 000000000000..940b4486ba60 --- /dev/null +++ b/cmd/launcher/internal/lifecycle_lock.go @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: MIT +package launcher + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + "golang.org/x/sys/unix" +) + +var errLauncherLocked = errors.New("another launcher already owns the runtime for this user") + +// userLock is the per-user launcher lock. It prevents a second launcher from +// starting a role that overlaps the first one's runtime. The kernel drops it +// when the holder dies, so a crash cannot leave a stale lock behind. The +// descriptor is close-on-exec: a runtime child never keeps the lock alive. +type userLock struct{ file *os.File } + +// acquireUserLock takes the lock file "launcher.lock" in dir without waiting. +func acquireUserLock(dir string) (*userLock, error) { + fd, err := unix.Open(filepath.Join(dir, "launcher.lock"), unix.O_RDWR|unix.O_CREAT|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0o600) + if err != nil { + return nil, fmt.Errorf("cannot open the launcher lock securely: %w", err) + } + f := os.NewFile(uintptr(fd), "launcher lock") + var st unix.Stat_t + if unix.Fstat(fd, &st) != nil || st.Mode&unix.S_IFMT != unix.S_IFREG || int64(st.Uid) != int64(os.Geteuid()) || st.Nlink != 1 { + f.Close() + return nil, errors.New("the launcher lock is not an owned regular file") + } + if err := f.Chmod(0o600); err != nil { + f.Close() + return nil, fmt.Errorf("cannot restrict the launcher lock: %w", err) + } + if err := unix.Flock(fd, unix.LOCK_EX|unix.LOCK_NB); err != nil { + f.Close() + if errors.Is(err, unix.EWOULDBLOCK) { + return nil, errLauncherLocked + } + return nil, fmt.Errorf("cannot lock the launcher lock: %w", err) + } + return &userLock{file: f}, nil +} + +// Release gives the lock up. It is safe to call twice. +func (l *userLock) Release() error { + if l == nil || l.file == nil { + return nil + } + f := l.file + l.file = nil + return f.Close() +} diff --git a/cmd/launcher/internal/lifecycle_test.go b/cmd/launcher/internal/lifecycle_test.go new file mode 100644 index 000000000000..e02fcdc2fda4 --- /dev/null +++ b/cmd/launcher/internal/lifecycle_test.go @@ -0,0 +1,312 @@ +// SPDX-License-Identifier: MIT +package launcher + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "sync" + "syscall" + "time" + + "github.com/mudler/LocalAI/pkg/workerledger" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// These specs run real processes and real process groups: the lifecycle +// controller exists to prove that nothing of an earlier role is still alive. +var _ = Describe("Serialized launcher lifecycle", func() { + var ( + dir string + ledger string + lc *lifecycle + ) + + scripts := 0 + script := func(body string) launchSpec { + // A new file per call: rewriting a script a shell is reading corrupts it. + scripts++ + path := filepath.Join(dir, fmt.Sprintf("runtime-%d.sh", scripts)) + Expect(os.WriteFile(path, []byte("#!/bin/sh\n"+body+"\n"), 0o700)).To(Succeed()) + return launchSpec{Binary: path, Dir: dir, LedgerPath: ledger} + } + // The marker proves the trap is installed: a signal sent earlier would take + // the shell's default action and the spec would test nothing. + const polite = `trap 'exit 0' TERM; : > "$(dirname "$0")/ready"; while :; do sleep 0.05; done` + const stubborn = `trap '' TERM; : > "$(dirname "$0")/ready"; while :; do sleep 0.05; done` + start := func(spec launchSpec) uint64 { + Expect(os.Remove(filepath.Join(dir, "ready"))).To(Or(Succeed(), MatchError(os.ErrNotExist))) + gen, err := lc.Start(spec) + Expect(err).ToNot(HaveOccurred()) + Eventually(func() error { _, err := os.Stat(filepath.Join(dir, "ready")); return err }, 5*time.Second, 10*time.Millisecond).Should(Succeed()) + return gen + } + + // backend starts a process in its own group, as a worker does, and records it. + backend := func(key string) int { + cmd := exec.Command("/bin/sleep", "300") + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + Expect(cmd.Start()).To(Succeed()) + go func() { _ = cmd.Wait() }() + DeferCleanup(func() { _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) }) + entries, err := workerledger.ReadFile(ledger) + Expect(err).ToNot(HaveOccurred()) + entries = append(entries, workerledger.Entry{Key: key, PID: cmd.Process.Pid, StartTime: workerledger.ReadStartTime(cmd.Process.Pid)}) + Expect(workerledger.WriteFile(ledger, entries)).To(Succeed()) + return cmd.Process.Pid + } + alive := func(pid int) func() bool { return func() bool { return workerledger.GroupAlive(pid) } } + + BeforeEach(func() { + dir = GinkgoT().TempDir() + ledger = filepath.Join(dir, "worker-processes.json") + lc = newLifecycle() + lc.termGrace = 400 * time.Millisecond + lc.killGrace = 2 * time.Second + realReap := workerledger.ReapTimeout + workerledger.ReapTimeout = time.Second + DeferCleanup(func() { workerledger.ReapTimeout = realReap }) + DeferCleanup(func() { _ = lc.Stop() }) + }) + + Describe("starting", func() { + It("runs one child and refuses a second start", func() { + gen, err := lc.Start(script(polite)) + Expect(err).ToNot(HaveOccurred()) + Expect(gen).ToNot(BeZero()) + Expect(lc.State()).To(Equal(lifecycleRunning)) + _, err = lc.Start(script(polite)) + Expect(errors.Is(err, errLifecycleBusy)).To(BeTrue()) + }) + + It("lets only one of several concurrent starts through", func() { + spec := script(polite) + var wg sync.WaitGroup + var mu sync.Mutex + succeeded := 0 + for i := 0; i < 8; i++ { + wg.Add(1) + go func() { + defer wg.Done() + if _, err := lc.Start(spec); err == nil { + mu.Lock() + succeeded++ + mu.Unlock() + } + }() + } + wg.Wait() + Expect(succeeded).To(Equal(1)) + }) + + It("kills a recorded backend group of an earlier run before it starts", func() { + pid := backend("model#0") + _, err := lc.Start(script(polite)) + Expect(err).ToNot(HaveOccurred()) + Expect(alive(pid)()).To(BeFalse()) + }) + + It("leaves a recycled process id alone", func() { + pid := backend("model#0") + entries, err := workerledger.ReadFile(ledger) + Expect(err).ToNot(HaveOccurred()) + entries[0].StartTime = "recycled" + Expect(workerledger.WriteFile(ledger, entries)).To(Succeed()) + _, err = lc.Start(script(polite)) + Expect(err).ToNot(HaveOccurred()) + Expect(alive(pid)()).To(BeTrue(), "an unrelated group is never signalled") + }) + + It("refuses to start over a record it cannot resolve and signals nothing", func() { + pid := backend("model#0") + real := workerledger.ReadStartTime + workerledger.ReadStartTime = func(int) string { return "" } + DeferCleanup(func() { workerledger.ReadStartTime = real }) + // A system that cannot read start times cannot prove ownership. + _, err := lc.Start(script(polite)) + Expect(err).To(HaveOccurred()) + Expect(lc.State()).To(Equal(lifecycleStopped)) + Expect(alive(pid)()).To(BeTrue()) + }) + + It("refuses to start over a corrupt ledger", func() { + Expect(os.WriteFile(ledger, []byte("{not json"), 0o600)).To(Succeed()) + _, err := lc.Start(script(polite)) + Expect(err).To(HaveOccurred()) + Expect(lc.State()).To(Equal(lifecycleStopped)) + }) + + It("reports a binary that cannot be executed and stays stopped", func() { + _, err := lc.Start(launchSpec{Binary: filepath.Join(dir, "missing"), Dir: dir, LedgerPath: ledger}) + Expect(err).To(HaveOccurred()) + Expect(lc.State()).To(Equal(lifecycleStopped)) + _, err = lc.Start(script(polite)) + Expect(err).ToNot(HaveOccurred()) + }) + }) + + Describe("stopping", func() { + It("asks for a graceful exit first", func() { + start(script(polite)) + began := time.Now() + Expect(lc.Stop()).To(Succeed()) + Expect(time.Since(began)).To(BeNumerically("<", lc.termGrace), "a child that honours SIGTERM is not killed") + Expect(lc.State()).To(Equal(lifecycleStopped)) + }) + + It("force-kills a child that ignores SIGTERM after the grace period", func() { + start(script(stubborn)) + pid := lc.PID() + began := time.Now() + Expect(lc.Stop()).To(Succeed()) + Expect(time.Since(began)).To(BeNumerically(">=", lc.termGrace)) + Expect(alive(pid)()).To(BeFalse(), "the whole runtime group is gone") + Expect(lc.State()).To(Equal(lifecycleStopped)) + }) + + It("kills backend groups that outlive the runtime", func() { + start(script(polite)) + first, second := backend("a#0"), backend("b#0") + Expect(lc.Stop()).To(Succeed()) + Expect(alive(first)()).To(BeFalse()) + Expect(alive(second)()).To(BeFalse()) + entries, err := workerledger.ReadFile(ledger) + Expect(err).ToNot(HaveOccurred()) + Expect(entries).To(BeEmpty(), "released ownership is dropped from the record") + }) + + It("kills a backend whose leader already exited while a member lives", func() { + start(script(polite)) + // The leader shell exits at once; its sleeping child keeps the group. + cmd := exec.Command("/bin/sh", "-c", "sleep 300 & exit 0") + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + Expect(cmd.Start()).To(Succeed()) + pid := cmd.Process.Pid + start := workerledger.ReadStartTime(pid) + DeferCleanup(func() { _ = syscall.Kill(-pid, syscall.SIGKILL) }) + Expect(workerledger.WriteFile(ledger, []workerledger.Entry{{Key: "m#0", PID: pid, StartTime: start}})).To(Succeed()) + _ = cmd.Wait() + Expect(alive(pid)()).To(BeTrue()) + Expect(lc.Stop()).To(Succeed()) + Expect(alive(pid)()).To(BeFalse()) + }) + + It("rejects start and a second stop while stopping", func() { + start(script(stubborn)) + done := make(chan error, 1) + go func() { done <- lc.Stop() }() + Eventually(lc.State).Should(Equal(lifecycleStopping)) + _, err := lc.Start(script(polite)) + Expect(errors.Is(err, errLifecycleBusy)).To(BeTrue()) + Expect(errors.Is(lc.Stop(), errLifecycleBusy)).To(BeTrue()) + Eventually(done, 5*time.Second).Should(Receive(BeNil())) + }) + + It("reports that nothing runs when nothing runs", func() { + Expect(errors.Is(lc.Stop(), errNotRunning)).To(BeTrue()) + }) + + It("is incomplete, blocks every start and keeps the record when a group cannot be emptied", func() { + start(script(polite)) + pid := backend("m#0") + realKill := workerledger.KillGroup + workerledger.KillGroup = func(int) error { return nil } // the signal never lands + workerledger.ReapTimeout = 200 * time.Millisecond + err := lc.Stop() + Expect(errors.Is(err, errShutdownIncomplete)).To(BeTrue()) + Expect(lc.State()).To(Equal(lifecycleIncomplete)) + _, err = lc.Start(script(polite)) + Expect(errors.Is(err, errShutdownIncomplete)).To(BeTrue(), "no replacement while ownership is unresolved") + entries, rerr := workerledger.ReadFile(ledger) + Expect(rerr).ToNot(HaveOccurred()) + Expect(entries).To(HaveLen(1)) + + // A retry with working signals finishes the job. + workerledger.KillGroup = realKill + Expect(lc.Stop()).To(Succeed()) + Expect(lc.State()).To(Equal(lifecycleStopped)) + Expect(alive(pid)()).To(BeFalse()) + }) + }) + + Describe("unexpected exit and stale callbacks", func() { + It("cleans up owned groups and records the exit when the runtime dies by itself", func() { + exits := make(chan uint64, 4) + spec := script(`sleep 0.2; exit 3`) + spec.OnExit = func(gen uint64, err error) { exits <- gen } + gen, err := lc.Start(spec) + Expect(err).ToNot(HaveOccurred()) + pid := backend("m#0") + Eventually(exits, 5*time.Second).Should(Receive(Equal(gen))) + Eventually(lc.State, 5*time.Second).Should(Equal(lifecycleStopped)) + Expect(alive(pid)()).To(BeFalse(), "a parent exit is cleanup work, not proof that descendants stopped") + g, exitErr := lc.LastExit() + Expect(g).To(Equal(gen)) + Expect(exitErr).To(HaveOccurred()) + }) + + It("ignores the exit of an older generation", func() { + var mu sync.Mutex + var seen []uint64 + record := func(gen uint64, _ error) { mu.Lock(); seen = append(seen, gen); mu.Unlock() } + first := script(polite) + first.OnExit = record + g1, err := lc.Start(first) + Expect(err).ToNot(HaveOccurred()) + Expect(lc.Stop()).To(Succeed()) + second := script(polite) + second.OnExit = record + g2, err := lc.Start(second) + Expect(err).ToNot(HaveOccurred()) + Expect(g2).To(BeNumerically(">", g1)) + Expect(lc.Running(g1)).To(BeFalse(), "a callback tagged with the old generation is stale") + Expect(lc.Running(g2)).To(BeTrue()) + mu.Lock() + defer mu.Unlock() + Expect(seen).To(ConsistOf(g1), "an expected stop is reported once, and never as the new generation") + }) + }) + + Describe("per-user lock", func() { + It("lets one launcher own the role at a time", func() { + first, err := acquireUserLock(dir) + Expect(err).ToNot(HaveOccurred()) + _, err = acquireUserLock(dir) + Expect(errors.Is(err, errLauncherLocked)).To(BeTrue()) + Expect(first.Release()).To(Succeed()) + again, err := acquireUserLock(dir) + Expect(err).ToNot(HaveOccurred()) + Expect(again.Release()).To(Succeed()) + }) + + It("is owner-only and refuses a symbolic link", func() { + lock, err := acquireUserLock(dir) + Expect(err).ToNot(HaveOccurred()) + info, err := os.Stat(filepath.Join(dir, "launcher.lock")) + Expect(err).ToNot(HaveOccurred()) + Expect(info.Mode().Perm()).To(Equal(os.FileMode(0o600))) + Expect(lock.Release()).To(Succeed()) + Expect(os.Remove(filepath.Join(dir, "launcher.lock"))).To(Succeed()) + Expect(os.Symlink(filepath.Join(dir, "elsewhere"), filepath.Join(dir, "launcher.lock"))).To(Succeed()) + _, err = acquireUserLock(dir) + Expect(err).To(HaveOccurred()) + }) + + It("is not inherited by the runtime child", func() { + // If the child kept the descriptor, a crashed launcher would leave a + // lock that a surviving runtime holds, or release it too early. + lock, err := acquireUserLock(dir) + Expect(err).ToNot(HaveOccurred()) + child := exec.Command("/bin/sleep", "300") + Expect(child.Start()).To(Succeed()) + DeferCleanup(func() { _ = child.Process.Kill(); _ = child.Wait() }) + Expect(lock.Release()).To(Succeed()) + again, err := acquireUserLock(dir) + Expect(err).ToNot(HaveOccurred(), "the live child must not hold the lock") + Expect(again.Release()).To(Succeed()) + }) + }) +}) diff --git a/cmd/launcher/internal/log_file.go b/cmd/launcher/internal/log_file.go new file mode 100644 index 000000000000..e799b9a3c73f --- /dev/null +++ b/cmd/launcher/internal/log_file.go @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: MIT +package launcher + +import ( + "crypto/rand" + "encoding/hex" + "fmt" + "golang.org/x/sys/unix" + "os" + "path/filepath" +) + +func secureLogFile(dataPath string) (*os.File, error) { + fail := func() (*os.File, error) { + return nil, fmt.Errorf("cannot create secure launcher log; check directory ownership and permissions") + } + path := filepath.Join(dataPath, "logs") + if e := os.Mkdir(path, 0700); e != nil && !os.IsExist(e) { + return fail() + } + fd, e := unix.Open(path, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0) + if e != nil { + return fail() + } + defer unix.Close(fd) + var st unix.Stat_t + if unix.Fstat(fd, &st) != nil || int64(st.Uid) != int64(os.Geteuid()) || st.Mode&0022 != 0 { + return fail() + } + if unix.Fchmod(fd, 0700) != nil { + return fail() + } + nonce := make([]byte, 16) + if _, e = rand.Read(nonce); e != nil { + return fail() + } + name := "localai_" + hex.EncodeToString(nonce) + ".log" + f, e := unix.Openat(fd, name, unix.O_WRONLY|unix.O_CREAT|unix.O_EXCL|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0600) + if e != nil { + return fail() + } + return os.NewFile(uintptr(f), filepath.Join(path, name)), nil +} diff --git a/cmd/launcher/internal/log_filter.go b/cmd/launcher/internal/log_filter.go new file mode 100644 index 000000000000..a5cf82870b44 --- /dev/null +++ b/cmd/launcher/internal/log_filter.go @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: MIT +package launcher + +import ( + "bufio" + "io" + "regexp" + "strings" +) + +const maxLogLine = 16 * 1024 +const maxLogFile = 1024 * 1024 +const maxLogBuffer = 100000 +const omittedLog = "[unsafe or incomplete log line omitted]" + +var sensitiveLog = regexp.MustCompile(`(?i)authorization|bearer\s|(?:token|password|secret|nats_jwt|user_seed)["']?\s*[=:]|[a-z][a-z0-9+.-]*://\S*[@?#]|eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+|S[UA][A-Z2-7]{40,}`) + +type logFilter struct{ secrets []string } + +func newLogFilter(secrets []string) logFilter { return logFilter{append([]string(nil), secrets...)} } +func (f logFilter) clean(line string) string { + if controls(line) || sensitiveLog.MatchString(line) { + return omittedLog + } + for _, s := range f.secrets { + if s != "" && strings.Contains(line, s) { + return omittedLog + } + } + return line +} + +// ReadSlice bounds memory and holds fragments until the newline. Overlong and +// unterminated records are discarded, never emitted as independently safe pieces. +func (f logFilter) consume(r io.Reader, emit func(string)) { + b := bufio.NewReaderSize(r, maxLogLine) + dropping := false + for { + line, e := b.ReadSlice('\n') + if e == bufio.ErrBufferFull { + dropping = true + continue + } + if e != nil { + if len(line) > 0 || dropping { + emit(omittedLog) + } + return + } + if dropping { + emit(omittedLog) + dropping = false + continue + } + emit(f.clean(strings.TrimSuffix(strings.TrimSuffix(string(line), "\n"), "\r"))) + } +} diff --git a/cmd/launcher/internal/settings_apply_test.go b/cmd/launcher/internal/settings_apply_test.go new file mode 100644 index 000000000000..54e922950a82 --- /dev/null +++ b/cmd/launcher/internal/settings_apply_test.go @@ -0,0 +1,203 @@ +// SPDX-License-Identifier: MIT +package launcher + +import ( + "errors" + "os" + "os/exec" + "path/filepath" + "syscall" + "time" + + "github.com/mudler/LocalAI/pkg/workerledger" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// Role changes and launch-setting changes never overlap a live runtime: the +// owner confirms a stop, the stop is verified, and only then is the new profile +// committed. Nothing starts the replacement automatically. +var _ = Describe("Applying settings to a running launcher", func() { + var ( + dir string + runtime string + l *Launcher + ) + + cfg := func(mode string) *Config { + c := &Config{Mode: mode, ModelsPath: filepath.Join(dir, "models"), BackendsPath: filepath.Join(dir, "backends"), Address: ":8080", LogLevel: "info"} + c.Worker.Connection = WorkerConnectionConfig{FrontendURL: "https://frontend.example", RegistrationToken: "token-secret", AdvertisedAddress: "worker.example:50051"} + c.Worker.NATS.URL = "tls://nats.example:4222" + return c + } + runStandalone := func() { + body := "#!/bin/sh\ntrap 'exit 0' TERM\n: > \"$(dirname \"$0\")/ready\"\nwhile :; do sleep 0.05; done\n" + Expect(os.WriteFile(runtime, []byte(body), 0o700)).To(Succeed()) + d := launchDescription{role: roleStandalone, args: []string{"run"}, env: []string{"PATH=/usr/bin:/bin"}, dir: dir} + Expect(l.launchRuntime(runtime, d)).To(Succeed()) + Eventually(func() error { _, err := os.Stat(filepath.Join(dir, "ready")); return err }, 5*time.Second, 10*time.Millisecond).Should(Succeed()) + } + saved := func() string { + data, err := os.ReadFile(filepath.Join(dir, ".localai", "launcher.json")) + Expect(err).ToNot(HaveOccurred()) + return string(data) + } + + BeforeEach(func() { + dir = GinkgoT().TempDir() + GinkgoT().Setenv("HOME", dir) + Expect(os.MkdirAll(filepath.Join(dir, ".localai"), 0o700)).To(Succeed()) + runtime = filepath.Join(dir, "local-ai") + l = &Launcher{config: cfg("standalone"), lockDir: dir, life: newLifecycle()} + l.life.termGrace = 500 * time.Millisecond + l.life.killGrace = 2 * time.Second + Expect(l.SetConfig(l.config)).To(Succeed()) + DeferCleanup(func() { _ = l.Shutdown() }) + }) + + It("commits a change at once when nothing runs", func() { + Expect(l.ApplySettings(cfg("worker"), false)).To(Succeed()) + Expect(l.GetConfig().Mode).To(Equal("worker")) + Expect(saved()).To(ContainSubstring(`"mode": "worker"`)) + Expect(l.IsRunning()).To(BeFalse(), "applying never starts a role") + }) + + It("asks for a confirmed stop before a role switch and changes nothing until then", func() { + runStandalone() + before := saved() + err := l.ApplySettings(cfg("worker"), false) + Expect(errors.Is(err, errStopRequired)).To(BeTrue()) + Expect(l.GetConfig().Mode).To(Equal("standalone"), "cancel leaves the active configuration unchanged") + Expect(saved()).To(Equal(before)) + Expect(l.IsRunning()).To(BeTrue()) + }) + + It("stops, verifies, then commits, and does not start the replacement", func() { + runStandalone() + pid := l.life.PID() + Expect(l.ApplySettings(cfg("worker"), true)).To(Succeed()) + Expect(l.IsRunning()).To(BeFalse()) + Expect(workerledger.GroupAlive(pid)).To(BeFalse()) + Expect(l.GetConfig().Mode).To(Equal("worker")) + Expect(saved()).To(ContainSubstring(`"mode": "worker"`)) + }) + + It("asks for a stop when a launch setting of the active profile changes", func() { + runStandalone() + next := cfg("standalone") + next.Address = ":9090" + Expect(errors.Is(l.ApplySettings(next, false), errStopRequired)).To(BeTrue()) + next = cfg("standalone") + next.ModelsPath = filepath.Join(dir, "other-models") + Expect(errors.Is(l.ApplySettings(next, false), errStopRequired)).To(BeTrue()) + }) + + It("lets an inactive profile and unrelated settings change under a running role", func() { + runStandalone() + next := cfg("standalone") + next.Worker.Connection.FrontendURL = "https://other.example" + welcome := false + next.ShowWelcome = &welcome + Expect(l.ApplySettings(next, false)).To(Succeed()) + Expect(l.IsRunning()).To(BeTrue(), "the running role is untouched") + Expect(l.GetConfig().Worker.Connection.FrontendURL).To(Equal("https://other.example")) + }) + + It("does not commit and keeps stop available when shutdown is incomplete", func() { + body := "#!/bin/sh\ntrap 'exit 0' TERM\n: > \"$(dirname \"$0\")/ready\"\nwhile :; do sleep 0.05; done\n" + Expect(os.WriteFile(runtime, []byte(body), 0o700)).To(Succeed()) + ledger := filepath.Join(dir, "data", "worker-processes.json") + d := launchDescription{role: roleStandalone, args: []string{"run"}, env: []string{"PATH=/usr/bin:/bin"}, dir: dir, ledger: ledger} + Expect(l.launchRuntime(runtime, d)).To(Succeed()) + Eventually(func() error { _, err := os.Stat(filepath.Join(dir, "ready")); return err }, 5*time.Second, 10*time.Millisecond).Should(Succeed()) + backend := exec.Command("/bin/sleep", "300") + backend.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + Expect(backend.Start()).To(Succeed()) + go func() { _ = backend.Wait() }() + DeferCleanup(func() { _ = syscall.Kill(-backend.Process.Pid, syscall.SIGKILL) }) + Expect(workerledger.WriteFile(ledger, []workerledger.Entry{{Key: "m#0", PID: backend.Process.Pid, StartTime: workerledger.ReadStartTime(backend.Process.Pid)}})).To(Succeed()) + + realKill, realReap := workerledger.KillGroup, workerledger.ReapTimeout + workerledger.KillGroup = func(int) error { return nil } // the signal never lands + workerledger.ReapTimeout = 200 * time.Millisecond + DeferCleanup(func() { workerledger.KillGroup, workerledger.ReapTimeout = realKill, realReap }) + + before := saved() + err := l.ApplySettings(cfg("worker"), true) + Expect(errors.Is(err, errShutdownIncomplete)).To(BeTrue()) + Expect(l.GetConfig().Mode).To(Equal("standalone"), "a role is never switched over unresolved ownership") + Expect(saved()).To(Equal(before)) + Expect(l.IsRunning()).To(BeTrue(), "stop stays available for a retry") + workerledger.KillGroup = realKill + }) + + It("refuses start, save and a second apply while a stop-and-apply is under way", func() { + body := "#!/bin/sh\ntrap '' TERM\n: > \"$(dirname \"$0\")/ready\"\nwhile :; do sleep 0.05; done\n" + Expect(os.WriteFile(runtime, []byte(body), 0o700)).To(Succeed()) + d := launchDescription{role: roleStandalone, args: []string{"run"}, env: []string{"PATH=/usr/bin:/bin"}, dir: dir} + Expect(l.launchRuntime(runtime, d)).To(Succeed()) + Eventually(func() error { _, err := os.Stat(filepath.Join(dir, "ready")); return err }, 5*time.Second, 10*time.Millisecond).Should(Succeed()) + done := make(chan error, 1) + go func() { done <- l.ApplySettings(cfg("worker"), true) }() + Eventually(l.Applying).Should(BeTrue()) + Expect(errors.Is(l.ApplySettings(cfg("worker"), true), errLifecycleBusy)).To(BeTrue()) + Expect(errors.Is(l.StartLocalAI(), errLifecycleBusy)).To(BeTrue()) + Expect(errors.Is(l.DownloadUpdate("v0.0.0", nil), errLifecycleBusy)).To(BeTrue()) + Eventually(done, 5*time.Second).Should(Receive(BeNil())) + Expect(l.Applying()).To(BeFalse()) + }) + + Describe("starting the selected mode when the launcher opens", func() { + It("keeps the legacy default for standalone and requires an explicit choice for a worker", func() { + c := cfg("standalone") + l.config = c + Expect(l.ShouldAutoStartServer()).To(BeTrue(), "unset still means enabled for standalone") + c = cfg("worker") + l.config = c + Expect(l.ShouldAutoStartServer()).To(BeFalse(), "joining a deployment is never an unasked default") + yes := true + c.AutoStart = &yes + Expect(l.ShouldAutoStartServer()).To(BeTrue()) + Expect(l.GetConfig().Mode).To(Equal("worker")) + }) + }) + + Describe("the deployment UI", func() { + It("is the saved frontend URL without credentials", func() { + c := cfg("worker") + c.Worker.Connection.FrontendURL = "https://frontend.example/prefix/" + l.config = c + u, err := l.DeploymentURL() + Expect(err).ToNot(HaveOccurred()) + Expect(u).To(Equal("https://frontend.example/prefix")) + Expect(u).ToNot(ContainSubstring("token-secret")) + Expect(l.GetWebUIURL()).To(Equal(u), "worker mode has no local server UI") + }) + + It("refuses an unsafe or missing frontend URL", func() { + for _, bad := range []string{"", "https://user:pw@frontend.example", "ftp://frontend.example", "https://frontend.example?x=1"} { + c := cfg("worker") + c.Worker.Connection.FrontendURL = bad + l.config = c + _, err := l.DeploymentURL() + Expect(err).To(HaveOccurred(), bad) + } + }) + }) + + Describe("plaintext connections", func() { + It("start only after an explicit acknowledgement for this launcher session", func() { + c := cfg("worker") + c.Worker.Connection.FrontendURL = "http://frontend.example" + l.config = c + _, err := l.describeLaunch() + Expect(err).To(HaveOccurred()) + l.SetPlaintextAcknowledged(true) + _, err = l.describeLaunch() + Expect(err).ToNot(HaveOccurred()) + l.SetPlaintextAcknowledged(false) + _, err = l.describeLaunch() + Expect(err).To(HaveOccurred()) + }) + }) +}) diff --git a/cmd/launcher/internal/settings_form.go b/cmd/launcher/internal/settings_form.go new file mode 100644 index 000000000000..e954a7609564 --- /dev/null +++ b/cmd/launcher/internal/settings_form.go @@ -0,0 +1,135 @@ +// SPDX-License-Identifier: MIT +package launcher + +import ( + "fmt" + "net" + "sort" + "strconv" + "strings" +) + +// secretEdit is the state of one masked field. A field left blank keeps the +// saved value: saving something else never clears a secret. +type secretEdit struct { + // Saved is true when the profile holds a value. The value itself is never + // copied into the form. + Saved bool + Replacement string + Clear bool +} + +// workerForm is the editable text of the worker profile. It holds no secret +// values, only edits to them. +type workerForm struct { + FrontendURL, AdvertisedAddress, NodeName, GRPCMaxPort string + NATSURL, TLSCA, TLSCert, TLSKey string + // EnvLines is one KEY=VALUE per line. + EnvLines string + Token, JWT, Seed secretEdit +} + +func newWorkerForm(c *Config) workerForm { + w := c.Worker + f := workerForm{ + FrontendURL: w.Connection.FrontendURL, + AdvertisedAddress: w.Connection.AdvertisedAddress, + NodeName: w.Connection.NodeName, + NATSURL: w.NATS.URL, + TLSCA: w.NATS.TLSCA, + TLSCert: w.NATS.TLSCert, + TLSKey: w.NATS.TLSKey, + Token: secretEdit{Saved: w.Connection.RegistrationToken != ""}, + JWT: secretEdit{Saved: w.NATS.JWT != ""}, + Seed: secretEdit{Saved: w.NATS.UserSeed != ""}, + } + if w.Connection.GRPCMaxPort != nil { + f.GRPCMaxPort = strconv.Itoa(*w.Connection.GRPCMaxPort) + } + keys := make([]string, 0, len(w.EnvironmentVars)) + for k := range w.EnvironmentVars { + keys = append(keys, k) + } + sort.Strings(keys) + lines := make([]string, 0, len(keys)) + for _, k := range keys { + lines = append(lines, k+"="+w.EnvironmentVars[k]) + } + f.EnvLines = strings.Join(lines, "\n") + return f +} + +func (e secretEdit) resolve(saved string) string { + switch { + case e.Clear: + return "" + case e.Replacement != "": + return e.Replacement + default: + return saved + } +} + +// Apply returns a copy of base with the form's worker profile. Saving needs no +// semantic validation: a blank or invalid profile is valid to retain, and the +// launch builder refuses to start it. Only text that cannot be stored is +// rejected, and errors never echo values. +func (f workerForm) Apply(base *Config) (*Config, error) { + next := base.clone() + c := &next.Worker.Connection + n := &next.Worker.NATS + c.FrontendURL, c.AdvertisedAddress, c.NodeName = f.FrontendURL, f.AdvertisedAddress, f.NodeName + n.URL, n.TLSCA, n.TLSCert, n.TLSKey = f.NATSURL, f.TLSCA, f.TLSCert, f.TLSKey + c.RegistrationToken = f.Token.resolve(c.RegistrationToken) + n.JWT = f.JWT.resolve(n.JWT) + n.UserSeed = f.Seed.resolve(n.UserSeed) + + c.GRPCMaxPort = nil + if s := strings.TrimSpace(f.GRPCMaxPort); s != "" { + port, err := decimalPort(s) + if err != nil { + return nil, fmt.Errorf("maximum gRPC port must be a number") + } + c.GRPCMaxPort = &port + } + + env := map[string]string{} + for i, line := range strings.Split(f.EnvLines, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + k, v, ok := strings.Cut(line, "=") + if !ok || strings.TrimSpace(k) == "" { + return nil, fmt.Errorf("environment line %d must be KEY=VALUE", i+1) + } + k = strings.TrimSpace(k) + if reservedLaunchKey(k) { + return nil, fmt.Errorf("environment line %d sets a reserved worker setting; use the connection settings", i+1) + } + env[k] = v + } + next.Worker.EnvironmentVars = env + return next, nil +} + +// reachabilityPreview explains which ports the frontend must reach, derived +// from the advertised address the worker will use. +func reachabilityPreview(address, maxPort string) (string, error) { + _, p, err := net.SplitHostPort(address) + if err != nil { + return "", fmt.Errorf("enter the address as host:port") + } + base, err := decimalPort(p) + if err != nil || base < 2 { + return "", fmt.Errorf("the base port must be 2-65535") + } + top := 65535 + if s := strings.TrimSpace(maxPort); s != "" { + top, err = decimalPort(s) + if err != nil || top < base { + return "", fmt.Errorf("the maximum gRPC port must be %d-65535", base) + } + } + return fmt.Sprintf("The frontend must reach reverse HTTP on port %d and gRPC on ports %d-%d. Each concurrent backend takes one gRPC port; choose a narrow range that fits expected concurrency. The worker needs outbound access to the frontend and the message bus.", base-1, base, top), nil +} diff --git a/cmd/launcher/internal/settings_form_test.go b/cmd/launcher/internal/settings_form_test.go new file mode 100644 index 000000000000..98f64cdefe32 --- /dev/null +++ b/cmd/launcher/internal/settings_form_test.go @@ -0,0 +1,128 @@ +// SPDX-License-Identifier: MIT +package launcher + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// The worker form edits a copy of the saved profile. Masked secrets are never +// shown, never lost by saving something else, and only changed on request. +var _ = Describe("Worker settings form", func() { + var base *Config + BeforeEach(func() { + base = &Config{Mode: "worker"} + base.Worker.Connection = WorkerConnectionConfig{FrontendURL: "https://frontend.example", RegistrationToken: "token-secret", AdvertisedAddress: "worker.example:50051", NodeName: "node"} + base.Worker.NATS = WorkerNATSConfig{URL: "tls://nats.example:4222", JWT: "jwt-secret", UserSeed: "seed-secret"} + base.Worker.EnvironmentVars = map[string]string{"CUDA_VISIBLE_DEVICES": "1"} + }) + + It("loads the visible fields and never the secrets", func() { + f := newWorkerForm(base) + Expect(f.FrontendURL).To(Equal("https://frontend.example")) + Expect(f.NATSURL).To(Equal("tls://nats.example:4222")) + Expect(f.EnvLines).To(Equal("CUDA_VISIBLE_DEVICES=1")) + Expect(f.Token.Replacement + f.JWT.Replacement + f.Seed.Replacement).To(BeEmpty()) + Expect(f.Token.Saved).To(BeTrue()) + Expect(f.JWT.Saved).To(BeTrue()) + }) + + It("keeps every saved secret when an unrelated field changes", func() { + f := newWorkerForm(base) + f.NodeName = "renamed" + got, err := f.Apply(base) + Expect(err).ToNot(HaveOccurred()) + Expect(got.Worker.Connection.NodeName).To(Equal("renamed")) + Expect(got.Worker.Connection.RegistrationToken).To(Equal("token-secret")) + Expect(got.Worker.NATS.JWT).To(Equal("jwt-secret")) + Expect(got.Worker.NATS.UserSeed).To(Equal("seed-secret")) + }) + + It("replaces or clears a secret only when asked", func() { + f := newWorkerForm(base) + f.Token.Replacement = "new-token" + f.JWT.Clear, f.Seed.Clear = true, true + got, err := f.Apply(base) + Expect(err).ToNot(HaveOccurred()) + Expect(got.Worker.Connection.RegistrationToken).To(Equal("new-token")) + Expect(got.Worker.NATS.JWT).To(BeEmpty()) + Expect(got.Worker.NATS.UserSeed).To(BeEmpty()) + }) + + It("does not modify the configuration it was given", func() { + f := newWorkerForm(base) + f.NodeName = "renamed" + f.Token.Clear = true + _, err := f.Apply(base) + Expect(err).ToNot(HaveOccurred()) + Expect(base.Worker.Connection.NodeName).To(Equal("node")) + Expect(base.Worker.Connection.RegistrationToken).To(Equal("token-secret")) + }) + + It("keeps the standalone profile untouched", func() { + base.EnvironmentVars = map[string]string{"STANDALONE": "x"} + base.Address = ":8123" + f := newWorkerForm(base) + f.FrontendURL = "https://elsewhere.example" + got, err := f.Apply(base) + Expect(err).ToNot(HaveOccurred()) + Expect(got.EnvironmentVars).To(Equal(map[string]string{"STANDALONE": "x"})) + Expect(got.Address).To(Equal(":8123")) + }) + + It("parses the maximum gRPC port and rejects a non-number", func() { + f := newWorkerForm(base) + f.GRPCMaxPort = "50100" + got, err := f.Apply(base) + Expect(err).ToNot(HaveOccurred()) + Expect(*got.Worker.Connection.GRPCMaxPort).To(Equal(50100)) + f.GRPCMaxPort = "" + got, err = f.Apply(base) + Expect(err).ToNot(HaveOccurred()) + Expect(got.Worker.Connection.GRPCMaxPort).To(BeNil(), "empty means the worker default") + f.GRPCMaxPort = "5x" + _, err = f.Apply(base) + Expect(err).To(HaveOccurred()) + }) + + It("parses environment lines and refuses a reserved key without echoing the value", func() { + f := newWorkerForm(base) + f.EnvLines = "CUDA_VISIBLE_DEVICES=2\n\nHSA_OVERRIDE=11.0\n" + got, err := f.Apply(base) + Expect(err).ToNot(HaveOccurred()) + Expect(got.Worker.EnvironmentVars).To(Equal(map[string]string{"CUDA_VISIBLE_DEVICES": "2", "HSA_OVERRIDE": "11.0"})) + for _, line := range []string{"LOCALAI_NATS_REQUIRE_AUTH=false", "LOCALAI_REGISTER_TO=secret-value", "NOEQUALS"} { + f.EnvLines = line + _, err = f.Apply(base) + Expect(err).To(HaveOccurred(), line) + Expect(err.Error()).ToNot(ContainSubstring("secret-value")) + } + }) + + It("does not validate the profile to save it: a blank or invalid inactive draft is retained", func() { + f := newWorkerForm(&Config{}) + f.FrontendURL = "not a url" + got, err := f.Apply(&Config{}) + Expect(err).ToNot(HaveOccurred()) + Expect(got.Worker.Connection.FrontendURL).To(Equal("not a url")) + }) + + Describe("reachability preview", func() { + It("shows the reverse HTTP port and the whole gRPC range", func() { + text, err := reachabilityPreview("worker.example:50051", "") + Expect(err).ToNot(HaveOccurred()) + Expect(text).To(ContainSubstring("50050")) + Expect(text).To(ContainSubstring("50051")) + Expect(text).To(ContainSubstring("65535")) + text, err = reachabilityPreview("[2001:db8::1]:50051", "50060") + Expect(err).ToNot(HaveOccurred()) + Expect(text).To(ContainSubstring("50051-50060")) + }) + It("reports an address or range it cannot derive", func() { + for _, c := range [][2]string{{"worker.example", ""}, {"worker.example:1", ""}, {"worker.example:50051", "50000"}, {"worker.example:50051", "70000"}} { + _, err := reachabilityPreview(c[0], c[1]) + Expect(err).To(HaveOccurred(), c[0]+" "+c[1]) + } + }) + }) +}) diff --git a/cmd/launcher/internal/settings_ui_test.go b/cmd/launcher/internal/settings_ui_test.go new file mode 100644 index 000000000000..033dcc7c8d1b --- /dev/null +++ b/cmd/launcher/internal/settings_ui_test.go @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: MIT +package launcher + +import ( + "path/filepath" + + "fyne.io/fyne/v2/test" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// The settings window starts with the mode selector. Both profiles keep their +// drafts when the owner switches, and saving never loses a masked secret. +var _ = Describe("Launcher settings window", func() { + var ( + ui *LauncherUI + l *Launcher + ) + + BeforeEach(func() { + DeferCleanup(test.NewApp().Quit) + dir := GinkgoT().TempDir() + GinkgoT().Setenv("HOME", dir) + ui = NewLauncherUI() + c := &Config{Mode: "standalone", ModelsPath: filepath.Join(dir, "models"), BackendsPath: filepath.Join(dir, "backends"), Address: ":8080", LogLevel: "info"} + c.Worker.Connection = WorkerConnectionConfig{FrontendURL: "https://frontend.example", RegistrationToken: "token-secret", AdvertisedAddress: "worker.example:50051"} + c.Worker.NATS = WorkerNATSConfig{URL: "tls://nats.example:4222", JWT: "jwt-secret", UserSeed: "seed-secret"} + l = &Launcher{config: c, lockDir: dir, life: newLifecycle(), ui: ui, releaseManager: NewReleaseManager()} + ui.CreateMainUI(l) + ui.LoadConfiguration() + }) + + It("offers the two modes and shows the local settings first for an existing install", func() { + Expect(ui.modeRadio.Options).To(Equal([]string{modeLocalLabel, modeJoinLabel})) + Expect(ui.modeRadio.Selected).To(Equal(modeLocalLabel)) + Expect(ui.localBox.Visible()).To(BeTrue()) + Expect(ui.workerBox.Visible()).To(BeFalse()) + }) + + It("opens on the worker settings when the saved mode is worker", func() { + l.config.Mode = "worker" + ui.LoadConfiguration() + Expect(ui.modeRadio.Selected).To(Equal(modeJoinLabel)) + Expect(ui.workerBox.Visible()).To(BeTrue()) + Expect(ui.localBox.Visible()).To(BeFalse()) + }) + + It("keeps both drafts when switching back and forth", func() { + ui.addressEntry.SetText(":9999") + ui.modeRadio.SetSelected(modeJoinLabel) + ui.frontendEntry.SetText("https://draft.example") + ui.modeRadio.SetSelected(modeLocalLabel) + ui.modeRadio.SetSelected(modeJoinLabel) + Expect(ui.frontendEntry.Text).To(Equal("https://draft.example")) + Expect(ui.addressEntry.Text).To(Equal(":9999")) + got, err := ui.collectConfig() + Expect(err).ToNot(HaveOccurred()) + Expect(got.Mode).To(Equal("worker")) + Expect(got.Address).To(Equal(":9999")) + Expect(got.Worker.Connection.FrontendURL).To(Equal("https://draft.example")) + }) + + It("masks secret fields, never shows their values and says they are saved", func() { + Expect(ui.tokenEntry.Password).To(BeTrue()) + Expect(ui.jwtEntry.Password).To(BeTrue()) + Expect(ui.seedEntry.Password).To(BeTrue()) + for _, text := range []string{ui.tokenEntry.Text, ui.jwtEntry.Text, ui.seedEntry.Text} { + Expect(text).To(BeEmpty()) + } + Expect(ui.tokenEntry.PlaceHolder).To(ContainSubstring("Saved")) + }) + + It("keeps secrets when unrelated settings are saved, and changes them only on request", func() { + ui.modeRadio.SetSelected(modeJoinLabel) + ui.nodeNameEntry.SetText("renamed") + got, err := ui.collectConfig() + Expect(err).ToNot(HaveOccurred()) + Expect(got.Worker.Connection.RegistrationToken).To(Equal("token-secret")) + Expect(got.Worker.NATS.JWT).To(Equal("jwt-secret")) + Expect(got.Worker.NATS.UserSeed).To(Equal("seed-secret")) + + ui.tokenEntry.SetText("new-token") + ui.clearJWTCheck.SetChecked(true) + got, err = ui.collectConfig() + Expect(err).ToNot(HaveOccurred()) + Expect(got.Worker.Connection.RegistrationToken).To(Equal("new-token")) + Expect(got.Worker.NATS.JWT).To(BeEmpty()) + Expect(got.Worker.NATS.UserSeed).To(Equal("seed-secret")) + }) + + It("saves an invalid inactive draft but reports an invalid saved number", func() { + ui.modeRadio.SetSelected(modeLocalLabel) + ui.frontendEntry.SetText("not a url") + got, err := ui.collectConfig() + Expect(err).ToNot(HaveOccurred()) + Expect(got.Worker.Connection.FrontendURL).To(Equal("not a url")) + ui.grpcMaxEntry.SetText("abc") + _, err = ui.collectConfig() + Expect(err).To(HaveOccurred()) + }) + + It("labels the automatic start for the selected mode, not for LocalAI only", func() { + Expect(ui.autoStartCheck.Text).To(Equal("Start selected mode when launcher opens")) + }) + + It("names the controls for the selected mode", func() { + ui.UpdateRunningState(false) + Expect(ui.startStopButton.Text).To(Equal("Start LocalAI")) + l.config.Mode = "worker" + ui.UpdateRunningState(false) + Expect(ui.startStopButton.Text).To(Equal("Start worker")) + ui.UpdateRunningState(true) + Expect(ui.startStopButton.Text).To(Equal("Stop worker")) + Expect(ui.webUIButton.Text).To(Equal("Open deployment UI")) + Expect(ui.webUIButton.Disabled()).To(BeFalse(), "the deployment UI does not depend on the worker running") + }) + + It("explains reverse connectivity and shows the derived ports", func() { + ui.modeRadio.SetSelected(modeJoinLabel) + ui.advertisedEntry.SetText("worker.example:50051") + ui.grpcMaxEntry.SetText("50060") + Expect(ui.previewLabel.Text).To(ContainSubstring("50050")) + Expect(ui.previewLabel.Text).To(ContainSubstring("50051-50060")) + ui.advertisedEntry.SetText("worker.example") + Expect(ui.previewLabel.Text).To(ContainSubstring("host:port")) + Expect(ui.joinExplanation.Text).To(ContainSubstring("does not create a deployment")) + }) + + It("records the plaintext acknowledgement for this session only", func() { + Expect(l.PlaintextAcknowledged()).To(BeFalse()) + ui.plaintextCheck.SetChecked(true) + Expect(l.PlaintextAcknowledged()).To(BeTrue()) + got, err := ui.collectConfig() + Expect(err).ToNot(HaveOccurred()) + data, merr := got.MarshalJSON() + Expect(merr).ToNot(HaveOccurred()) + Expect(string(data)).ToNot(ContainSubstring("plaintext")) + }) +}) diff --git a/cmd/launcher/internal/systray_manager.go b/cmd/launcher/internal/systray_manager.go index ba0fb238524b..c7400daea084 100644 --- a/cmd/launcher/internal/systray_manager.go +++ b/cmd/launcher/internal/systray_manager.go @@ -75,7 +75,9 @@ func (sm *SystrayManager) toggleLocalAI() { // openWebUI opens the LocalAI WebUI in the default browser func (sm *SystrayManager) openWebUI() { - if !sm.launcher.IsRunning() { + // A worker has no local server: its entry opens the deployment UI, which + // does not depend on the worker running. + if !sm.launcher.IsRunning() && sm.launcher.GetConfig().Mode != roleWorker { return // LocalAI is not running } @@ -98,29 +100,52 @@ func (sm *SystrayManager) updateStartStopItem() { sm.recreateMenu() } +// roleMenuItems builds the start/stop and open-UI entries for a role. In worker +// mode there is no local server UI, so the entry opens the deployment UI. +func roleMenuItems(role string, running, hasDeploymentURL bool, toggle, openUI func()) []*fyne.MenuItem { + if role == roleWorker { + label := "Start worker" + if running { + label = "Stop worker" + } + items := []*fyne.MenuItem{fyne.NewMenuItem(label, toggle)} + if hasDeploymentURL { + items = append(items, fyne.NewMenuItem("Open deployment UI", openUI)) + } + return items + } + label := "▶️ Start LocalAI" + if running { + label = "🛑 Stop LocalAI" + } + items := []*fyne.MenuItem{fyne.NewMenuItem(label, toggle)} + if running { + items = append(items, fyne.NewMenuItem("Open WebUI", openUI)) + } + return items +} + // recreateMenu recreates the entire menu with updated state func (sm *SystrayManager) recreateMenu() { if sm.desk == nil { return } - // Determine the action based on LocalAI installation and running state - var actionItem *fyne.MenuItem - if !sm.launcher.GetReleaseManager().IsLocalAIInstalled() { + // Determine the actions based on LocalAI installation, mode and running state + installed := sm.launcher.GetReleaseManager().IsLocalAIInstalled() + role := roleStandalone + if cfg := sm.launcher.GetConfig(); cfg != nil && cfg.Mode == roleWorker { + role = roleWorker + } + _, urlErr := sm.launcher.DeploymentURL() + var roleItems []*fyne.MenuItem + if !installed { // LocalAI not installed - show install option - actionItem = fyne.NewMenuItem("📥 Install Latest Version", func() { + roleItems = []*fyne.MenuItem{fyne.NewMenuItem("📥 Install Latest Version", func() { sm.launcher.showDownloadLocalAIDialog() - }) - } else if sm.launcher.IsRunning() { - // LocalAI is running - show stop option - actionItem = fyne.NewMenuItem("🛑 Stop LocalAI", func() { - sm.toggleLocalAI() - }) + })} } else { - // LocalAI is installed but not running - show start option - actionItem = fyne.NewMenuItem("▶️ Start LocalAI", func() { - sm.toggleLocalAI() - }) + roleItems = roleMenuItems(role, sm.launcher.IsRunning(), urlErr == nil, sm.toggleLocalAI, sm.openWebUI) } menuItems := []*fyne.MenuItem{} @@ -155,18 +180,7 @@ func (sm *SystrayManager) recreateMenu() { } // Core actions - menuItems = append(menuItems, - actionItem, - ) - - // Only show WebUI option if LocalAI is installed - if sm.launcher.GetReleaseManager().IsLocalAIInstalled() && sm.launcher.IsRunning() { - menuItems = append(menuItems, - fyne.NewMenuItem("Open WebUI", func() { - sm.openWebUI() - }), - ) - } + menuItems = append(menuItems, roleItems...) menuItems = append(menuItems, fyne.NewMenuItemSeparator(), diff --git a/cmd/launcher/internal/systray_menu_test.go b/cmd/launcher/internal/systray_menu_test.go new file mode 100644 index 000000000000..000194ad6c39 --- /dev/null +++ b/cmd/launcher/internal/systray_menu_test.go @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: MIT +package launcher + +import ( + "fyne.io/fyne/v2" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Tray actions by mode", func() { + labels := func(items []*fyne.MenuItem) []string { + var out []string + for _, i := range items { + if !i.IsSeparator { + out = append(out, i.Label) + } + } + return out + } + It("names the worker actions and opens the deployment UI, not a local server UI", func() { + got := labels(roleMenuItems(roleWorker, false, true, func() {}, func() {})) + Expect(got).To(ConsistOf("Start worker", "Open deployment UI")) + got = labels(roleMenuItems(roleWorker, true, true, func() {}, func() {})) + Expect(got).To(ConsistOf("Stop worker", "Open deployment UI")) + }) + It("hides the deployment UI action when no valid frontend URL is saved", func() { + Expect(labels(roleMenuItems(roleWorker, false, false, func() {}, func() {}))).To(ConsistOf("Start worker")) + }) + It("keeps the standalone actions unchanged", func() { + Expect(labels(roleMenuItems(roleStandalone, false, true, func() {}, func() {}))).To(ConsistOf("▶️ Start LocalAI")) + Expect(labels(roleMenuItems(roleStandalone, true, true, func() {}, func() {}))).To(ConsistOf("🛑 Stop LocalAI", "Open WebUI")) + }) +}) diff --git a/cmd/launcher/internal/ui.go b/cmd/launcher/internal/ui.go index bf4f73a9a92c..9ec0919e00da 100644 --- a/cmd/launcher/internal/ui.go +++ b/cmd/launcher/internal/ui.go @@ -1,6 +1,7 @@ package launcher import ( + "errors" "fmt" "log" "net/url" @@ -37,6 +38,30 @@ type LauncherUI struct { autoStartCheck *widget.Check startOnBootCheck *widget.Check + // Mode selection and the worker profile. Both profiles keep their drafts in + // these widgets when the owner switches. + modeRadio *widget.RadioGroup + localBox *fyne.Container + workerBox *fyne.Container + frontendEntry *widget.Entry + tokenEntry *widget.Entry + clearTokenCheck *widget.Check + advertisedEntry *widget.Entry + nodeNameEntry *widget.Entry + grpcMaxEntry *widget.Entry + natsURLEntry *widget.Entry + tlsCAEntry *widget.Entry + tlsCertEntry *widget.Entry + tlsKeyEntry *widget.Entry + jwtEntry *widget.Entry + clearJWTCheck *widget.Check + seedEntry *widget.Entry + clearSeedCheck *widget.Check + workerEnvEntry *widget.Entry + plaintextCheck *widget.Check + previewLabel *widget.Label + joinExplanation *widget.Label + // Environment Variables envVarsData []EnvVar newEnvKeyEntry *widget.Entry @@ -66,7 +91,30 @@ func NewLauncherUI() *LauncherUI { statusLabel := widget.NewLabel("Initializing...") statusLabel.Truncation = fyne.TextTruncateEllipsis + maskedEntry := func() *widget.Entry { + e := widget.NewPasswordEntry() + return e + } return &LauncherUI{ + modeRadio: widget.NewRadioGroup([]string{modeLocalLabel, modeJoinLabel}, nil), + frontendEntry: widget.NewEntry(), + tokenEntry: maskedEntry(), + clearTokenCheck: widget.NewCheck("Clear the saved registration token", nil), + advertisedEntry: widget.NewEntry(), + nodeNameEntry: widget.NewEntry(), + grpcMaxEntry: widget.NewEntry(), + natsURLEntry: widget.NewEntry(), + tlsCAEntry: widget.NewEntry(), + tlsCertEntry: widget.NewEntry(), + tlsKeyEntry: widget.NewEntry(), + jwtEntry: maskedEntry(), + clearJWTCheck: widget.NewCheck("Clear the saved NATS JWT", nil), + seedEntry: maskedEntry(), + clearSeedCheck: widget.NewCheck("Clear the saved NATS user seed", nil), + workerEnvEntry: widget.NewMultiLineEntry(), + plaintextCheck: widget.NewCheck("I accept plaintext http:// or nats:// inside a protected network (this session only)", nil), + previewLabel: widget.NewLabel(""), + joinExplanation: widget.NewLabel("Joining contributes this machine's inference capacity to an existing deployment. It does not create a deployment."), statusLabel: statusLabel, versionLabel: widget.NewLabel("Version: Unknown"), startStopButton: widget.NewButton("Start LocalAI", nil), @@ -76,7 +124,7 @@ func NewLauncherUI() *LauncherUI { backendsPathEntry: widget.NewEntry(), addressEntry: widget.NewEntry(), logLevelSelect: widget.NewSelect([]string{"error", "warn", "info", "debug", "trace"}, nil), - autoStartCheck: widget.NewCheck("Start LocalAI when the launcher opens", nil), + autoStartCheck: widget.NewCheck("Start selected mode when launcher opens", nil), startOnBootCheck: widget.NewCheck("Start LocalAI on system boot", nil), logText: widget.NewMultiLineEntry(), progressBar: widget.NewProgressBar(), @@ -111,11 +159,14 @@ func (ui *LauncherUI) createConfigTab() *fyne.Container { ui.backendsPathEntry, )) - // Server configuration - serverCard := widget.NewCard("Server", "", container.NewVBox( + // Server configuration (local mode) + serverCard := widget.NewCard("Server", "", container.NewGridWithColumns(2, + widget.NewLabel("Address:"), + ui.addressEntry, + )) + // Shared by both modes + sharedCard := widget.NewCard("Startup", "", container.NewVBox( container.NewGridWithColumns(2, - widget.NewLabel("Address:"), - ui.addressEntry, widget.NewLabel("Log Level:"), ui.logLevelSelect, ), @@ -131,14 +182,152 @@ func (ui *LauncherUI) createConfigTab() *fyne.Container { // Environment Variables section envCard := ui.createEnvironmentSection() + ui.localBox = container.NewVBox(serverCard, envCard) + ui.workerBox = ui.createWorkerSection() + ui.workerBox.Hide() + ui.modeRadio.Horizontal = true + ui.modeRadio.Required = true + ui.modeRadio.OnChanged = func(selected string) { ui.showMode(selected) } + ui.modeRadio.SetSelected(modeLocalLabel) + return container.NewVBox( + widget.NewCard("Mode", "Run LocalAI here, or join an existing distributed server. Only one runs at a time.", ui.modeRadio), pathsCard, - serverCard, - envCard, + sharedCard, + ui.localBox, + ui.workerBox, saveButton, ) } +const ( + modeLocalLabel = "Run locally" + modeJoinLabel = "Join a distributed server" +) + +// showMode shows one profile's settings. The other profile's widgets keep +// their drafts. +func (ui *LauncherUI) showMode(selected string) { + if ui.localBox == nil || ui.workerBox == nil { + return + } + if selected == modeJoinLabel { + ui.localBox.Hide() + ui.workerBox.Show() + } else { + ui.workerBox.Hide() + ui.localBox.Show() + } + ui.refreshModeControls() +} + +// selectedRole is the role the controls act on: the saved mode, so a draft +// switch does not relabel a running role's controls. +func (ui *LauncherUI) selectedRole() string { + if ui.launcher != nil && ui.launcher.config != nil && ui.launcher.config.Mode == roleWorker { + return roleWorker + } + return roleStandalone +} + +func (ui *LauncherUI) refreshModeControls() { + ui.UpdateRunningState(ui.launcher != nil && ui.launcher.IsRunning()) +} + +// createWorkerSection builds the "Join a distributed server" settings. +func (ui *LauncherUI) createWorkerSection() *fyne.Container { + ui.frontendEntry.SetPlaceHolder("https://controller.example") + ui.advertisedEntry.SetPlaceHolder("worker.example:50051") + ui.nodeNameEntry.SetPlaceHolder("optional") + ui.grpcMaxEntry.SetPlaceHolder("65535") + ui.natsURLEntry.SetPlaceHolder("tls://nats.example:4222") + ui.workerEnvEntry.SetPlaceHolder("KEY=VALUE, one per line") + ui.workerEnvEntry.SetMinRowsVisible(3) + ui.joinExplanation.Wrapping = fyne.TextWrapWord + ui.previewLabel.Wrapping = fyne.TextWrapWord + ui.plaintextCheck.OnChanged = func(v bool) { + if ui.launcher != nil { + ui.launcher.SetPlaintextAcknowledged(v) + } + } + refresh := func(string) { ui.refreshPreview() } + ui.advertisedEntry.OnChanged = refresh + ui.grpcMaxEntry.OnChanged = refresh + + connection := widget.NewCard("Connection", "", container.NewVBox( + ui.joinExplanation, + container.NewGridWithColumns(2, + widget.NewLabel("Frontend URL:"), ui.frontendEntry, + widget.NewLabel("Registration token:"), ui.tokenEntry, + widget.NewLabel("Advertised address:"), ui.advertisedEntry, + widget.NewLabel("Node name:"), ui.nodeNameEntry, + ), + ui.clearTokenCheck, + ui.previewLabel, + )) + advanced := widget.NewAccordion(widget.NewAccordionItem("Advanced connection settings", container.NewVBox( + container.NewGridWithColumns(2, + widget.NewLabel("NATS URL:"), ui.natsURLEntry, + widget.NewLabel("NATS CA file:"), ui.tlsCAEntry, + widget.NewLabel("NATS client certificate:"), ui.tlsCertEntry, + widget.NewLabel("NATS client key:"), ui.tlsKeyEntry, + widget.NewLabel("Maximum gRPC port:"), ui.grpcMaxEntry, + widget.NewLabel("NATS JWT override:"), ui.jwtEntry, + widget.NewLabel("NATS user seed override:"), ui.seedEntry, + ), + ui.clearJWTCheck, + ui.clearSeedCheck, + ui.plaintextCheck, + widget.NewLabel("Runtime environment overrides:"), + ui.workerEnvEntry, + widget.NewLabel("Secrets are stored in plaintext in a protected file; they are not encrypted. Leave a secret blank to keep the saved value."), + ))) + return container.NewVBox(connection, advanced) +} + +func (ui *LauncherUI) refreshPreview() { + text, err := reachabilityPreview(ui.advertisedEntry.Text, ui.grpcMaxEntry.Text) + if err != nil { + text = err.Error() + } + ui.previewLabel.SetText(text) +} + +// collectConfig builds the candidate configuration from the widgets: both +// profiles, the selected mode, and masked secrets kept unless changed. +func (ui *LauncherUI) collectConfig() (*Config, error) { + config := ui.launcher.GetConfig() + config.ModelsPath = ui.modelsPathEntry.Text + config.BackendsPath = ui.backendsPathEntry.Text + config.Address = ui.addressEntry.Text + config.LogLevel = ui.logLevelSelect.Selected + autoStart := ui.autoStartCheck.Checked + config.AutoStart = &autoStart + config.StartOnBoot = ui.startOnBootCheck.Checked + config.EnvironmentVars = make(map[string]string) + for _, envVar := range ui.envVarsData { + config.EnvironmentVars[envVar.Key] = envVar.Value + } + form := workerForm{ + FrontendURL: ui.frontendEntry.Text, AdvertisedAddress: ui.advertisedEntry.Text, NodeName: ui.nodeNameEntry.Text, + GRPCMaxPort: ui.grpcMaxEntry.Text, NATSURL: ui.natsURLEntry.Text, TLSCA: ui.tlsCAEntry.Text, + TLSCert: ui.tlsCertEntry.Text, TLSKey: ui.tlsKeyEntry.Text, EnvLines: ui.workerEnvEntry.Text, + Token: secretEdit{Replacement: ui.tokenEntry.Text, Clear: ui.clearTokenCheck.Checked}, + JWT: secretEdit{Replacement: ui.jwtEntry.Text, Clear: ui.clearJWTCheck.Checked}, + Seed: secretEdit{Replacement: ui.seedEntry.Text, Clear: ui.clearSeedCheck.Checked}, + } + next, err := form.Apply(config) + if err != nil { + return nil, err + } + if ui.modeRadio.Selected == modeJoinLabel { + next.Mode = roleWorker + } else { + next.Mode = roleStandalone + } + return next, nil +} + // createEnvironmentSection creates the environment variables section for the config tab func (ui *LauncherUI) createEnvironmentSection() *fyne.Container { // Initialize environment variables widgets @@ -206,9 +395,6 @@ func (ui *LauncherUI) addEnvironmentVariable() { key := ui.newEnvKeyEntry.Text value := ui.newEnvValueEntry.Text - log.Printf("addEnvironmentVariable: attempting to add %s=%s", key, value) - log.Printf("addEnvironmentVariable: current ui.envVarsData has %d items: %v", len(ui.envVarsData), ui.envVarsData) - if key == "" { log.Printf("addEnvironmentVariable: key is empty, showing error") dialog.ShowError(fmt.Errorf("environment variable name cannot be empty"), ui.launcher.window) @@ -218,15 +404,12 @@ func (ui *LauncherUI) addEnvironmentVariable() { // Check if key already exists for _, envVar := range ui.envVarsData { if envVar.Key == key { - log.Printf("addEnvironmentVariable: key %s already exists, showing error", key) dialog.ShowError(fmt.Errorf("environment variable '%s' already exists", key), ui.launcher.window) return } } - log.Printf("addEnvironmentVariable: adding new env var %s=%s", key, value) ui.envVarsData = append(ui.envVarsData, EnvVar{Key: key, Value: value}) - log.Printf("addEnvironmentVariable: after adding, ui.envVarsData has %d items: %v", len(ui.envVarsData), ui.envVarsData) fyne.Do(func() { if ui.updateEnvironmentDisplay != nil { @@ -237,9 +420,7 @@ func (ui *LauncherUI) addEnvironmentVariable() { ui.newEnvValueEntry.SetText("") }) - log.Printf("addEnvironmentVariable: calling saveEnvironmentVariables") - // Save to configuration - ui.saveEnvironmentVariables() + // The new variable is a draft until Save, like every other setting. } // removeEnvironmentVariable removes an environment variable by index @@ -251,34 +432,6 @@ func (ui *LauncherUI) removeEnvironmentVariable(index int) { ui.updateEnvironmentDisplay() } }) - ui.saveEnvironmentVariables() - } -} - -// saveEnvironmentVariables saves environment variables to the configuration -func (ui *LauncherUI) saveEnvironmentVariables() { - if ui.launcher == nil { - log.Printf("saveEnvironmentVariables: launcher is nil") - return - } - - config := ui.launcher.GetConfig() - log.Printf("saveEnvironmentVariables: before - Environment vars: %v", config.EnvironmentVars) - - config.EnvironmentVars = make(map[string]string) - for _, envVar := range ui.envVarsData { - config.EnvironmentVars[envVar.Key] = envVar.Value - log.Printf("saveEnvironmentVariables: adding %s=%s", envVar.Key, envVar.Value) - } - - log.Printf("saveEnvironmentVariables: after - Environment vars: %v", config.EnvironmentVars) - log.Printf("saveEnvironmentVariables: calling SetConfig with %d environment variables", len(config.EnvironmentVars)) - - err := ui.launcher.SetConfig(config) - if err != nil { - log.Printf("saveEnvironmentVariables: failed to save config: %v", err) - } else { - log.Printf("saveEnvironmentVariables: config saved successfully") } } @@ -318,14 +471,8 @@ func (ui *LauncherUI) setupBindings() { ui.checkForUpdates() } - // Log level selection - ui.logLevelSelect.OnChanged = func(selected string) { - if ui.launcher != nil { - config := ui.launcher.GetConfig() - config.LogLevel = selected - ui.launcher.SetConfig(config) - } - } + // The log level is a draft until Save; a change may need the running role + // stopped, which only the save flow can ask for. } // startLocalAI starts the LocalAI service @@ -333,7 +480,7 @@ func (ui *LauncherUI) startLocalAI() { fyne.Do(func() { ui.startStopButton.Disable() }) - ui.UpdateStatus("Starting LocalAI...") + ui.UpdateStatus("Starting...") go func() { err := ui.launcher.StartLocalAI() @@ -343,10 +490,7 @@ func (ui *LauncherUI) startLocalAI() { dialog.ShowError(err, ui.launcher.window) }) } else { - fyne.Do(func() { - ui.startStopButton.SetText("Stop LocalAI") - ui.webUIButton.Enable() - }) + ui.UpdateRunningState(true) } fyne.Do(func() { ui.startStopButton.Enable() @@ -359,19 +503,17 @@ func (ui *LauncherUI) stopLocalAI() { fyne.Do(func() { ui.startStopButton.Disable() }) - ui.UpdateStatus("Stopping LocalAI...") + ui.UpdateStatus("Stopping...") go func() { err := ui.launcher.StopLocalAI() if err != nil { + // Stop stays available until shutdown is confirmed. fyne.DoAndWait(func() { dialog.ShowError(err, ui.launcher.window) }) } else { - fyne.Do(func() { - ui.startStopButton.SetText("Start LocalAI") - ui.webUIButton.Disable() - }) + ui.UpdateRunningState(false) } fyne.Do(func() { ui.startStopButton.Enable() @@ -392,38 +534,43 @@ func (ui *LauncherUI) openWebUI() { fyne.CurrentApp().OpenURL(parsedURL) } -// saveConfiguration saves the current configuration +// saveConfiguration validates and persists both profiles and the selected mode. +// A change that affects a running role asks for a confirmed stop first. func (ui *LauncherUI) saveConfiguration() { - log.Printf("saveConfiguration: starting to save configuration") - - config := ui.launcher.GetConfig() - log.Printf("saveConfiguration: current config Environment vars: %v", config.EnvironmentVars) - log.Printf("saveConfiguration: ui.envVarsData has %d items: %v", len(ui.envVarsData), ui.envVarsData) - - config.ModelsPath = ui.modelsPathEntry.Text - config.BackendsPath = ui.backendsPathEntry.Text - config.Address = ui.addressEntry.Text - config.LogLevel = ui.logLevelSelect.Selected - autoStart := ui.autoStartCheck.Checked - config.AutoStart = &autoStart - config.StartOnBoot = ui.startOnBootCheck.Checked - - // Ensure environment variables are included in the configuration - config.EnvironmentVars = make(map[string]string) - for _, envVar := range ui.envVarsData { - config.EnvironmentVars[envVar.Key] = envVar.Value - log.Printf("saveConfiguration: adding env var %s=%s", envVar.Key, envVar.Value) - } - - log.Printf("saveConfiguration: final config Environment vars: %v", config.EnvironmentVars) - - err := ui.launcher.SetConfig(config) + candidate, err := ui.collectConfig() if err != nil { - log.Printf("saveConfiguration: failed to save config: %v", err) dialog.ShowError(err, ui.launcher.window) - } else { - log.Printf("saveConfiguration: config saved successfully") + return + } + err = ui.launcher.ApplySettings(candidate, false) + switch { + case err == nil: dialog.ShowInformation("Configuration", "Configuration saved successfully", ui.launcher.window) + ui.LoadConfiguration() + case errors.Is(err, errStopRequired): + message := widget.NewLabel("This change needs the running role to stop first. Stop and apply waits until the runtime and its backend processes are confirmed gone, saves the settings, and does not start anything. Active inference may be interrupted.") + message.Wrapping = fyne.TextWrapWord + dialog.NewCustomConfirm("Stop and apply?", "Stop and apply", "Cancel", message, func(stop bool) { + if !stop { + return // Cancel leaves the active configuration unchanged + } + ui.UpdateStatus("Stopping...") + go func() { + applyErr := ui.launcher.ApplySettings(candidate, true) + fyne.Do(func() { + if applyErr != nil { + ui.UpdateStatus("Failed: " + applyErr.Error()) + dialog.ShowError(applyErr, ui.launcher.window) + return + } + ui.UpdateStatus("Settings applied. Start the selected mode when you are ready.") + ui.LoadConfiguration() + }) + }() + }, ui.launcher.window).Show() + default: + log.Printf("saveConfiguration: failed to save config") + dialog.ShowError(err, ui.launcher.window) } } @@ -549,7 +696,11 @@ func (ui *LauncherUI) OnLogUpdate(logLine string) { if ui.logText != nil { fyne.Do(func() { currentText := ui.logText.Text - ui.logText.SetText(currentText + logLine) + text := currentText + logLine + if len(text) > maxLogBuffer { + text = text[len(text)-maxLogBuffer:] + } + ui.logText.SetText(text) // Auto-scroll to bottom (simplified) ui.logText.CursorRow = len(ui.logText.Text) @@ -580,17 +731,48 @@ func (ui *LauncherUI) LoadConfiguration() { } config := ui.launcher.GetConfig() - log.Printf("UI LoadConfiguration: loading config - ModelsPath=%s, BackendsPath=%s, Address=%s, LogLevel=%s", - config.ModelsPath, config.BackendsPath, config.Address, config.LogLevel) - log.Printf("UI LoadConfiguration: Environment vars: %v", config.EnvironmentVars) ui.modelsPathEntry.SetText(config.ModelsPath) ui.backendsPathEntry.SetText(config.BackendsPath) ui.addressEntry.SetText(config.Address) ui.logLevelSelect.SetSelected(config.LogLevel) - ui.autoStartCheck.SetChecked(config.AutoStart == nil || *config.AutoStart) + ui.autoStartCheck.SetChecked(autoStartChecked(config)) ui.startOnBootCheck.SetChecked(config.StartOnBoot) + // Worker profile. Secret values are never loaded into widgets. + form := newWorkerForm(config) + ui.frontendEntry.SetText(form.FrontendURL) + ui.advertisedEntry.SetText(form.AdvertisedAddress) + ui.nodeNameEntry.SetText(form.NodeName) + ui.grpcMaxEntry.SetText(form.GRPCMaxPort) + ui.natsURLEntry.SetText(form.NATSURL) + ui.tlsCAEntry.SetText(form.TLSCA) + ui.tlsCertEntry.SetText(form.TLSCert) + ui.tlsKeyEntry.SetText(form.TLSKey) + ui.workerEnvEntry.SetText(form.EnvLines) + for _, m := range []struct { + entry *widget.Entry + clear *widget.Check + saved bool + }{{ui.tokenEntry, ui.clearTokenCheck, form.Token.Saved}, {ui.jwtEntry, ui.clearJWTCheck, form.JWT.Saved}, {ui.seedEntry, ui.clearSeedCheck, form.Seed.Saved}} { + m.entry.SetText("") + m.clear.SetChecked(false) + if m.saved { + m.entry.SetPlaceHolder("Saved. Leave blank to keep, or type a new value.") + m.clear.Show() + } else { + m.entry.SetPlaceHolder("Not set") + m.clear.Hide() + } + } + ui.plaintextCheck.SetChecked(ui.launcher.PlaintextAcknowledged()) + ui.refreshPreview() + if config.Mode == roleWorker { + ui.modeRadio.SetSelected(modeJoinLabel) + } else { + ui.modeRadio.SetSelected(modeLocalLabel) + } + // Load environment variables ui.envVarsData = []EnvVar{} for key, value := range config.EnvironmentVars { @@ -609,17 +791,52 @@ func (ui *LauncherUI) LoadConfiguration() { log.Printf("UI LoadConfiguration: configuration loaded successfully") } -// UpdateRunningState updates UI based on LocalAI running state +// autoStartChecked is the automatic-start checkbox state for a profile. Legacy +// standalone profiles keep their default-enabled behavior; a worker profile +// starts only when the owner chose it. +func autoStartChecked(c *Config) bool { + if c.Mode == roleWorker { + return c.AutoStart != nil && *c.AutoStart + } + return c.AutoStart == nil || *c.AutoStart +} + +// UpdateRunningState updates UI based on the running state, named for the role. +// Stop stays available while a runtime or unresolved ownership of one exists. func (ui *LauncherUI) UpdateRunningState(isRunning bool) { - fyne.Do(func() { - if isRunning { + update := func() { + worker := ui.selectedRole() == roleWorker + switch { + case worker && isRunning: + ui.startStopButton.SetText("Stop worker") + case worker: + ui.startStopButton.SetText("Start worker") + case isRunning: ui.startStopButton.SetText("Stop LocalAI") + default: + ui.startStopButton.SetText("Start LocalAI") + } + if worker { + ui.webUIButton.SetText("Open deployment UI") + if _, err := ui.launcher.DeploymentURL(); err == nil { + ui.webUIButton.Enable() + } else { + ui.webUIButton.Disable() + } + return + } + ui.webUIButton.SetText("Open WebUI") + if isRunning { ui.webUIButton.Enable() } else { - ui.startStopButton.SetText("Start LocalAI") ui.webUIButton.Disable() } - }) + } + if ui.launcher != nil && ui.launcher.app == nil { + update() + return + } + fyne.Do(update) } // WelcomeDontShowAgainChecked reports the initial state of the welcome diff --git a/cmd/launcher/internal/worker_status.go b/cmd/launcher/internal/worker_status.go new file mode 100644 index 000000000000..50a4c03693d1 --- /dev/null +++ b/cmd/launcher/internal/worker_status.go @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: MIT +package launcher + +import ( + "context" + "errors" + "os/exec" + "strings" + "time" + + "github.com/mudler/LocalAI/pkg/workerstatus" +) + +var errWorkerUpgrade = errors.New("worker runtime status contract unavailable; upgrade LocalAI before starting a worker") + +// capabilityOutput drains but never retains unbounded or diagnostic output. +type capabilityOutput struct { + data []byte + overflow bool +} + +func (w *capabilityOutput) Write(p []byte) (int, error) { + n := len(p) + if len(w.data)+n > 256 { + w.overflow = true + return n, nil + } + w.data = append(w.data, p...) + return n, nil +} +func checkWorkerRuntime(ctx context.Context, binary string) error { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, binary, "worker-status-capability") + // No saved or inherited credentials, paths, or role selection reach a probe. + cmd.Env = []string{"PATH=/usr/bin:/bin"} + cmd.WaitDelay = time.Second + var output capabilityOutput + cmd.Stdout = &output + if cmd.Run() != nil || output.overflow || strings.TrimSpace(string(output.data)) != workerstatus.Capability { + return errWorkerUpgrade + } + return nil +} + +// workerStatusReader is deliberately not connected to StartLocalAI yet. The +// lifecycle controller must establish ownership before polling or displaying it. +type workerStatusReader struct { + path string + tracker workerstatus.Tracker +} + +func (r *workerStatusReader) Read(now time.Time) string { + s, err := workerstatus.Read(r.path, r.tracker.Generation) + if err != nil { + return "status_unavailable" + } + return r.tracker.Classify(s, now) +} diff --git a/cmd/launcher/internal/worker_status_test.go b/cmd/launcher/internal/worker_status_test.go new file mode 100644 index 000000000000..9ca0b6ef32d2 --- /dev/null +++ b/cmd/launcher/internal/worker_status_test.go @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: MIT +package launcher + +import ( + "context" + "github.com/alecthomas/kong" + "github.com/mudler/LocalAI/pkg/workerstatus" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "os" + "path/filepath" + "time" +) + +var _ = Describe("Worker status", func() { + It("checks exact bounded CLI capability output, not a version threshold", func() { + dir := GinkgoT().TempDir() + binary := filepath.Join(dir, "runtime") + for _, body := range []string{"echo old-version", "exit 1", "printf '%02000d' 1", "echo localai-worker-status-v2"} { + Expect(os.WriteFile(binary, []byte("#!/bin/sh\n"+body+"\n"), 0700)).To(Succeed()) + Expect(checkWorkerRuntime(context.Background(), binary)).To(MatchError(errWorkerUpgrade)) + } + Expect(os.WriteFile(binary, []byte("#!/bin/sh\n[ \"$1\" = worker-status-capability ] || exit 2\n[ -z \"$LOCALAI_REGISTRATION_TOKEN\" ] || exit 3\necho localai-worker-status-v1\n"), 0700)).To(Succeed()) + Expect(checkWorkerRuntime(context.Background(), binary)).To(Succeed()) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + Expect(checkWorkerRuntime(ctx, binary)).NotTo(Succeed()) + }) + It("parses the actual capability command without worker connection flags", func() { + var cli struct { + Capability workerstatus.CapabilityCommand `cmd:"" name:"worker-status-capability"` + } + parser, err := kong.New(&cli) + Expect(err).NotTo(HaveOccurred()) + ctx, err := parser.Parse([]string{"worker-status-capability"}) + Expect(err).NotTo(HaveOccurred()) + Expect(ctx.Run()).To(Succeed()) + }) + It("reads only current private evidence and never treats liveness as ready", func() { + now := time.Now() + path, generation, err := workerstatus.NewSession(GinkgoT().TempDir()) + Expect(err).NotTo(HaveOccurred()) + reader := workerStatusReader{path: path, tracker: workerstatus.Tracker{ExpectedPID: os.Getpid(), Generation: generation, Started: now}} + Expect(reader.Read(now)).To(Equal("status_unavailable")) + s := workerstatus.Snapshot{PID: os.Getpid(), Schema: 1, Generation: generation, Sequence: 1, Time: now, Registration: "starting", HeartbeatInterval: 10 * time.Second} + Expect(workerstatus.Write(path, s)).To(Succeed()) + Expect(reader.Read(now)).To(Equal("starting")) + Expect(reader.Read(now.Add(10 * time.Second))).To(Equal("status_unavailable")) + s.Sequence++ + s.Generation = "previous" + Expect(workerstatus.Write(path, s)).To(Succeed()) + Expect(reader.Read(now)).To(Equal("status_unavailable")) + }) +}) diff --git a/cmd/local-ai/main.go b/cmd/local-ai/main.go index ff520b6d939c..99cd07d72092 100644 --- a/cmd/local-ai/main.go +++ b/cmd/local-ai/main.go @@ -10,12 +10,22 @@ import ( "github.com/mudler/LocalAI/core/cli" "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/internal" + "github.com/mudler/LocalAI/pkg/workerstatus" "github.com/mudler/xlog" _ "github.com/mudler/LocalAI/swagger" ) func main() { + // Capability probes must not load credentials or user .env files. The same + // command is registered with Kong for help and normal CLI discovery. + if len(os.Args) == 2 && os.Args[1] == "worker-status-capability" { + if (&workerstatus.CapabilityCommand{}).Run() != nil { + os.Exit(1) + } + return + } + var err error // Initialize xlog at a level of INFO, we will set the desired level after we parse the CLI options diff --git a/contrib/launcher-worker/README.md b/contrib/launcher-worker/README.md new file mode 100644 index 000000000000..f1ce28eb460d --- /dev/null +++ b/contrib/launcher-worker/README.md @@ -0,0 +1,110 @@ +# Launcher worker baseline verification + +LW01 supplies verification infrastructure only, not worker-mode product features. +On a Linux amd64 host, invoke Bash explicitly: + +```sh +export LAUNCHER_VERIFY_WORKDIR="$PWD/assets/work/launcher-worker-validation" +bash contrib/launcher-worker/runner_test.sh +bash contrib/launcher-worker/verify.sh baseline +``` + +Host requirements: Bash, Docker, Git, GNU tar/coreutils (including `timeout`), +util-linux `flock`, and Python 3 for contract tests. Nothing is installed on the +host. The optional work directory defaults to the path shown above; an absolute +path outside the checkout is supported. At least **10 GiB free** is required +before starting. This is a minimum preflight, not a guarantee that every future +build fits: allow more space for caches, snapshots and Docker images. + +Runs have unique evidence directories and share a serially locked cache. The +runner snapshots tracked working-tree files, including uncommitted modifications, +and its own infrastructure. Stage new product/test files before running: other +untracked files and ignored generated outputs are intentionally excluded. The +checkout is never mounted writable. Logs, snapshots, toolchains and caches stay +under the work directory. Container-created files may be root-owned. Docker +images and testcontainers volumes use the daemon's normal storage. + +## Execution + +- Ubuntu 22.04 Linux amd64, four CPUs, 12 GiB RAM, 2048 PIDs, one-hour outer + timeout. No host network, privileged container, GPU access or host installs. +- Exact Go version from `go.mod`, verified against official download checksum + metadata, with `GOTOOLCHAIN=local` (no automatic version substitution). +- Native Fyne/OpenGL/X11 libraries and Xvfb; no Fyne `ci` stub driver. +- Repository `make protogen-go` and `make generate` prerequisites. +- Serial uncached `go test -count=1 -timeout=20m -tags debug,p2p` runs for + `cmd/launcher/internal`, `core/cli/workerregistry`, `core/services/worker` + and `core/services/nodes`. All four are attempted after successful setup, + even if an earlier package fails. There are no focus or skip filters. +- Each Ginkgo report must contain passing `It` specs, with no skipped, pending + or failed specs. Missing/empty/malformed reports fail the gate independently + of the test command's status. + +The nodes suite uses real PostgreSQL testcontainers. It requires access to the +local Docker socket (`/var/run/docker.sock`, overridden with +`LAUNCHER_WORKER_DOCKER_SOCKET`). **Socket access grants daemon-level authority**: +this is build isolation, not a security sandbox for untrusted source. Never use a +production daemon. Testcontainers manages its temporary database containers and +reaper; those sibling containers are outside the runner's resource cgroup. +`host.docker.internal:host-gateway` provides access to their mapped ports without +host networking. Network access is required for apt, Go, modules and test images. + +The scratch `/tmp` bind is explicitly mode **1777**, independent of caller umask. +Its ancestors in the container are `/`, not restrictive host directory ancestors. +The contract test confirms that `_apt` can create `apt.conf.*` through the direct +bind even beneath a mode-0700 host ancestor. It also tests free-space boundaries, +unsupported modes, report rejection (including under `python -O`), command exit +propagation through `tee`, and failed evidence writes. + +## Evidence and boundaries + +The runner prints its evidence directory. `logs/verify.exit`, `docker.exit`, +`container.exit`, per-step `.exit` files, package `.log` files, Ginkgo `.json` +reports and `environment.txt` distinguish setup and test failures. `source.txt` +records source HEAD, diff summary, Docker version and starting free space; the +actual tested snapshot is retained. Go may buffer package output until a package +finishes; an empty in-progress log is not evidence of a skipped run or a pass. +Nonzero statuses survive log piping, and failed log writes fail verification. +If wrapping the invocation in another pipeline, use `bash -o pipefail -c '...'`. + +`config` runs the actual “Secure launcher configuration” specs followed by the +entire affected launcher suite. It uses the same isolated Ubuntu/Xvfb setup, +without mounting the Docker socket or rerunning worker/nodes suites: + +```sh +bash contrib/launcher-worker/verify.sh config +``` + +The focused report requires every selected configuration spec to pass and only +allows skipped specs outside that exact group. The full launcher report still +rejects all skips. Baseline remains unchanged. Other modes fail with exit 2. Synthetic contract +reports are tests of infrastructure, not substitutes for real baseline evidence. +This does not certify distributed inference, lifecycle changes, race coverage, +AppImage packaging, interactive GUI/tray behavior or native macOS execution. + +### Launch builder gate (LW03) + +Run `bash contrib/launcher-worker/verify.sh launch`. Like `config`, this runs only +`cmd/launcher/internal`: the focused `Secure launcher launch` specs followed by +the entire launcher suite. Focused reports must account for every excluded spec; +the full report rejects skips, pending specs, empty or missing reports. The launch +tests parse the real worker Config and global CLI tags without starting a worker. +Baseline package selection and the `config` gate are unchanged. Worker startup +remains disabled pending later runtime/status/ownership integration. + +## LW04 status gate + +`bash contrib/launcher-worker/verify.sh status` runs focused launcher status +specs plus the full launcher, workerregistry, worker, messaging and workerstatus suites, +then repeats all five with the race detector. The unchanged slow nodes suite is +not rerun. Strict JSON evidence rejects missing, empty, failed or skipped specs. +The `launcherstatus` test tag enables a bounded real JWT-authenticated NATS +fixture with HTTP registration/approval and heartbeat responses. The container +builds pinned `nats-server` v2.10.26 via Go's module checksum verification; nothing +is installed on the host. It tests subscription denial, disconnect/reconnect, +and rejected heartbeats. This protocol fixture is not the LW08 full frontend, +verified TLS, model-loading and inference acceptance run. + +The status gate also builds the genuine React distribution (`npm ci` and +`npm run build` in Node 22.16.0 Docker), builds `cmd/local-ai`, and checks its +actual capability output. No placeholder embedded frontend is used. diff --git a/contrib/launcher-worker/check-report.py b/contrib/launcher-worker/check-report.py new file mode 100644 index 000000000000..1d996bfec29f --- /dev/null +++ b/contrib/launcher-worker/check-report.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""Require complete passing Ginkgo evidence, even under python -O.""" +import json +import sys + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +try: + require(len(sys.argv) in (2, 3), "invalid arguments") + focused = len(sys.argv) == 3 + require(not focused or sys.argv[2] in ("--config-focus", "--launch-focus", "--status-focus"), "unknown report mode") + with open(sys.argv[1], encoding="utf-8") as stream: + suites = json.load(stream) + require(isinstance(suites, list) and suites, "no suites") + count = 0 + for suite in suites: + require(suite["SuiteSucceeded"] is True, "suite failed") + stats = suite["PreRunStats"] + total = stats["TotalSpecs"] + scheduled = stats["SpecsThatWillRun"] + require(type(total) is int and total > 0, "invalid TotalSpecs") + require(type(scheduled) is int and scheduled > 0, "invalid SpecsThatWillRun") + require(focused or total == scheduled, "not all specs scheduled") + specs = suite["SpecReports"] + require(isinstance(specs, list) and specs, "no specs") + if focused: + require(sum(s.get("LeafNodeType") == "It" for s in specs) == total, + "focused report does not account for all specs") + selected = [s for s in specs if ({"--launch-focus": "Secure launcher launch", "--config-focus": "Secure launcher configuration", "--status-focus": "Worker status"}[sys.argv[2]]) in s.get("ContainerHierarchyTexts", [])] + excluded = [s for s in specs if s not in selected] + require(selected, "no configuration specs") + require(all(s["State"] == "skipped" for s in excluded), "unexpected specs outside focus") + specs = selected + require(all(s["State"] == "passed" for s in specs), "non-passing/skipped spec") + executed = sum(s.get("LeafNodeType") == "It" for s in specs) + require(executed == scheduled, "It report count does not match scheduled specs") + count += executed + print(f"Verified {len(suites)} suite(s), {count} passing specs; no skips") +except (OSError, ValueError, KeyError, TypeError, AttributeError, IndexError) as error: + print(f"Invalid baseline report: {error}", file=sys.stderr) + sys.exit(1) diff --git a/contrib/launcher-worker/check_report_test.py b/contrib/launcher-worker/check_report_test.py new file mode 100644 index 000000000000..cc3e30a0b7af --- /dev/null +++ b/contrib/launcher-worker/check_report_test.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""Exercise report completeness with normal and optimized Python.""" +import copy +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + + +PARSER = Path(__file__).with_name("check-report.py") + + +def suite(count=1): + return { + "SuiteSucceeded": True, + "PreRunStats": {"TotalSpecs": count, "SpecsThatWillRun": count}, + "SpecReports": [{"State": "passed", "LeafNodeType": "It"}] * count, + } + + +class ReportTests(unittest.TestCase): + def check_report(self, report, status, focused=False): + with tempfile.TemporaryDirectory(dir=os.environ["LAUNCHER_VERIFY_WORKDIR"]) as tmp: + path = Path(tmp) / "report.json" + path.write_text(json.dumps(report), encoding="utf-8") + for flags in ([], ["-O"]): + with self.subTest(flags=flags): + result = subprocess.run( + [sys.executable, *flags, str(PARSER), str(path), *(["--status-focus" if focused == "status" else "--launch-focus" if focused == "launch" else "--config-focus"] if focused else [])], + capture_output=True, text=True, check=False, + ) + self.assertEqual(result.returncode, status, result.stdout + result.stderr) + if status == 0: + count = sum(s["PreRunStats"]["SpecsThatWillRun"] for s in report) + self.assertIn(f"{count} passing specs; no skips", result.stdout) + + def test_config_focus(self): + report = suite(2) + report["PreRunStats"]["SpecsThatWillRun"] = 1 + report["SpecReports"] = [ + {"State": "passed", "LeafNodeType": "It", "ContainerHierarchyTexts": ["Secure launcher configuration"]}, + {"State": "skipped", "LeafNodeType": "It", "ContainerHierarchyTexts": ["Launcher"]}, + ] + self.check_report([report], 0, focused=True) + for state in ("failed", "pending", "skipped"): + broken = copy.deepcopy(report) + broken["SpecReports"][0]["State"] = state + self.check_report([broken], 1, focused=True) + broken = copy.deepcopy(report) + broken["SpecReports"].pop() + self.check_report([broken], 1, focused=True) + broken = copy.deepcopy(report) + broken["SpecReports"][0]["ContainerHierarchyTexts"] = ["Other"] + self.check_report([broken], 1, focused=True) + + def test_launch_focus(self): + report = suite(2) + report["PreRunStats"]["SpecsThatWillRun"] = 1 + report["SpecReports"] = [ + {"State": "passed", "LeafNodeType": "It", "ContainerHierarchyTexts": ["Secure launcher launch"]}, + {"State": "skipped", "LeafNodeType": "It", "ContainerHierarchyTexts": ["Launcher"]}, + ] + self.check_report([report], 0, focused="launch") + self.check_report([report], 1) + report["SpecReports"][0]["State"] = "skipped" + self.check_report([report], 1, focused="launch") + + def test_status_focus(self): + report = suite(2) + report["PreRunStats"]["SpecsThatWillRun"] = 1 + report["SpecReports"] = [ + {"State": "passed", "LeafNodeType": "It", "ContainerHierarchyTexts": ["Worker status"]}, + {"State": "skipped", "LeafNodeType": "It", "ContainerHierarchyTexts": ["Launcher"]}, + ] + self.check_report([report], 0, focused="status") + self.check_report([report], 1) + report["SpecReports"][0]["State"] = "skipped" + self.check_report([report], 1, focused="status") + + def test_truncated_launcher(self): + report = suite(41) + report["SpecReports"] = report["SpecReports"][:1] + self.check_report([report], 1) + + def test_missing_or_malformed_stats(self): + for stats in (None, [], "stats", 1, True, {}): + with self.subTest(stats=stats): + report = suite() + report["PreRunStats"] = stats + self.check_report([report], 1) + report = suite() + del report["PreRunStats"] + self.check_report([report], 1) + + def test_missing_or_malformed_counts(self): + for field in ("TotalSpecs", "SpecsThatWillRun"): + for value in (None, True, False, "1", 1.0, [], {}, -1, 0): + with self.subTest(field=field, value=value): + report = suite() + report["PreRunStats"][field] = value + self.check_report([report], 1) + report = suite() + del report["PreRunStats"][field] + self.check_report([report], 1) + + def test_inconsistent_counts(self): + for total, scheduled, actual in ((2, 1, 1), (1, 2, 1), (1, 1, 2), (2, 2, 1)): + with self.subTest(total=total, scheduled=scheduled, actual=actual): + report = suite(actual) + report["PreRunStats"] = {"TotalSpecs": total, "SpecsThatWillRun": scheduled} + self.check_report([report], 1) + + def test_counts_reconciled_per_suite(self): + first, second = suite(1), suite(2) + first["PreRunStats"], second["PreRunStats"] = second["PreRunStats"], first["PreRunStats"] + self.check_report([first, second], 1) + + def test_complete_suites(self): + reports = [suite(count) for count in (41, 9, 235, 782)] + for report in reports: + self.check_report([report], 0) + self.check_report(reports, 0) + + def test_hooks_are_not_it_specs(self): + report = suite() + report["SpecReports"] += [ + {"State": "passed", "LeafNodeType": hook} + for hook in ("BeforeSuite", "AfterSuite", "SynchronizedBeforeSuite", + "SynchronizedAfterSuite", "ReportBeforeSuite", "ReportAfterSuite") + ] + self.check_report([report], 0) + incomplete = copy.deepcopy(report) + incomplete["PreRunStats"] = {"TotalSpecs": 7, "SpecsThatWillRun": 7} + self.check_report([incomplete], 1) + report["SpecReports"][-1]["State"] = "failed" + self.check_report([report], 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/contrib/launcher-worker/lib.sh b/contrib/launcher-worker/lib.sh new file mode 100644 index 000000000000..72a653d40f5e --- /dev/null +++ b/contrib/launcher-worker/lib.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MIT +run_logged() { + local output=$1; shift + local status=0 + (set -o pipefail; "$@" 2>&1 | tee "$output.log") || status=$? + printf '%s\n' "$status" > "$output.exit" || return 1 + return "$status" +} +prepare_tmp() { + mkdir -p "$1" + chmod 1777 "$1" +} +require_space() { + local available + available=$(df -PB1 "$1" | awk 'NR==2 {print $4}') + if [[ ! $available =~ ^[0-9]+$ ]] || (( available < 10737418240 )); then + echo 'Verification requires at least 10 GiB free in its work directory' >&2 + return 1 + fi +} diff --git a/contrib/launcher-worker/run-container.sh b/contrib/launcher-worker/run-container.sh new file mode 100644 index 000000000000..1390fb5c9590 --- /dev/null +++ b/contrib/launcher-worker/run-container.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MIT +set -euo pipefail +cd /validation/source +source contrib/launcher-worker/lib.sh +trap 's=$?; echo "$s" > /validation/logs/container.exit' EXIT +export DEBIAN_FRONTEND=noninteractive HOME=/validation/home TMPDIR=/tmp +export GOCACHE=/cache/go-build GOPATH=/cache/go GOTOOLCHAIN=local +export GOMAXPROCS=4 GOFLAGS=-p=4 +mkdir -p "$HOME" /cache/downloads /validation/toolchain +run_logged /validation/logs/apt-update apt-get update +run_logged /validation/logs/apt-install apt-get install -y --no-install-recommends \ + build-essential pkg-config libgl1-mesa-dev xorg-dev libxkbcommon-dev \ + xvfb xauth curl git python3 ca-certificates unzip libopus-dev +version=$(awk '$1 == "go" {print $2}' go.mod) +[[ $version =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] +archive="go${version}.linux-amd64.tar.gz" +curl -fL --retry 3 'https://go.dev/dl/?mode=json&include=all' -o /cache/downloads/go-releases.json +if [[ ! -f /cache/downloads/$archive ]]; then + curl -fL --retry 3 "https://go.dev/dl/$archive" -o "/cache/downloads/$archive" +fi +python3 - "$archive" <<'PY' +import hashlib, json, sys +name = sys.argv[1] +with open('/cache/downloads/go-releases.json') as stream: + releases = json.load(stream) +expected = next(f['sha256'] for r in releases for f in r['files'] if f['filename'] == name) +with open('/cache/downloads/' + name, 'rb') as stream: + actual = hashlib.sha256(stream.read()).hexdigest() +if actual != expected: + sys.exit('Go archive checksum mismatch') +print('Verified Go archive SHA256:', actual) +PY +tar -xzf "/cache/downloads/$archive" -C /validation/toolchain +export PATH=/validation/toolchain/go/bin:$GOPATH/bin:$PATH +{ cat /etc/os-release; go version; uname -a; } | tee /validation/logs/environment.txt +run_logged /validation/logs/protogen make protogen-go +run_logged /validation/logs/generate make generate +packages=(cmd/launcher/internal core/cli/workerregistry core/services/worker core/services/nodes) +status=0 +tags=debug,p2p +if [[ ${LAUNCHER_VERIFY_MODE:-baseline} == status ]]; then + tags=debug,p2p,launcherstatus + run_logged /validation/logs/nats-install go install github.com/nats-io/nats-server/v2@v2.10.26 + run_logged /validation/logs/runtime-build go build -tags "$tags" -o /validation/local-ai ./cmd/local-ai + run_logged /validation/logs/runtime-capability timeout 10s /validation/local-ai worker-status-capability + [[ $(cat /validation/logs/runtime-capability.log) == localai-worker-status-v1 ]] || exit 1 + +fi +if [[ ${LAUNCHER_VERIFY_MODE:-baseline} != baseline ]]; then + mode=$LAUNCHER_VERIFY_MODE + focus="Secure launcher configuration" + [[ $mode != launch ]] || focus="Secure launcher launch" + packages=(cmd/launcher/internal) + if [[ $mode == status ]]; then + focus="Worker status" + packages=(cmd/launcher/internal core/cli/workerregistry core/services/worker core/services/messaging pkg/workerstatus core/http/endpoints/localai core/http/routes) + fi + # A focused report intentionally excludes unrelated specs. The full + # launcher report below still requires every spec to run. + report=/validation/logs/launcher-$mode.json + run_logged /validation/logs/launcher-$mode xvfb-run -a go test -count=1 -timeout=20m \ + -tags "$tags" ./cmd/launcher/internal -args -ginkgo.no-color \ + -ginkgo.focus="$focus" -ginkgo.fail-on-empty \ + -ginkgo.fail-on-pending -ginkgo.json-report="$report" || status=1 + run_logged /validation/logs/launcher-$mode-report python3 contrib/launcher-worker/check-report.py "$report" --$mode-focus || status=1 +fi +for package in "${packages[@]}"; do + name=${package//\//-} + report="/validation/logs/$name.json" + if ! run_logged "/validation/logs/$name" xvfb-run -a go test -count=1 -timeout=20m \ + -tags "$tags" "./$package" -args -ginkgo.no-color \ + -ginkgo.fail-on-empty -ginkgo.fail-on-pending -ginkgo.json-report="$report"; then + status=1 + fi + run_logged "/validation/logs/$name-report" python3 contrib/launcher-worker/check-report.py "$report" || status=1 +done +if [[ ${LAUNCHER_VERIFY_MODE:-baseline} == status ]]; then + for package in pkg/workerstatus core/cli/workerregistry core/services/worker core/services/messaging cmd/launcher/internal; do + name=${package//\//-} + report="/validation/logs/race-$name.json" + run_logged "/validation/logs/race-$name" xvfb-run -a go test -race -count=1 -timeout=15m -tags "$tags" "./$package" -args -ginkgo.no-color -ginkgo.fail-on-empty -ginkgo.fail-on-pending -ginkgo.json-report="$report" || status=1 + run_logged "/validation/logs/race-$name-report" python3 contrib/launcher-worker/check-report.py "$report" || status=1 + done +fi +exit "$status" diff --git a/contrib/launcher-worker/runner_test.sh b/contrib/launcher-worker/runner_test.sh new file mode 100644 index 000000000000..51cc884b4439 --- /dev/null +++ b/contrib/launcher-worker/runner_test.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MIT +set -euo pipefail +here=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +: "${LAUNCHER_VERIFY_WORKDIR:?Set LAUNCHER_VERIFY_WORKDIR for test scratch}" +mkdir -p "$LAUNCHER_VERIFY_WORKDIR" +tmp=$(mktemp -d "$LAUNCHER_VERIFY_WORKDIR/contract.XXXXXX") +trap 'rm -rf "$tmp"' EXIT +expect_status() { + local expected=$1; shift + local actual=0 + "$@" > "$tmp/output" 2>&1 || actual=$? + if [[ $actual != "$expected" ]]; then + cat "$tmp/output" + echo "expected $expected, got $actual: $*" >&2 + exit 1 + fi +} +for mode in '' ownership lifecycle ui integration release unknown; do + expect_status 2 bash "$here/verify.sh" "$mode" +done +expect_status 2 bash "$here/verify.sh" baseline extra +expect_status 23 bash -c 'source "$1/lib.sh"; run_logged "$2/failure" bash -c "echo failed; exit 23"' _ "$here" "$tmp" +[[ $(cat "$tmp/failure.exit") == 23 ]] +grep -q failed "$tmp/failure.log" +expect_status 0 bash -c 'source "$1/lib.sh"; run_logged "$2/pass" true' _ "$here" "$tmp" +# A failed evidence sink must turn a successful command into failure. +expect_status 1 bash -c 'source "$1/lib.sh"; run_logged "$2/missing/output" true' _ "$here" "$tmp" +expect_status 1 python3 "$here/check-report.py" "$tmp/missing.json" +for data in 'not json' '{}' 'null' '[null]' '[]' \ + '[{"SuiteSucceeded":true,"PreRunStats":{"TotalSpecs":1,"SpecsThatWillRun":1},"SpecReports":[]}]' \ + '[{"SuiteSucceeded":false,"PreRunStats":{"TotalSpecs":1,"SpecsThatWillRun":1},"SpecReports":[{"State":"passed","LeafNodeType":"It"}]}]' \ + '[{"SuiteSucceeded":true,"PreRunStats":{"TotalSpecs":1,"SpecsThatWillRun":1},"SpecReports":[{"State":"passed","LeafNodeType":"BeforeSuite"}]}]' \ + '[{"SuiteSucceeded":true,"PreRunStats":{"TotalSpecs":1,"SpecsThatWillRun":1},"SpecReports":[{"State":"passed","LeafNodeType":"It"},{"State":"skipped","LeafNodeType":"It"}]}]' \ + '[{"SuiteSucceeded":true,"PreRunStats":{"TotalSpecs":1,"SpecsThatWillRun":1},"SpecReports":[{"State":"pending"}]}]' \ + '[{"SuiteSucceeded":true,"PreRunStats":{"TotalSpecs":1,"SpecsThatWillRun":1},"SpecReports":[{"State":"failed"}]}]'; do + printf '%s\n' "$data" > "$tmp/report.json" + expect_status 1 python3 "$here/check-report.py" "$tmp/report.json" + expect_status 1 python3 -O "$here/check-report.py" "$tmp/report.json" +done +printf '%s\n' '[{"SuiteSucceeded":true,"PreRunStats":{"TotalSpecs":1,"SpecsThatWillRun":1},"SpecReports":[{"State":"passed","LeafNodeType":"It"}]}]' > "$tmp/report.json" +expect_status 0 python3 "$here/check-report.py" "$tmp/report.json" +expect_status 0 python3 -O "$here/check-report.py" "$tmp/report.json" +python3 "$here/check_report_test.py" +source "$here/lib.sh" +# Mock df only for the boundary contract; real baseline uses the actual filesystem. +for bytes in 0 10737418239 invalid; do + expect_status 1 bash -c 'source "$1/lib.sh"; bytes=$2; df() { printf "header\nfs 999 0 %s\n" "$bytes"; }; require_space .' _ "$here" "$bytes" +done +expect_status 0 bash -c 'source "$1/lib.sh"; df() { printf "header\nfs 999 0 10737418240\n"; }; require_space .' _ "$here" +(umask 077; prepare_tmp "$tmp/private/tmp") +[[ $(stat -c %a "$tmp/private") == 700 ]] +[[ $(stat -c %a "$tmp/private/tmp") == 1777 ]] +# Direct /tmp bind uses container ancestors, not restrictive host ancestors. +docker run --rm --platform linux/amd64 --user _apt \ + -v "$tmp/private/tmp:/tmp" ubuntu:22.04 sh -ec \ + 'test -x /; test -w /tmp; f=$(mktemp /tmp/apt.conf.XXXXXX); rm "$f"' +echo 'Runner contract: PASS' diff --git a/contrib/launcher-worker/verify.sh b/contrib/launcher-worker/verify.sh new file mode 100644 index 000000000000..c1a1f4ed145d --- /dev/null +++ b/contrib/launcher-worker/verify.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MIT +set -euo pipefail +if [[ $# != 1 || ( $1 != baseline && $1 != config && $1 != launch && $1 != status ) ]]; then + echo 'Usage: bash contrib/launcher-worker/verify.sh {baseline|config|launch|status}' >&2 + exit 2 +fi +mode=$1 +here=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +repo=$(cd "$here/../.." && pwd) +source "$here/lib.sh" +work=${LAUNCHER_VERIFY_WORKDIR:-$repo/assets/work/launcher-worker-validation} +mkdir -p "$work" +work=$(cd "$work" && pwd) +for cmd in docker git tar timeout flock; do command -v "$cmd" >/dev/null; done +exec 9>"$work/runner.lock" +flock -n 9 || { echo 'Verification already running in this work directory' >&2; exit 1; } +require_space "$work" +run=$(mktemp -d "$work/$mode.XXXXXX") +mkdir -p "$run/source" "$run/logs" "$work/cache" +prepare_tmp "$run/tmp" +name=localai-launcher-worker-$(basename "$run" | tr '[:upper:].' '[:lower:]-') +cleanup() { + local status=$? + trap - EXIT + docker rm -f "$name" >> "$run/logs/cleanup.log" 2>&1 || true + printf '%s\n' "$status" > "$run/logs/verify.exit" || status=1 + echo "Verification exit $status; evidence: $run" + exit "$status" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM +(cd "$repo"; git ls-files -z | tar --null -T - -cf -) | tar -xf - -C "$run/source" +mkdir -p "$run/source/contrib/launcher-worker" +cp "$here/"* "$run/source/contrib/launcher-worker/" +{ git -C "$repo" rev-parse HEAD; git -C "$repo" diff --stat HEAD; docker version; df -h "$work"; } > "$run/logs/source.txt" +docker_options=() +if [[ $mode == baseline || $mode == status ]]; then +socket=${LAUNCHER_WORKER_DOCKER_SOCKET:-/var/run/docker.sock} +[[ -S $socket ]] || { echo "Docker socket required for real testcontainers: $socket" >&2; exit 1; } +docker_options=(--add-host=host.docker.internal:host-gateway -e TESTCONTAINERS_HOST_OVERRIDE=host.docker.internal -v "$socket:/var/run/docker.sock") +fi +echo "Evidence: $run" +if [[ $mode == status ]]; then + # The real CLI embeds the React distribution; build its actual prerequisites, + # never substitute a placeholder embed just to make the capability test link. + run_logged "$run/logs/react-build" timeout --signal=TERM --kill-after=30s 900s \ + docker run --rm --name "$name" --platform linux/amd64 --cpus=4 --memory=6g --pids-limit=1024 \ + -v "$run/source:/source" -v "$work/cache/npm:/root/.npm" -w /source/core/http/react-ui \ + node:22.16.0-bookworm bash -ec 'npm ci --no-audit --no-fund && npm run build' +fi + +run_logged "$run/logs/docker" timeout --signal=TERM --kill-after=30s 3600s \ + docker run --rm --name "$name" --platform linux/amd64 \ + --cpus=4 --memory=12g --pids-limit=2048 \ + "${docker_options[@]}" -e LAUNCHER_VERIFY_MODE="$mode" \ + -v "$run:/validation" -v "$work/cache:/cache" -v "$run/tmp:/tmp" \ + ubuntu:22.04 bash /validation/source/contrib/launcher-worker/run-container.sh diff --git a/core/cli/cli.go b/core/cli/cli.go index 2b17ff96d5d0..f492fdf7c92b 100644 --- a/core/cli/cli.go +++ b/core/cli/cli.go @@ -4,11 +4,13 @@ import ( "github.com/mudler/LocalAI/core/cli/benchmark" cliContext "github.com/mudler/LocalAI/core/cli/context" "github.com/mudler/LocalAI/core/cli/worker" + "github.com/mudler/LocalAI/pkg/workerstatus" ) var CLI struct { - cliContext.Context `embed:""` - Benchmark benchmark.Command `cmd:"" help:"Benchmark configured text models against a running LocalAI server"` + WorkerStatusCapability workerstatus.CapabilityCommand `cmd:"" name:"worker-status-capability" help:"Print the supported private worker status contract and exit"` + cliContext.Context `embed:""` + Benchmark benchmark.Command `cmd:"" help:"Benchmark configured text models against a running LocalAI server"` Run RunCMD `cmd:"" help:"Run LocalAI, this the default command if no other command is specified. Run 'local-ai run --help' for more information" default:"withargs"` Chat ChatCMD `cmd:"" help:"Run the built-in terminal agent against a LocalAI server"` diff --git a/core/cli/workerregistry/client.go b/core/cli/workerregistry/client.go index cf46455c95c0..cf8dea87a88a 100644 --- a/core/cli/workerregistry/client.go +++ b/core/cli/workerregistry/client.go @@ -10,6 +10,7 @@ import ( "encoding/json" "fmt" "net/http" + "net/url" "strings" "sync" "time" @@ -21,11 +22,12 @@ import ( // RegistrationClient talks to the frontend's /api/node/* endpoints. type RegistrationClient struct { - FrontendURL string - RegistrationToken string - HTTPTimeout time.Duration // used for registration calls; defaults to 10s - client *http.Client - clientOnce sync.Once + FrontendURL string + RegistrationToken string + ObserveRegistration func(*RegisterResponse, error) // optional local evidence callback; set before first use + HTTPTimeout time.Duration // used for registration calls; defaults to 10s + client *http.Client + clientOnce sync.Once } // httpTimeout returns the configured timeout or a sensible default. @@ -69,7 +71,12 @@ type RegisterResponse struct { // response (node ID, approval status, and optional API token / NATS creds). // Re-registration is idempotent: the frontend preserves the node row and mints // a fresh NATS JWT each call, so this doubles as the credential-refresh call. -func (c *RegistrationClient) RegisterFull(ctx context.Context, body map[string]any) (*RegisterResponse, error) { +func (c *RegistrationClient) RegisterFull(ctx context.Context, body map[string]any) (resultResponse *RegisterResponse, resultError error) { + defer func() { + if c.ObserveRegistration != nil { + c.ObserveRegistration(resultResponse, resultError) + } + }() jsonBody, _ := json.Marshal(body) url := c.baseURL() + "/api/node/register" @@ -148,6 +155,9 @@ func (c *RegistrationClient) Heartbeat(ctx context.Context, nodeID string, body return err } defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return fmt.Errorf("heartbeat failed with status %d", resp.StatusCode) + } return nil } @@ -284,3 +294,29 @@ func (c *RegistrationClient) GracefulDeregister(nodeID string) { xlog.Info("Deregistered from frontend") } } + +// Approval reads current controller state without any credential or registration +// side effects. Older controllers fail closed; heartbeat success is not approval. +func (c *RegistrationClient) Approval(ctx context.Context, nodeID string) (string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL()+"/api/node/"+url.PathEscape(nodeID)+"/status", nil) + if err != nil { + return "", err + } + c.setAuth(req) + resp, err := c.httpClient().Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("approval query failed with status %d", resp.StatusCode) + } + var result RegisterResponse + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return "", err + } + if result.ID != nodeID { + return "", fmt.Errorf("approval identity mismatch") + } + return result.Status, nil +} diff --git a/core/cli/workerregistry/credentials.go b/core/cli/workerregistry/credentials.go index 24dd6f3c8ed7..39303edde505 100644 --- a/core/cli/workerregistry/credentials.go +++ b/core/cli/workerregistry/credentials.go @@ -175,7 +175,7 @@ func (m *NATSCredentialManager) RefreshLoop(ctx context.Context) error { } res, err := m.register(ctx) - if err == nil && res.NatsJWT != "" && res.NatsUserSeed != "" { + if err == nil && res.Status != statusPending && res.NatsJWT != "" && res.NatsUserSeed != "" { m.store(res) failures = 0 xlog.Info("Refreshed NATS credentials", "node", res.ID) diff --git a/core/cli/workerregistry/status_test.go b/core/cli/workerregistry/status_test.go new file mode 100644 index 000000000000..b5de73e0f9da --- /dev/null +++ b/core/cli/workerregistry/status_test.go @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +package workerregistry + +import ( + "context" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "net/http" + "net/http/httptest" + "sync" +) + +var _ = Describe("Worker status heartbeat", func() { + It("rejects non-2xx acknowledgements without echoing the response", func() { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(403); _, _ = w.Write([]byte("secret")) })) + defer server.Close() + c := RegistrationClient{FrontendURL: server.URL} + err := c.Heartbeat(context.Background(), "node", nil) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).NotTo(ContainSubstring("secret")) + }) +}) + +var _ = Describe("Worker status registration", func() { + It("observes static retry approval and errors rather than discarding status", func() { + var mu sync.Mutex + response := `{"id":"node","status":"pending"}` + code := http.StatusCreated + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + defer mu.Unlock() + w.WriteHeader(code) + _, _ = w.Write([]byte(response)) + })) + defer server.Close() + outcome := "" + failed := false + client := RegistrationClient{FrontendURL: server.URL, ObserveRegistration: func(res *RegisterResponse, err error) { + failed = err != nil + if res != nil { + outcome = res.Status + } + }} + _, _, _, _, err := client.RegisterWithRetry(context.Background(), nil, 1) + Expect(err).NotTo(HaveOccurred()) + Expect(outcome).To(Equal("pending")) + mu.Lock() + response = `{"id":"node","status":"healthy"}` + mu.Unlock() + _, _, _, _, err = client.RegisterWithRetry(context.Background(), nil, 1) + Expect(err).NotTo(HaveOccurred()) + Expect(outcome).To(Equal("healthy")) + mu.Lock() + code = 403 + mu.Unlock() + _, err = client.RegisterFull(context.Background(), nil) + Expect(err).To(HaveOccurred()) + Expect(failed).To(BeTrue()) + }) +}) + +var _ = Describe("Worker status approval query", func() { + It("requires matching identity and a successful read-only authenticated response", func() { + response := `{"id":"other","status":"healthy"}` + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + Expect(r.Method).To(Equal(http.MethodGet)) + Expect(r.Header.Get("Authorization")).To(Equal("Bearer secret")) + _, _ = w.Write([]byte(response)) + })) + defer server.Close() + client := RegistrationClient{FrontendURL: server.URL, RegistrationToken: "secret"} + _, err := client.Approval(context.Background(), "node") + Expect(err).To(HaveOccurred()) + }) +}) diff --git a/core/http/endpoints/localai/nodes.go b/core/http/endpoints/localai/nodes.go index b38cb5a6033f..c06eb3e6cd97 100644 --- a/core/http/endpoints/localai/nodes.go +++ b/core/http/endpoints/localai/nodes.go @@ -75,6 +75,8 @@ func GetNodeEndpoint(registry *nodes.NodeRegistry) echo.HandlerFunc { // RegisterNodeRequest is the request body for registering a new worker node. type RegisterNodeRequest struct { + // RenewNodeID renews credentials without restoring status or mutating registration. + RenewNodeID string `json:"renew_node_id,omitempty"` Name string `json:"name"` NodeType string `json:"node_type,omitempty"` // "backend" (default) or "agent" Address string `json:"address"` @@ -119,6 +121,15 @@ type RegisterNodeRequest struct { // RegisterNodeEndpoint registers a new backend node. // expectedToken is the registration token configured on the frontend (may be empty to disable auth). // autoApprove controls whether new nodes go directly to "healthy" or require admin approval. +// @Tags nodes +// @Summary Register a worker or renew its credentials +// @Description Supply renew_node_id and the configured registration token to renew credentials without changing approval or telemetry. Renewal requires authentication and only healthy nodes receive credentials. +// @Accept json +// @Produce json +// @Param request body RegisterNodeRequest true "Registration or credential renewal" +// @Success 200 {object} map[string]interface{} "Node ID, status, and credentials when available" +// @Failure 400,401,403,404 {object} schema.ErrorResponse +// @Router /api/node/register [post] func RegisterNodeEndpoint(registry *nodes.NodeRegistry, expectedToken string, autoApprove bool, authDB *gorm.DB, hmacSecret string, natsCfg natsauth.Config) echo.HandlerFunc { return func(c echo.Context) error { var req RegisterNodeRequest @@ -138,6 +149,21 @@ func RegisterNodeEndpoint(registry *nodes.NodeRegistry, expectedToken string, au } } + if req.RenewNodeID != "" { + if expectedToken == "" { + return c.JSON(http.StatusForbidden, nodeError(http.StatusForbidden, "authenticated renewal required")) + } + node, err := registry.Get(c.Request().Context(), req.RenewNodeID) + if err != nil { + return c.JSON(http.StatusNotFound, nodeError(http.StatusNotFound, "node not found")) + } + response := map[string]any{"id": node.ID, "status": node.Status} + if node.Status == nodes.StatusHealthy { + attachNatsJWT(response, node, natsCfg) + } + return c.JSON(http.StatusOK, response) + } + // Determine node type nodeType := req.NodeType if nodeType == "" { @@ -1366,3 +1392,25 @@ func proxyHTTPToWorker(ctx context.Context, dialFor nodes.WorkerNetDialerFor, no client := httpclient.NewWithTimeout(15*time.Second, httpclient.WithTransport(t)) return client.Do(req) } + +// NodeApprovalEndpoint observes approval without re-registering, restoring an +// offline node, refreshing heartbeat, or minting credentials. It uses the same +// registration-token boundary as the other worker self-service endpoints. +// @Tags nodes +// @Summary Observe worker approval +// @Description Read approval without changing registration, heartbeat, or credentials. Requires the configured registration token when authentication is enabled. +// @Produce json +// @Param id path string true "Node ID" +// @Param Authorization header string false "Bearer registration token" +// @Success 200 {object} map[string]string "Node ID and current status" +// @Failure 401,404 {object} schema.ErrorResponse +// @Router /api/node/{id}/status [get] +func NodeApprovalEndpoint(registry *nodes.NodeRegistry) echo.HandlerFunc { + return func(c echo.Context) error { + node, err := registry.Get(c.Request().Context(), c.Param("id")) + if err != nil { + return c.JSON(http.StatusNotFound, nodeError(http.StatusNotFound, "node not found")) + } + return c.JSON(http.StatusOK, map[string]string{"id": node.ID, "status": string(node.Status)}) + } +} diff --git a/core/http/endpoints/localai/nodes_test.go b/core/http/endpoints/localai/nodes_test.go index 736f4bde51ea..0f4d07666947 100644 --- a/core/http/endpoints/localai/nodes_test.go +++ b/core/http/endpoints/localai/nodes_test.go @@ -57,6 +57,35 @@ var _ = Describe("Node HTTP handlers", func() { Expect(err).ToNot(HaveOccurred()) }) + It("Worker status observation and renewal preserve offline status and heartbeat", func() { + ctx := context.Background() + node := &nodes.BackendNode{Name: "approval-test", Address: "127.0.0.1:50051"} + Expect(registry.Register(ctx, node, true)).To(Succeed()) + Expect(registry.MarkOffline(ctx, node.ID)).To(Succeed()) + before, err := registry.Get(ctx, node.ID) + Expect(err).NotTo(HaveOccurred()) + e := echo.New() + rec := httptest.NewRecorder() + c := e.NewContext(httptest.NewRequest(http.MethodGet, "/", nil), rec) + c.SetParamNames("id") + c.SetParamValues(node.ID) + Expect(NodeApprovalEndpoint(registry)(c)).To(Succeed()) + Expect(rec.Code).To(Equal(200)) + Expect(rec.Body.String()).To(ContainSubstring("offline")) + body := `{"renew_node_id":"` + node.ID + `","token":"secret"}` + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body)) + req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON) + rec = httptest.NewRecorder() + Expect(RegisterNodeEndpoint(registry, "secret", true, nil, "", natsauth.Config{})(e.NewContext(req, rec))).To(Succeed()) + Expect(rec.Code).To(Equal(200)) + Expect(rec.Body.String()).To(ContainSubstring("offline")) + Expect(rec.Body.String()).NotTo(ContainSubstring("nats_jwt")) + after, err := registry.Get(ctx, node.ID) + Expect(err).NotTo(HaveOccurred()) + Expect(after.Status).To(Equal(before.Status)) + Expect(after.LastHeartbeat).To(Equal(before.LastHeartbeat)) + }) + Describe("RegisterNodeEndpoint", func() { It("binds, persists, and lists CPU telemetry", func() { e := echo.New() diff --git a/core/http/routes/nodes.go b/core/http/routes/nodes.go index 2e55cd10b545..beb9bd56a9da 100644 --- a/core/http/routes/nodes.go +++ b/core/http/routes/nodes.go @@ -50,6 +50,10 @@ func RegisterNodeSelfServiceRoutes(e *echo.Echo, registry *nodes.NodeRegistry, r node.POST("/:id/drain", localai.DrainNodeEndpoint(registry, nil)) node.POST("/:id/resume", localai.ResumeNodeEndpoint(registry)) node.POST("/:id/deregister", localai.DeactivateNodeEndpoint(registry)) + // Approval evidence is never exposed by an unauthenticated deployment. + if registrationToken != "" { + node.GET("/:id/status", localai.NodeApprovalEndpoint(registry)) + } node.GET("/:id/models", localai.GetNodeModelsEndpoint(registry)) node.DELETE("/:id", localai.DeactivateNodeEndpoint(registry)) } diff --git a/core/http/routes/worker_status_test.go b/core/http/routes/worker_status_test.go new file mode 100644 index 000000000000..70ca2e416031 --- /dev/null +++ b/core/http/routes/worker_status_test.go @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: MIT +package routes + +import ( + "github.com/labstack/echo/v4" + "github.com/mudler/LocalAI/core/services/nodes" + "github.com/mudler/LocalAI/pkg/natsauth" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "net/http" + "net/http/httptest" +) + +var _ = Describe("Worker status authentication", func() { + It("rejects missing and invalid tokens before accessing registry", func() { + e := echo.New() + RegisterNodeSelfServiceRoutes(e, &nodes.NodeRegistry{}, "secret", false, nil, "", natsauth.Config{}) + for _, token := range []string{"", "Bearer wrong"} { + req := httptest.NewRequest(http.MethodGet, "/api/node/id/status", nil) + req.Header.Set("Authorization", token) + rec := httptest.NewRecorder() + e.ServeHTTP(rec, req) + Expect(rec.Code).To(Equal(http.StatusUnauthorized)) + } + }) + It("does not register approval route on unauthenticated deployments", func() { + e := echo.New() + RegisterNodeSelfServiceRoutes(e, &nodes.NodeRegistry{}, "", false, nil, "", natsauth.Config{}) + rec := httptest.NewRecorder() + e.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/node/id/status", nil)) + Expect(rec.Code).To(Equal(http.StatusNotFound)) + }) +}) diff --git a/core/services/messaging/client.go b/core/services/messaging/client.go index b8ab08059c16..4c97f015de9d 100644 --- a/core/services/messaging/client.go +++ b/core/services/messaging/client.go @@ -6,6 +6,7 @@ import ( "fmt" "strings" "sync" + "sync/atomic" "time" "github.com/mudler/LocalAI/pkg/sanitize" @@ -20,8 +21,9 @@ const subscribeConfirmTimeout = 5 * time.Second // Client wraps a NATS connection and provides helpers for pub/sub and queue subscriptions. type Client struct { - conn *nats.Conn - mu sync.RWMutex + subscriptionDenied atomic.Bool + conn *nats.Conn + mu sync.RWMutex // reconnectCbs are invoked after the underlying connection is // re-established. nats.go transparently resubscribes existing @@ -47,6 +49,10 @@ func New(url string, opts ...Option) (*Client, error) { nats.RetryOnFailedConnect(true), nats.MaxReconnects(-1), nats.DisconnectErrHandler(func(_ *nats.Conn, err error) { + // A new connection replays the subscriptions under current credentials. + // Permission failures on that connection are latched again by the + // ordered async error callback. Never clear them merely on a flush. + c.subscriptionDenied.Store(false) if err != nil { xlog.Warn("NATS disconnected", "error", err) } @@ -68,6 +74,7 @@ func New(url string, opts ...Option) (*Client, error) { subject = sub.Subject } if errors.Is(err, nats.ErrPermissionViolation) { + c.subscriptionDenied.Store(true) xlog.Error("NATS permission violation — check JWT pub/sub allow lists", "subject", subject, "error", err) return } @@ -76,6 +83,11 @@ func New(url string, opts ...Option) (*Client, error) { } switch { case cfg.jwtProvider != nil: + // Dynamic credentials can recover after expiry during a renewal outage. + // MaxReconnects alone does not prevent nats.go from closing permanently + // after two identical auth errors. Keep retrying with the provider; + // static credentials retain the default terminal auth-error behavior. + natsOpts = append(natsOpts, nats.IgnoreAuthErrorAbort()) // Fetch creds on every (re)connect so a refresh loop can rotate the JWT // before expiry; the server expiring the old JWT triggers a reconnect // that transparently picks up the new one. @@ -219,6 +231,7 @@ func (c *Client) confirmSubscription(subject string, mk func(*nats.Conn) (*nats. if lerr := conn.LastError(); lerr != nil && errors.Is(lerr, nats.ErrPermissionViolation) && strings.Contains(lerr.Error(), `Subscription to "`+subject+`"`) { + c.subscriptionDenied.Store(true) _ = sub.Unsubscribe() return nil, fmt.Errorf("subscription to %s denied by NATS server (check JWT sub allow list): %w", subject, lerr) } @@ -331,3 +344,26 @@ func (c *Client) Close() { c.conn.FlushTimeout(5 * time.Second) } } + +// SubscriptionsReady is stricter than Subscribe's historical buffered success: +// a managed worker needs a completed server round trip and no permission error. +// Reconnect replays subscriptions before responding to this flush. +func (c *Client) SubscriptionsReady() bool { + c.mu.RLock() + defer c.mu.RUnlock() + if c.conn == nil || !c.conn.IsConnected() { + return false + } + if c.conn.FlushTimeout(time.Second) != nil { + return false + } + return !c.subscriptionDenied.Load() && !errors.Is(c.conn.LastError(), nats.ErrPermissionViolation) +} + +// Authenticated requires a connected server that demanded authentication, not +// merely configured credentials sent to an anonymous server. +func (c *Client) Authenticated() bool { + c.mu.RLock() + defer c.mu.RUnlock() + return c.conn != nil && c.conn.IsConnected() && c.conn.AuthRequired() +} diff --git a/core/services/messaging/client_auth_retry_test.go b/core/services/messaging/client_auth_retry_test.go new file mode 100644 index 000000000000..a3e7f063d15f --- /dev/null +++ b/core/services/messaging/client_auth_retry_test.go @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: MIT +package messaging + +import ( + "fmt" + "net" + + "github.com/mudler/LocalAI/pkg/natsauth" + "github.com/nats-io/nkeys" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Dynamic credential auth retry scope", func() { + It("opts out of terminal auth retries only for a non-nil provider", func() { + account, err := nkeys.CreateAccount() + Expect(err).NotTo(HaveOccurred()) + defer account.Wipe() + seed, err := account.Seed() + Expect(err).NotTo(HaveOccurred()) + token, userSeed, err := (natsauth.Config{AccountSeed: string(seed)}).MintWorkerJWT("retry-scope", "backend") + Expect(err).NotTo(HaveOccurred()) + // RetryOnFailedConnect lets us inspect the actual connection options without + // a broker; the live worker fixture separately proves recovery behavior. + listener, err := net.Listen("tcp", "127.0.0.1:0") + Expect(err).NotTo(HaveOccurred()) + url := fmt.Sprintf("nats://%s", listener.Addr()) + Expect(listener.Close()).To(Succeed()) + provider := WithUserJWTProvider(func() (string, string) { return token, userSeed }) + for _, tc := range []struct { + opts []Option + retry bool + }{ + {nil, false}, + {[]Option{WithUserJWT(token, userSeed)}, false}, + {[]Option{WithUserJWTProvider(nil)}, false}, + {[]Option{provider}, true}, + {[]Option{WithUserJWT(token, userSeed), provider}, true}, + } { + client, err := New(url, tc.opts...) + Expect(err).NotTo(HaveOccurred()) + Expect(client.conn.Opts.IgnoreAuthErrorAbort).To(Equal(tc.retry)) + client.Close() + } + }) +}) diff --git a/core/services/worker/operations_orphan_test.go b/core/services/worker/operations_orphan_test.go index c3f8aaf8095c..9637e64d4bfa 100644 --- a/core/services/worker/operations_orphan_test.go +++ b/core/services/worker/operations_orphan_test.go @@ -9,6 +9,7 @@ import ( "strings" "time" + "github.com/mudler/LocalAI/pkg/workerledger" process "github.com/mudler/go-processmanager" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -28,18 +29,20 @@ var _ = Describe("Backends a crashed worker left behind", func() { It("are killed, group and grandchild, when the next worker starts", func() { if runtime.GOOS != "linux" { - Skip("the orphan sweep identifies a process by the start time in /proc, which only Linux has; elsewhere it kills nothing") + Skip("the orphan sweep identifies a process by the start time in /proc, which is verified only on Linux here; macOS ps identity is covered by the fallback spec, not by a native run") } proc, grandchild := startGroupProcess() killGroupAtEnd(grandchild) path := filepath.Join(GinkgoT().TempDir(), "processes.json") pid, err := strconv.Atoi(proc.CurrentPID()) Expect(err).ToNot(HaveOccurred()) - newProcessLedger(path).add("model#0", pid) + Expect(newProcessLedger(path).add("model#0", pid)).To(Succeed()) // The worker dies here: its in-memory state is gone, only the file is left. restarted := newProcessLedger(path) - Expect(restarted.sweepStale()).To(Equal(1)) + killed, err := restarted.sweepStale() + Expect(err).ToNot(HaveOccurred()) + Expect(killed).To(Equal(1)) Eventually(func() bool { return grandAlive(grandchild) }, 10*time.Second, 50*time.Millisecond).Should(BeFalse(), "an orphaned grandchild keeps GPU memory and a port") @@ -48,13 +51,13 @@ var _ = Describe("Backends a crashed worker left behind", func() { It("kills a group whose leader already exited but whose grandchild lives on", func() { if runtime.GOOS != "linux" { - Skip("the orphan sweep identifies a process by the start time in /proc, which only Linux has; elsewhere it kills nothing") + Skip("the orphan sweep identifies a process by the start time in /proc, which is verified only on Linux here; macOS ps identity is covered by the fallback spec, not by a native run") } proc, grandchild := startGroupProcess() killGroupAtEnd(grandchild) path := filepath.Join(GinkgoT().TempDir(), "processes.json") pid, _ := strconv.Atoi(proc.CurrentPID()) - newProcessLedger(path).add("model#0", pid) + Expect(newProcessLedger(path).add("model#0", pid)).To(Succeed()) // Only the leader dies. Expect(syscallKill(pid)).To(Succeed()) // A killed leader may stay a zombie until it is reaped. Dead or zombie @@ -62,7 +65,9 @@ var _ = Describe("Backends a crashed worker left behind", func() { Eventually(func() bool { return leaderGone(pid) }, 20*time.Second, 50*time.Millisecond).Should(BeTrue()) Expect(grandAlive(grandchild)).To(BeTrue()) - Expect(newProcessLedger(path).sweepStale()).To(Equal(1)) + killed, err := newProcessLedger(path).sweepStale() + Expect(err).ToNot(HaveOccurred()) + Expect(killed).To(Equal(1)) Eventually(func() bool { return grandAlive(grandchild) }, 10*time.Second, 50*time.Millisecond).Should(BeFalse()) }) @@ -73,11 +78,13 @@ var _ = Describe("Backends a crashed worker left behind", func() { path := filepath.Join(GinkgoT().TempDir(), "processes.json") pid, _ := strconv.Atoi(other.CurrentPID()) ledger := newProcessLedger(path) - ledger.add("model#0", pid) + Expect(ledger.add("model#0", pid)).To(Succeed()) // The recorded start time no longer matches: the pid belongs to someone else. ledger.corruptStartTimeForTest("model#0") - Expect(newProcessLedger(path).sweepStale()).To(BeZero()) + killed, err := newProcessLedger(path).sweepStale() + Expect(err).ToNot(HaveOccurred()) + Expect(killed).To(BeZero()) Expect(pidAlive(other.CurrentPID())).To(BeTrue()) }) @@ -88,11 +95,14 @@ var _ = Describe("Backends a crashed worker left behind", func() { path := filepath.Join(GinkgoT().TempDir(), "processes.json") pid, _ := strconv.Atoi(other.CurrentPID()) ledger := newProcessLedger(path) - ledger.add("model#0", pid) - ledger.remove("model#0") + Expect(ledger.add("model#0", pid)).To(Succeed()) + Expect(other.Stop()).To(Succeed()) + Eventually(func() bool { return pidAlive(other.CurrentPID()) }, 10*time.Second, 50*time.Millisecond).Should(BeFalse()) + Expect(ledger.release("model#0")).To(Succeed()) - Expect(newProcessLedger(path).sweepStale()).To(BeZero()) - Expect(pidAlive(other.CurrentPID())).To(BeTrue()) + killed, err := newProcessLedger(path).sweepStale() + Expect(err).ToNot(HaveOccurred()) + Expect(killed).To(BeZero()) }) It("kills nothing where start times cannot be read, as on macOS", func() { @@ -102,14 +112,16 @@ var _ = Describe("Backends a crashed worker left behind", func() { path := filepath.Join(GinkgoT().TempDir(), "processes.json") pid, _ := strconv.Atoi(other.CurrentPID()) ledger := newProcessLedger(path) - ledger.add("model#0", pid) // recorded with a start time, as on Linux + Expect(ledger.add("model#0", pid)).To(Succeed()) // recorded with a start time, as on Linux ledger.corruptStartTimeForTest("model#0") - real := readStartTime - readStartTime = func(int) string { return "" } // no /proc - DeferCleanup(func() { readStartTime = real }) + real := workerledger.ReadStartTime + workerledger.ReadStartTime = func(int) string { return "" } // no /proc + DeferCleanup(func() { workerledger.ReadStartTime = real }) - Expect(ledger.sweepStale()).To(BeZero(), "with no way to tell whose it is, nothing is killed") + killed, err := ledger.sweepStale() + Expect(err).To(HaveOccurred(), "ownership of a live group that cannot be identified blocks startup") + Expect(killed).To(BeZero(), "with no way to tell whose it is, nothing is killed") Expect(pidAlive(other.CurrentPID())).To(BeTrue()) }) @@ -120,19 +132,25 @@ var _ = Describe("Backends a crashed worker left behind", func() { path := filepath.Join(GinkgoT().TempDir(), "processes.json") pid, _ := strconv.Atoi(other.CurrentPID()) ledger := newProcessLedger(path) - ledger.add("model#0", pid) + Expect(ledger.add("model#0", pid)).To(Succeed()) ledger.forgetStartTimeForTest("model#0") // as on a system with no /proc - Expect(newProcessLedger(path).sweepStale()).To(BeZero()) + killed, err := newProcessLedger(path).sweepStale() + Expect(err).To(HaveOccurred(), "an unidentifiable live group blocks startup instead of being forgotten") + Expect(killed).To(BeZero()) Expect(pidAlive(other.CurrentPID())).To(BeTrue()) }) It("is a no-op with no ledger file", func() { - Expect(newProcessLedger(filepath.Join(GinkgoT().TempDir(), "missing.json")).sweepStale()).To(BeZero()) + killed, err := newProcessLedger(filepath.Join(GinkgoT().TempDir(), "missing.json")).sweepStale() + Expect(err).ToNot(HaveOccurred()) + Expect(killed).To(BeZero()) var none *processLedger - Expect(none.sweepStale()).To(BeZero()) - none.add("x", 1) // must not panic - none.remove("x") + killed, err = none.sweepStale() + Expect(err).ToNot(HaveOccurred()) + Expect(killed).To(BeZero()) + Expect(none.add("x", 1)).To(Succeed()) // must not panic + Expect(none.release("x")).To(Succeed()) }) }) diff --git a/core/services/worker/process_ledger.go b/core/services/worker/process_ledger.go index db8b898ea57e..fbb799959176 100644 --- a/core/services/worker/process_ledger.go +++ b/core/services/worker/process_ledger.go @@ -1,121 +1,143 @@ package worker import ( - "encoding/json" - "os" - "path/filepath" - "strconv" + "fmt" "strings" "sync" + "github.com/mudler/LocalAI/pkg/workerledger" "github.com/mudler/xlog" ) -// processLedger remembers the backend processes this worker started, in a small -// file, so the next worker can find the ones this one left behind. +// processLedger is the durable record of the backend process groups this +// worker owns, kept in a small file so a later worker, or the launcher that +// manages this one, can find the groups a crashed worker left behind. // // Backends run in their own process groups. When the worker is killed (SIGKILL, // the OOM killer) nothing stops them: they keep their port and their GPU -// memory, and a restarted worker has no record of them. The ledger is that -// record. A restarted worker kills every group it lists before it serves. +// memory. The ledger is the record that survives. // -// An entry is the leader's pid and its start time. The start time is what keeps -// a recycled pid from being mistaken for a backend. A nil ledger does nothing. +// An entry is the group leader's pid, its start time and the launch generation +// that started it. The start time keeps a recycled pid from being mistaken for a +// backend. Ownership is released only once the whole group is gone, not when the +// leader exits: descendants can outlive their leader. A nil ledger does nothing. type processLedger struct { path string + // generation is the launch generation stored with each new record. Empty + // for a worker that no launcher manages. + generation string mu sync.Mutex entries map[string]ledgerEntry } -type ledgerEntry struct { - Key string `json:"key"` - PID int `json:"pid"` - StartTime string `json:"start_time"` -} +type ledgerEntry = workerledger.Entry func newProcessLedger(path string) *processLedger { return &processLedger{path: path, entries: map[string]ledgerEntry{}} } -// add records a started process. A failure is logged and ignored: losing the -// ledger costs the orphan sweep, never a start. -func (l *processLedger) add(key string, pid int) { - if l == nil || pid <= 0 { - return +// add records a started process before the worker relies on it. An error means +// ownership is not durable, and the caller must not run the backend. +func (l *processLedger) add(key string, pid int) error { + if l == nil { + return nil + } + if pid <= 0 { + return fmt.Errorf("recording ownership of %s: no process id", key) + } + start := workerledger.ReadStartTime(pid) + if start == "" && workerledger.StartTimesReadable() { + return fmt.Errorf("recording ownership of %s: cannot read the start time of process %d", key, pid) + } + generation := l.generation + if len(generation) > 128 { + generation = generation[:128] } l.mu.Lock() defer l.mu.Unlock() - l.entries[key] = ledgerEntry{Key: key, PID: pid, StartTime: readStartTime(pid)} - l.flushLocked() + previous, had := l.entries[key] + l.entries[key] = ledgerEntry{Key: key, PID: pid, StartTime: start, Generation: generation} + if err := l.flushLocked(); err != nil { + if had { + l.entries[key] = previous + } else { + delete(l.entries, key) + } + return fmt.Errorf("recording ownership of %s: %w", key, err) + } + return nil } -// remove forgets a process the worker stopped itself. -func (l *processLedger) remove(key string) { +// release gives up ownership of a process the worker stopped. The record is +// dropped only when the group is verified empty: a group that still has members +// is killed first, and stays recorded if it cannot be emptied or identified. +func (l *processLedger) release(key string) error { if l == nil { - return + return nil } l.mu.Lock() defer l.mu.Unlock() - if _, ok := l.entries[key]; !ok { - return + e, ok := l.entries[key] + if !ok { + return nil + } + switch workerledger.Classify(e) { + case workerledger.Uncertain: + return fmt.Errorf("ownership of process group %d (%s) cannot be verified", e.PID, key) + case workerledger.Owned: + if err := workerledger.Reap(e); err != nil { + return fmt.Errorf("releasing %s: %w", key, err) + } } delete(l.entries, key) - l.flushLocked() + return l.flushLocked() } -// sweepStale kills the process groups a previous worker recorded and did not -// stop, and returns how many it killed. It runs once, before this worker starts -// any backend, so every entry in the file belongs to a predecessor. -func (l *processLedger) sweepStale() int { +// sweepStale cleans up the groups a previous worker recorded and did not +// release. It runs once, before this worker starts any backend. It returns how +// many groups it killed and an error when any record could not be resolved +// (unverifiable identity, a group that survived, an unreadable ledger). Such +// records are kept and nothing unverified is signalled; the caller must not +// start backends. +func (l *processLedger) sweepStale() (int, error) { if l == nil { - return 0 + return 0, nil } l.mu.Lock() defer l.mu.Unlock() - data, err := os.ReadFile(l.path) + stale, err := workerledger.ReadFile(l.path) if err != nil { - return 0 - } - // Where the start time of a process cannot be read at all, no entry can be - // told apart from an unrelated process that reused its pid. Nothing is killed. - if !startTimesReadable() { - l.entries = map[string]ledgerEntry{} - l.flushLocked() - return 0 - } - var stale []ledgerEntry - if err := json.Unmarshal(data, &stale); err != nil { - xlog.Warn("Ignoring an unreadable worker process ledger", "path", l.path, "error", err) + return 0, fmt.Errorf("worker process ledger %s: %w", l.path, err) } killed := 0 + remaining := map[string]ledgerEntry{} + var problems []string for _, e := range stale { - if e.PID <= 0 { - continue - } - // Without the leader's start time at the time it was recorded, there is no - // way to tell the process apart from an unrelated one that reused the pid - // (the start time is only readable on Linux). Such an entry is never - // killed. - if e.StartTime == "" { - continue - } - // The leader still exists with another start time: its pid was reused by - // an unrelated process. A group that lost its leader cannot be reused (the - // kernel keeps the pid while the group lives), so a missing leader is - // ours. - if now := readStartTime(e.PID); now != "" && now != e.StartTime { - continue - } - if err := killProcessGroup(e.PID); err == nil { + switch workerledger.Classify(e) { + case workerledger.Gone, workerledger.Foreign: + case workerledger.Uncertain: + remaining[e.Key] = e + problems = append(problems, fmt.Sprintf("%s (group %d: ownership cannot be verified)", e.Key, e.PID)) + case workerledger.Owned: + if err := workerledger.Reap(e); err != nil { + remaining[e.Key] = e + problems = append(problems, fmt.Sprintf("%s (%v)", e.Key, err)) + continue + } killed++ xlog.Warn("Killed a backend process group left behind by a previous worker", "processKey", e.Key, "pid", e.PID) } } - l.entries = map[string]ledgerEntry{} - l.flushLocked() - return killed + l.entries = remaining + if err := l.flushLocked(); err != nil { + problems = append(problems, err.Error()) + } + if len(problems) > 0 { + return killed, fmt.Errorf("backend ownership is unresolved: %s", strings.Join(problems, "; ")) + } + return killed, nil } // corruptStartTimeForTest makes an entry look like a recycled pid. @@ -125,7 +147,7 @@ func (l *processLedger) corruptStartTimeForTest(key string) { e := l.entries[key] e.StartTime = "not-the-start-time" l.entries[key] = e - l.flushLocked() + _ = l.flushLocked() } // forgetStartTimeForTest makes an entry look like one recorded where the start @@ -136,54 +158,17 @@ func (l *processLedger) forgetStartTimeForTest(key string) { e := l.entries[key] e.StartTime = "" l.entries[key] = e - l.flushLocked() + _ = l.flushLocked() } -func (l *processLedger) flushLocked() { +func (l *processLedger) flushLocked() error { list := make([]ledgerEntry, 0, len(l.entries)) for _, e := range l.entries { list = append(list, e) } - data, err := json.Marshal(list) - if err == nil { - err = os.MkdirAll(filepath.Dir(l.path), 0o750) - } - if err == nil { - tmp := l.path + ".tmp" - if err = os.WriteFile(tmp, data, 0o600); err == nil { - err = os.Rename(tmp, l.path) - } - } + err := workerledger.WriteFile(l.path, list) if err != nil { xlog.Warn("Failed to write the worker process ledger", "path", l.path, "error", err) } -} - -// startTimesReadable reports whether this system can tell a process's start -// time (Linux, through /proc). -func startTimesReadable() bool { return readStartTime(os.Getpid()) != "" } - -// procStartTime returns a process's start time as the kernel reports it, or "" -// when it cannot be read: the process is gone, or there is no /proc (the sweep -// is Linux only; elsewhere an entry is never killed). -var readStartTime = procStartTime - -func procStartTime(pid int) string { - data, err := os.ReadFile("/proc/" + strconv.Itoa(pid) + "/stat") - if err != nil { - return "" - } - // The command name is in parentheses and may hold spaces; the fields after - // the last ')' are space separated and start at field 3. Field 22 is the - // start time. - s := string(data) - i := strings.LastIndex(s, ")") - if i < 0 { - return "" - } - fields := strings.Fields(s[i+1:]) - if len(fields) < 20 { - return "" - } - return fields[19] + return err } diff --git a/core/services/worker/process_ledger_ownership_test.go b/core/services/worker/process_ledger_ownership_test.go new file mode 100644 index 000000000000..aed52ccb614e --- /dev/null +++ b/core/services/worker/process_ledger_ownership_test.go @@ -0,0 +1,206 @@ +package worker + +import ( + "os" + "path/filepath" + "runtime" + "strconv" + "time" + + "github.com/mudler/LocalAI/pkg/workerledger" + process "github.com/mudler/go-processmanager" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// Ownership of a backend process group has to stay recorded until every member +// of the group is gone, has to survive a crash, and has to say so loudly when it +// cannot be established. These specs use real process groups. +var _ = Describe("Durable backend process ownership", func() { + sleeper := func() (*process.Process, int) { + p := process.New(process.WithTemporaryStateDir(), process.WithName("/bin/sleep"), process.WithArgs("300")) + Expect(p.Run()).To(Succeed()) + DeferCleanup(func() { _ = p.Stop() }) + pid, err := strconv.Atoi(p.CurrentPID()) + Expect(err).ToNot(HaveOccurred()) + return p, pid + } + + Describe("recording", func() { + It("reports a failure to persist the record", func() { + dir := GinkgoT().TempDir() + blocker := filepath.Join(dir, "blocker") + Expect(os.WriteFile(blocker, nil, 0o600)).To(Succeed()) + // The parent of the ledger path is a file: nothing can be written. + ledger := newProcessLedger(filepath.Join(blocker, "processes.json")) + _, pid := sleeper() + Expect(ledger.add("model#0", pid)).ToNot(Succeed()) + }) + + It("refuses a process it cannot identify", func() { + ledger := newProcessLedger(filepath.Join(GinkgoT().TempDir(), "processes.json")) + Expect(ledger.add("model#0", 0)).ToNot(Succeed()) + }) + + It("stores the launch generation next to the process identity", func() { + path := filepath.Join(GinkgoT().TempDir(), "processes.json") + _, pid := sleeper() + ledger := newProcessLedger(path) + ledger.generation = "launch-7" + Expect(ledger.add("model#0", pid)).To(Succeed()) + data, err := os.ReadFile(path) + Expect(err).ToNot(HaveOccurred()) + Expect(string(data)).To(ContainSubstring(`"generation":"launch-7"`)) + Expect(string(data)).To(ContainSubstring(`"start_time":"`)) + info, err := os.Stat(path) + Expect(err).ToNot(HaveOccurred()) + Expect(info.Mode().Perm()).To(Equal(os.FileMode(0o600))) + }) + }) + + Describe("start time identity", func() { + It("falls back to ps where /proc does not exist, as on macOS", func() { + real := workerledger.ProcRoot + workerledger.ProcRoot = filepath.Join(GinkgoT().TempDir(), "no-proc") + DeferCleanup(func() { workerledger.ProcRoot = real }) + first := workerledger.ReadStartTime(os.Getpid()) + Expect(first).ToNot(BeEmpty()) + Expect(workerledger.ReadStartTime(os.Getpid())).To(Equal(first), "stable for one process") + _, pid := sleeper() + other := workerledger.ReadStartTime(pid) + Expect(other).ToNot(BeEmpty()) + Expect(workerledger.ReadStartTime(1<<30)).To(BeEmpty(), "a pid that does not exist has no start time") + }) + }) + + Describe("release", func() { + It("keeps ownership and kills the group when the leader exited before a descendant", func() { + if runtime.GOOS == "windows" { + Skip("no process groups") + } + proc, grandchild := startGroupProcess() + DeferCleanup(func() { + if grandAlive(grandchild) { + _ = syscallKill(grandchild) + } + }) + path := filepath.Join(GinkgoT().TempDir(), "processes.json") + pid, _ := strconv.Atoi(proc.CurrentPID()) + ledger := newProcessLedger(path) + Expect(ledger.add("model#0", pid)).To(Succeed()) + Expect(syscallKill(pid)).To(Succeed()) + Eventually(func() bool { return leaderGone(pid) }, 20*time.Second, 50*time.Millisecond).Should(BeTrue()) + Expect(grandAlive(grandchild)).To(BeTrue()) + + Expect(ledger.release("model#0")).To(Succeed()) + + Eventually(func() bool { return grandAlive(grandchild) }, 10*time.Second, 50*time.Millisecond).Should(BeFalse(), + "forgetting the record while a descendant lives would orphan it") + data, _ := os.ReadFile(path) + Expect(string(data)).ToNot(ContainSubstring("model#0")) + }) + + It("retains the record when the group cannot be reaped", func() { + proc, grandchild := startGroupProcess() + DeferCleanup(func() { + if grandAlive(grandchild) { + _ = syscallKill(grandchild) + } + _ = proc.Stop() + }) + path := filepath.Join(GinkgoT().TempDir(), "processes.json") + pid, _ := strconv.Atoi(proc.CurrentPID()) + ledger := newProcessLedger(path) + Expect(ledger.add("model#0", pid)).To(Succeed()) + + realKill, realWait := workerledger.KillGroup, workerledger.ReapTimeout + workerledger.KillGroup = func(int) error { return nil } // the signal never lands + workerledger.ReapTimeout = 300 * time.Millisecond + DeferCleanup(func() { workerledger.KillGroup, workerledger.ReapTimeout = realKill, realWait }) + + Expect(ledger.release("model#0")).ToNot(Succeed()) + data, _ := os.ReadFile(path) + Expect(string(data)).To(ContainSubstring("model#0"), "a live group stays owned") + }) + + It("drops a record whose pid now belongs to an unrelated process", func() { + _, pid := sleeper() + path := filepath.Join(GinkgoT().TempDir(), "processes.json") + ledger := newProcessLedger(path) + Expect(ledger.add("model#0", pid)).To(Succeed()) + ledger.corruptStartTimeForTest("model#0") + + Expect(ledger.release("model#0")).To(Succeed()) + Expect(pidAlive(strconv.Itoa(pid))).To(BeTrue(), "an unrelated process is never signalled") + data, _ := os.ReadFile(path) + Expect(string(data)).ToNot(ContainSubstring("model#0")) + }) + }) + + Describe("recovery", func() { + It("blocks, and kills nothing, when ownership of a live group is uncertain", func() { + _, pid := sleeper() + path := filepath.Join(GinkgoT().TempDir(), "processes.json") + ledger := newProcessLedger(path) + Expect(ledger.add("model#0", pid)).To(Succeed()) + ledger.forgetStartTimeForTest("model#0") + + killed, err := newProcessLedger(path).sweepStale() + Expect(err).To(HaveOccurred()) + Expect(killed).To(BeZero()) + Expect(pidAlive(strconv.Itoa(pid))).To(BeTrue()) + data, _ := os.ReadFile(path) + Expect(string(data)).To(ContainSubstring("model#0"), "an unresolved record is kept for the next start") + }) + + It("blocks when the ledger cannot be parsed and leaves it in place", func() { + path := filepath.Join(GinkgoT().TempDir(), "processes.json") + Expect(os.WriteFile(path, []byte("{not json"), 0o600)).To(Succeed()) + _, err := newProcessLedger(path).sweepStale() + Expect(err).To(HaveOccurred()) + data, _ := os.ReadFile(path) + Expect(string(data)).To(Equal("{not json")) + }) + + It("blocks, and keeps the record, when a verified group survives the kill", func() { + proc, grandchild := startGroupProcess() + DeferCleanup(func() { + if grandAlive(grandchild) { + _ = syscallKill(grandchild) + } + _ = proc.Stop() + }) + path := filepath.Join(GinkgoT().TempDir(), "processes.json") + pid, _ := strconv.Atoi(proc.CurrentPID()) + Expect(newProcessLedger(path).add("model#0", pid)).To(Succeed()) + + realKill, realWait := workerledger.KillGroup, workerledger.ReapTimeout + workerledger.KillGroup = func(int) error { return nil } + workerledger.ReapTimeout = 300 * time.Millisecond + DeferCleanup(func() { workerledger.KillGroup, workerledger.ReapTimeout = realKill, realWait }) + + killed, err := newProcessLedger(path).sweepStale() + Expect(err).To(HaveOccurred()) + Expect(killed).To(BeZero()) + data, _ := os.ReadFile(path) + Expect(string(data)).To(ContainSubstring("model#0")) + }) + }) + + Describe("supervisor start gate", func() { + It("refuses to run a backend whose ownership cannot be recorded", func() { + dir := GinkgoT().TempDir() + blocker := filepath.Join(dir, "blocker") + Expect(os.WriteFile(blocker, nil, 0o600)).To(Succeed()) + s := &backendSupervisor{cfg: &Config{}, ledger: newProcessLedger(filepath.Join(blocker, "p.json"))} + _, pid := sleeper() + Expect(s.recordOwnership("model#0", strconv.Itoa(pid))).ToNot(Succeed()) + Expect(s.recordOwnership("model#0", "not-a-pid")).ToNot(Succeed()) + }) + + It("is a no-op without a ledger, as in unit tests", func() { + s := &backendSupervisor{cfg: &Config{}} + Expect(s.recordOwnership("model#0", "")).To(Succeed()) + }) + }) +}) diff --git a/core/services/worker/status_runtime.go b/core/services/worker/status_runtime.go new file mode 100644 index 000000000000..f7893af9c513 --- /dev/null +++ b/core/services/worker/status_runtime.go @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: MIT +package worker + +import ( + "context" + "net" + "sync/atomic" + "time" + + "github.com/mudler/LocalAI/core/cli/workerregistry" + "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/core/services/nodes" + "github.com/mudler/LocalAI/pkg/workerstatus" +) + +// observeApproval is deliberately independent of heartbeat and JWT renewal. +// It never calls RegisterFull: registration restores offline nodes and mints +// credentials, neither of which is an observation of administrator intent. +func observeApproval(ctx context.Context, status *workerstatus.Publisher, client *workerregistry.RegistrationClient, nodeID string, interval time.Duration) { + if status == nil { + return + } + tick := time.NewTicker(interval) + defer tick.Stop() + for { + outcome, err := client.Approval(ctx, nodeID) + status.Update(func(s *workerstatus.Snapshot) { + s.RegistrationTime = time.Now() + s.Registration = "unknown" + if err != nil { + s.Registration = "error" + } else if outcome == "healthy" || outcome == "pending" { + s.Registration = outcome + } + }) + select { + case <-ctx.Done(): + return + case <-tick.C: + } + } +} + +// armWorkerStatus is shared by Run and its bounded production-component tests. +// Backend reachability remains part of readiness even with healthy NATS. +func armWorkerStatus(status *workerstatus.Publisher, readiness *nodes.WorkerReadiness, bus *messaging.Client, backends nodes.BackendAddressLister) { + readiness.Set(nodes.CompositeReadiness(nodes.NATSReadiness(bus), nodes.BackendDataPathReadiness(backends, func(addr string) error { + conn, err := net.DialTimeout("tcp", addr, 2*time.Second) + if err != nil { + return err + } + return conn.Close() + }))) + status.Arm(func() (bool, bool, bool, bool) { + return bus.IsConnected(), bus.Authenticated(), bus.SubscriptionsReady(), readiness.Check() == nil + }) +} + +// workerRegistration shares Run's acquisition and renewal wiring with status tests. +func workerRegistration(cfg *Config, status *workerstatus.Publisher, regClient *workerregistry.RegistrationClient, acquired *atomic.Bool, nodeID *string, registrationBody map[string]any) workerregistry.RegisterFunc { + regClient.ObserveRegistration = func(res *workerregistry.RegisterResponse, err error) { + if status == nil { + return + } + if acquired.Load() { + // Approval and retained credentials remain independent of renewal. + // Only a successful renewal can clear this categorical failure. + status.Update(func(s *workerstatus.Snapshot) { + switch { + case err != nil || res == nil: + s.RenewalFailure = "request_failed" + case res.Status == "pending": + s.RenewalFailure = "pending" + case res.NatsJWT == "" || res.NatsUserSeed == "": + s.RenewalFailure = "missing_credentials" + default: + s.RenewalFailure = "" + } + }) + return + } + if res == nil { + status.Registration("error", false, err) + return + } + credentials := (cfg.NatsJWT != "" && cfg.NatsUserSeed != "") || (res.NatsJWT != "" && res.NatsUserSeed != "") + status.Registration(res.Status, credentials, err) + } + + return func(ctx context.Context) (*workerregistry.RegisterResponse, error) { + if acquired.Load() && status != nil { + return regClient.RegisterFull(ctx, map[string]any{"renew_node_id": *nodeID, "token": cfg.RegistrationToken}) + } + return regClient.RegisterFull(ctx, registrationBody) + } +} diff --git a/core/services/worker/status_test.go b/core/services/worker/status_test.go new file mode 100644 index 000000000000..e4cbb005a7fb --- /dev/null +++ b/core/services/worker/status_test.go @@ -0,0 +1,378 @@ +//go:build launcherstatus + +// SPDX-License-Identifier: MIT +package worker + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "sync/atomic" + "syscall" + "time" + + "github.com/mudler/LocalAI/core/cli/workerregistry" + "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/core/services/nodes" + "github.com/mudler/LocalAI/pkg/natsauth" + "github.com/mudler/LocalAI/pkg/workerstatus" + "github.com/nats-io/jwt/v2" + "github.com/nats-io/nats.go" + "github.com/nats-io/nkeys" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// This bounded production-component fixture executes the approval observer and +// composite readiness used by Run, plus expiring credential renewal through the actual RefreshLoop. It +// does not execute all of Run or the full frontend/model/inference LW08 fixture. +var _ = Describe("Worker status", func() { + DescribeTable("observes approval and renewal with real NATS recovery", func(renewalFailure int) { + dir := GinkgoT().TempDir() + operator, err := nkeys.CreateOperator() + Expect(err).NotTo(HaveOccurred()) + defer operator.Wipe() + op, err := operator.PublicKey() + Expect(err).NotTo(HaveOccurred()) + operatorJWT, err := jwt.NewOperatorClaims(op).Encode(operator) + Expect(err).NotTo(HaveOccurred()) + account, err := nkeys.CreateAccount() + Expect(err).NotTo(HaveOccurred()) + defer account.Wipe() + ap, err := account.PublicKey() + Expect(err).NotTo(HaveOccurred()) + accountJWT, err := jwt.NewAccountClaims(ap).Encode(operator) + Expect(err).NotTo(HaveOccurred()) + seed, err := account.Seed() + Expect(err).NotTo(HaveOccurred()) + auth := natsauth.Config{AccountSeed: string(seed)} + token, userSeed, err := auth.MintWorkerJWT("status-node", "backend") + Expect(err).NotTo(HaveOccurred()) + claims, err := jwt.DecodeUserClaims(token) + Expect(err).NotTo(HaveOccurred()) + claims.Expires = 0 + if renewalFailure != 0 { + claims.Expires = time.Now().Add(12 * time.Second).Unix() + } + token, err = claims.Encode(account) + Expect(err).NotTo(HaveOccurred()) + listener, err := net.Listen("tcp", "127.0.0.1:0") + Expect(err).NotTo(HaveOccurred()) + port := listener.Addr().(*net.TCPAddr).Port + Expect(listener.Close()).To(Succeed()) + config := filepath.Join(dir, "nats.conf") + Expect(os.WriteFile(config, []byte(fmt.Sprintf("listen: 127.0.0.1:%d\noperator: %s\nresolver: MEMORY\nresolver_preload: { %s: %s }\n", port, operatorJWT, ap, accountJWT)), 0600)).To(Succeed()) + binary, err := exec.LookPath("nats-server") + Expect(err).NotTo(HaveOccurred(), "status verifier supplies pinned nats-server") + var server *exec.Cmd + start := func() { + server = exec.Command(binary, "-c", config) + Expect(server.Start()).To(Succeed()) + Eventually(func() bool { + c, e := net.DialTimeout("tcp", fmt.Sprintf("127.0.0.1:%d", port), 100*time.Millisecond) + if e != nil { + return false + } + c.Close() + return true + }, 5*time.Second).Should(BeTrue()) + } + stop := func() { + if server != nil { + Expect(server.Process.Signal(os.Interrupt)).To(Succeed()) + Expect(server.Wait()).To(Succeed()) + server = nil + } + } + start() + DeferCleanup(stop) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + path, generation, err := workerstatus.NewSession(dir) + Expect(err).NotTo(HaveOccurred()) + publisher, err := workerstatus.NewPublisher(ctx, path, generation, cancel) + Expect(err).NotTo(HaveOccurred()) + var approved, issued, rejected atomic.Bool + var renewalCode atomic.Int64 + renewalCode.Store(int64(renewalFailure)) + var renewals atomic.Int64 + renewedToken, renewedSeed, err := auth.MintWorkerJWT("status-node", "backend") + Expect(err).NotTo(HaveOccurred()) + frontend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer registration-secret" { + w.WriteHeader(401) + return + } + if r.URL.Path == "/api/node/register" || r.URL.Path == "/api/node/status-node/status" { + if r.Method == http.MethodPost { + var body map[string]any + if json.NewDecoder(r.Body).Decode(&body) != nil { + w.WriteHeader(400) + return + } + if id, ok := body["renew_node_id"]; ok { + if id != "status-node" || body["token"] != "registration-secret" || len(body) != 2 { + w.WriteHeader(400) + return + } + renewals.Add(1) + code := renewalCode.Load() + if code == 403 || code == 500 { + w.WriteHeader(int(code)) + return + } + res := workerregistry.RegisterResponse{ID: "status-node", Status: "healthy"} + if code == 0 { + res.NatsJWT, res.NatsUserSeed = renewedToken, renewedSeed + } + _ = json.NewEncoder(w).Encode(res) + return + } + } + res := workerregistry.RegisterResponse{ID: "status-node", Status: "pending"} + if approved.Load() { + res.Status = "healthy" + } + if issued.Load() && r.Method == http.MethodPost { + res.NatsJWT = token + res.NatsUserSeed = userSeed + } + _ = json.NewEncoder(w).Encode(res) + return + } + if rejected.Load() { + w.WriteHeader(403) + return + } + w.WriteHeader(204) + })) + defer frontend.Close() + client := &workerregistry.RegistrationClient{FrontendURL: frontend.URL, RegistrationToken: "registration-secret"} + var acquisitionDone atomic.Bool + var nodeID string + register := workerRegistration(&Config{RegistrationToken: "registration-secret"}, publisher, client, &acquisitionDone, &nodeID, map[string]any{"name": "fixture"}) + + tracker := workerstatus.Tracker{ExpectedPID: os.Getpid(), Generation: generation, Started: time.Now()} + classify := func() string { + s, e := workerstatus.Read(path, generation) + if e != nil { + return "unavailable" + } + return tracker.Classify(s, time.Now()) + } + manager := workerregistry.NewNATSCredentialManager(register, true) + acquired := make(chan error, 1) + go func() { _, e := manager.Acquire(ctx); acquired <- e }() + Eventually(classify, 3*time.Second).Should(Equal("pending")) + approved.Store(true) + Eventually(classify, 5*time.Second).Should(Equal("credentials")) + issued.Store(true) + Eventually(acquired, 10*time.Second).Should(Receive(BeNil())) + nodeID = manager.NodeID() + acquisitionDone.Store(true) + bus, err := messaging.New(fmt.Sprintf("nats://127.0.0.1:%d", port), messaging.WithUserJWTProvider(manager.Provider())) + Expect(err).NotTo(HaveOccurred()) + defer bus.Close() + readiness := &nodes.WorkerReadiness{} + Expect(readiness.Check()).To(Succeed()) + Expect(classify()).NotTo(Equal("ready")) + supervisor := &backendSupervisor{nodeID: "status-node"} + Expect(supervisor.registerLifecycleVerbs(newNATSControlServer(bus, "status-node"))).To(Succeed()) + armWorkerStatus(publisher, readiness, bus, supervisor) + Expect(classify()).To(Equal("heartbeat")) + publisher.Heartbeat(client.Heartbeat(ctx, "status-node", nil)) + Expect(classify()).To(Equal("ready")) + // Execute the production observer, independently of RefreshLoop. Static + // overrides use this identical observer; it never touches credentials. + approvalCtx, cancelApproval := context.WithCancel(ctx) + defer cancelApproval() + approvalDone := make(chan struct{}) + go func() { + defer close(approvalDone) + observeApproval(approvalCtx, publisher, client, "status-node", 20*time.Millisecond) + }() + refreshed := make(chan error, 1) + go func() { refreshed <- manager.RefreshLoop(ctx) }() + if renewalFailure != 0 { + Eventually(renewals.Load, 15*time.Second).Should(BeNumerically(">=", 1)) + Eventually(classify).Should(Equal("credentials")) + publisher.Heartbeat(client.Heartbeat(ctx, "status-node", nil)) + Expect(bus.IsConnected()).To(BeTrue()) + Consistently(classify, 100*time.Millisecond).Should(Equal("credentials")) + s, e := workerstatus.Read(path, generation) + Expect(e).NotTo(HaveOccurred()) + Expect(s.Registration).To(Equal("healthy")) + reason := "request_failed" + if renewalFailure == 200 { + reason = "missing_credentials" + } + Expect(s.RenewalFailure).To(Equal(reason)) + Expect(s.Credentials).To(BeTrue(), "retained credentials are independent of renewal failure") + // Leave the same authenticated broker live through expiry and the real + // 30s renewal retry. An independent connection continuously proves it + // remains responsive; reconnect must not need a broker restart. + probe, e := nats.Connect(fmt.Sprintf("nats://127.0.0.1:%d", port), nats.UserJWTAndSeed(renewedToken, renewedSeed), nats.NoReconnect()) + Expect(e).NotTo(HaveOccurred()) + defer probe.Close() + broker := server.Process + var probeFailed atomic.Bool + probeCtx, stopProbe := context.WithCancel(ctx) + probeDone := make(chan struct{}) + defer func() { stopProbe(); <-probeDone }() + go func() { + defer close(probeDone) + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-probeCtx.Done(): + return + case <-ticker.C: + if broker.Signal(syscall.Signal(0)) != nil || probe.FlushTimeout(time.Second) != nil { + probeFailed.Store(true) + } + } + } + }() + Eventually(bus.IsConnected, 15*time.Second).Should(BeFalse()) + Expect(time.Now().Unix()).To(BeNumerically(">=", claims.Expires)) + // JWT timestamps have second precision; wait past the expiry boundary + // and allow several identical reconnect rejections before renewal. + Consistently(bus.IsConnected, 6*time.Second).Should(BeFalse()) + // Prove the broker rejects the original JWT, not just wall-clock expiry. + old, e := nats.Connect(fmt.Sprintf("nats://127.0.0.1:%d", port), nats.UserJWTAndSeed(token, userSeed), nats.NoReconnect()) + if old != nil { + old.Close() + } + Expect(e).To(MatchError(nats.ErrAuthorization)) + Expect(classify()).NotTo(Equal("ready")) + renewalCode.Store(0) + Eventually(func() string { token, _ := manager.Current(); return token }, 45*time.Second).Should(Equal(renewedToken)) + Expect(renewals.Load()).To(BeNumerically(">=", 2)) + Eventually(bus.IsConnected, 10*time.Second).Should(BeTrue()) + Expect(bus.SubscriptionsReady()).To(BeTrue()) + stopProbe() + Eventually(probeDone).Should(BeClosed()) + Expect(probeFailed.Load()).To(BeFalse(), "same broker must remain live throughout renewal") + Expect(server.Process).To(BeIdenticalTo(broker)) + publisher.Heartbeat(client.Heartbeat(ctx, "status-node", nil)) + Eventually(classify).Should(Equal("ready")) + } else { + Eventually(refreshed).Should(Receive(BeNil())) + Expect(renewals.Load()).To(BeZero()) + } + approved.Store(false) + Eventually(classify, 3*time.Second).Should(Equal("pending")) + publisher.Heartbeat(client.Heartbeat(ctx, "status-node", nil)) + Expect(bus.IsConnected()).To(BeTrue()) + Expect(classify()).To(Equal("pending")) + approved.Store(true) + Eventually(classify, 3*time.Second).Should(Equal("ready")) + supervisor.mu.Lock() + supervisor.processes = map[string]*backendProcess{"dead": {addr: "127.0.0.1:1", serving: true}} + supervisor.mu.Unlock() + publisher.Update(nil) + Expect(classify()).To(Equal("readiness")) + supervisor.mu.Lock() + supervisor.processes = nil + supervisor.mu.Unlock() + publisher.Update(nil) + Expect(classify()).To(Equal("ready")) + rejected.Store(true) + publisher.Heartbeat(client.Heartbeat(ctx, "status-node", nil)) + Expect(classify()).To(Equal("heartbeat")) + rejected.Store(false) + publisher.Heartbeat(client.Heartbeat(ctx, "status-node", nil)) + Expect(classify()).To(Equal("ready")) + stop() + Eventually(func() bool { return bus.IsConnected() }, 5*time.Second).Should(BeFalse()) + publisher.Update(nil) + Expect(classify()).To(Equal("nats")) + start() + Eventually(func() bool { return bus.IsConnected() }, 10*time.Second).Should(BeTrue()) + publisher.Update(nil) + publisher.Heartbeat(client.Heartbeat(ctx, "status-node", nil)) + Expect(classify()).To(Equal("ready")) + // A real permission denial must not be promoted by a successful local probe. + _, err = bus.Subscribe("nodes.other-node.backend.install", func([]byte) {}) + Expect(err).To(MatchError(ContainSubstring("denied by NATS server"))) + Expect(errors.Is(err, nats.ErrPermissionViolation)).To(BeTrue()) + Expect(bus.SubscriptionsReady()).To(BeFalse()) + stop() + start() + Eventually(bus.SubscriptionsReady, 10*time.Second).Should(BeTrue()) + data, err := os.ReadFile(path) + Expect(err).NotTo(HaveOccurred()) + Expect(string(data)).NotTo(ContainSubstring(token)) + Expect(string(data)).NotTo(ContainSubstring(userSeed)) + Expect(string(data)).NotTo(ContainSubstring("registration-secret")) + cancelApproval() + Eventually(approvalDone).Should(BeClosed()) + Expect(os.Chmod(path, 0644)).To(Succeed()) + publisher.Update(nil) + Eventually(ctx.Done()).Should(BeClosed()) + if renewalFailure != 0 { + Eventually(refreshed).Should(Receive(BeNil())) + } + _, err = os.Stat(path) + Expect(os.IsNotExist(err)).To(BeTrue()) + }, + Entry("non-expiring credentials and independent approval", 0), + Entry("expiring JWT renewal rejected with 403", 403), + Entry("expiring JWT renewal rejected with 500", 500), + Entry("expiring JWT renewal approved without credentials", 200), + ) +}) + +var _ = Describe("Worker status production approval observer", func() { + It("observes static credential revocation without registration or credential mutation", func() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + path, generation, err := workerstatus.NewSession(GinkgoT().TempDir()) + Expect(err).NotTo(HaveOccurred()) + publisher, err := workerstatus.NewPublisher(ctx, path, generation, cancel) + Expect(err).NotTo(HaveOccurred()) + publisher.Registration("healthy", true, nil) + var revoked atomic.Bool + var writes atomic.Int64 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + writes.Add(1) + w.WriteHeader(405) + return + } + state := "healthy" + if revoked.Load() { + state = "pending" + } + _ = json.NewEncoder(w).Encode(workerregistry.RegisterResponse{ID: "static", Status: state}) + })) + defer server.Close() + client := &workerregistry.RegistrationClient{FrontendURL: server.URL} + done := make(chan struct{}) + go func() { defer close(done); observeApproval(ctx, publisher, client, "static", 10*time.Millisecond) }() + revoked.Store(true) + Eventually(func() string { + s, e := workerstatus.Read(path, generation) + if e != nil { + return "" + } + return s.Registration + }).Should(Equal("pending")) + publisher.Heartbeat(nil) + s, err := workerstatus.Read(path, generation) + Expect(err).NotTo(HaveOccurred()) + Expect(s.Registration).To(Equal("pending")) + Expect(s.Credentials).To(BeTrue()) + Expect(writes.Load()).To(BeZero()) + cancel() + Eventually(done).Should(BeClosed()) + }) +}) diff --git a/core/services/worker/supervisor.go b/core/services/worker/supervisor.go index 09c918d4b25c..8fbacd8dad42 100644 --- a/core/services/worker/supervisor.go +++ b/core/services/worker/supervisor.go @@ -565,8 +565,20 @@ func (s *backendSupervisor) startBackend(backend, backendName, backendPath strin backendDirID: dirInfo, instance: uuid.NewString(), } - if pid, convErr := strconv.Atoi(proc.CurrentPID()); convErr == nil { - s.ledger.add(backend, pid) + // Ownership must be durable before the backend is used: a process nobody + // can find again after a crash is an orphan that holds a port and GPU memory. + if ownErr := s.recordOwnership(backend, proc.CurrentPID()); ownErr != nil { + delete(s.processes, backend) + if pid, convErr := strconv.Atoi(proc.CurrentPID()); convErr == nil && pid > 0 { + _ = killProcessGroup(pid) + } + if stopErr := proc.Stop(); stopErr != nil { + xlog.Warn("Stopping a backend whose ownership could not be recorded", "backend", backend, "error", stopErr) + } + s.cleanupProcessRuntime(proc) + s.releasePortForKey(backend, port) + s.mu.Unlock() + return "", fmt.Errorf("refusing to run backend %s: %w", backend, ownErr) } xlog.Info("Backend process started", "backend", backend, "addr", clientAddr) @@ -676,7 +688,7 @@ func (s *backendSupervisor) markBackendServing(key string, bp *backendProcess) b func (s *backendSupervisor) reapDeadProcess(key string, bp *backendProcess) { xlog.Warn("Backend process died unexpectedly, restarting", "backend", key) delete(s.processes, key) - s.ledger.remove(key) + s.releaseOwnership(key) if bp == nil { return } @@ -698,7 +710,7 @@ func (s *backendSupervisor) releaseBackendStart(key string, bp *backendProcess) return } delete(s.processes, key) - s.ledger.remove(key) + s.releaseOwnership(key) s.cleanupProcessRuntime(bp.proc) if bp.port <= 0 { xlog.Error("Cannot recycle backend port: startup has invalid recorded port", "backend", key, "addr", bp.addr, "port", bp.port) @@ -1079,7 +1091,7 @@ func (s *backendSupervisor) finishBackendStop(key string, bp *backendProcess, st return fmt.Errorf("stopping backend process %s: %w", key, stopErr) } delete(s.processes, key) - s.ledger.remove(key) + s.releaseOwnership(key) s.cleanupProcessRuntime(bp.proc) if bp.port <= 0 { xlog.Error("Cannot recycle backend port: process has invalid recorded port", "backend", key, "addr", bp.addr, "port", bp.port) @@ -1167,3 +1179,25 @@ func (s *backendSupervisor) LoadedBackendAddresses() []string { } return addrs } + +// recordOwnership stores the identity of a just-started backend process group +// in the ledger. It is a no-op without a ledger (unit tests). +func (s *backendSupervisor) recordOwnership(key, pid string) error { + if s.ledger == nil { + return nil + } + n, err := strconv.Atoi(pid) + if err != nil { + return fmt.Errorf("recording ownership of %s: process id %q is not a number", key, pid) + } + return s.ledger.add(key, n) +} + +// releaseOwnership ends ownership of a stopped backend. A group that cannot be +// emptied stays recorded, so the next worker start reports it instead of +// silently forgetting a live process. +func (s *backendSupervisor) releaseOwnership(key string) { + if err := s.ledger.release(key); err != nil { + xlog.Error("Backend process group is still owned and was kept in the ledger", "backend", key, "error", err) + } +} diff --git a/core/services/worker/worker.go b/core/services/worker/worker.go index 79df4e884203..1ff531763311 100644 --- a/core/services/worker/worker.go +++ b/core/services/worker/worker.go @@ -5,10 +5,10 @@ import ( "context" "encoding/json" "fmt" - "net" "os" "os/signal" "path/filepath" + "sync/atomic" "syscall" "time" @@ -23,12 +23,20 @@ import ( "github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/sanitize" "github.com/mudler/LocalAI/pkg/system" + "github.com/mudler/LocalAI/pkg/workerstatus" "github.com/mudler/xlog" ) // Run starts the distributed agent worker: registers with the frontend, // subscribes to NATS lifecycle subjects, and blocks on signals. func Run(ctx *cliContext.Context, cfg *Config) error { + shutdownCtx, shutdownCancel := context.WithCancel(context.Background()) + defer shutdownCancel() + status, statusErr := workerstatus.FromEnvironment(shutdownCtx, shutdownCancel) + if statusErr != nil { + return statusErr + } + defer status.Update(func(s *workerstatus.Snapshot) { s.Failure = "stopping"; s.Armed = false }) xlog.Info("Starting worker", "advertise", cfg.advertiseAddr(), "basePort", cfg.effectiveBasePort()) // Fail fast (before prefetch/registration/NATS) when enforcement is on but no @@ -72,7 +80,7 @@ func Run(ctx *cliContext.Context, cfg *Config) error { // Actually: keep it before registration so a worker that's still warming // the cache isn't yet announced as ready. The fast no-op path on a hot // PVC keeps restarts cheap. - prefetchModels(context.Background(), cfg, systemState, ml, galleries, nil) + prefetchModels(shutdownCtx, cfg, systemState, ml, galleries, nil) // Self-registration with frontend (with retry) regClient := &workerregistry.RegistrationClient{ @@ -82,10 +90,11 @@ func Run(ctx *cliContext.Context, cfg *Config) error { // Context cancelled on shutdown — used by registration waits, heartbeat, and // other background goroutines. - shutdownCtx, shutdownCancel := context.WithCancel(context.Background()) - defer shutdownCancel() + var acquired atomic.Bool + var nodeID string registrationBody := cfg.registrationBody() + register := workerRegistration(cfg, status, regClient, &acquired, &nodeID, registrationBody) natsTLS := messaging.TLSFiles{CA: cfg.NatsTLSCA, Cert: cfg.NatsTLSCert, Key: cfg.NatsTLSKey} // Resolve how to connect to NATS. Static env credentials cannot be re-minted, @@ -94,7 +103,6 @@ func Run(ctx *cliContext.Context, cfg *Config) error { // refreshes them before the minted JWT expires, so the connection survives // expiry via a transparent reconnect. var ( - nodeID string connectNats func() (*messaging.Client, error) ) if cfg.NatsJWT != "" || cfg.NatsUserSeed != "" { @@ -108,9 +116,7 @@ func Run(ctx *cliContext.Context, cfg *Config) error { } } else { credMgr := workerregistry.NewNATSCredentialManager( - func(ctx context.Context) (*workerregistry.RegisterResponse, error) { - return regClient.RegisterFull(ctx, registrationBody) - }, + register, cfg.NatsAuthRequired(), ) res, regErr := credMgr.Acquire(shutdownCtx) @@ -139,12 +145,18 @@ func Run(ctx *cliContext.Context, cfg *Config) error { } } + acquired.Store(true) + go observeApproval(shutdownCtx, status, regClient, nodeID, workerstatus.ApprovalInterval) xlog.Info("Registered with frontend", "nodeID", nodeID, "frontend", cfg.RegisterTo) heartbeatInterval, err := time.ParseDuration(cfg.HeartbeatInterval) if err != nil && cfg.HeartbeatInterval != "" { xlog.Warn("invalid heartbeat interval, using default 10s", "input", cfg.HeartbeatInterval, "error", err) } heartbeatInterval = cmp.Or(heartbeatInterval, 10*time.Second) + if heartbeatInterval <= 0 { + heartbeatInterval = 10 * time.Second + } + status.Update(func(s *workerstatus.Snapshot) { s.HeartbeatInterval = heartbeatInterval }) // Start HTTP file transfer server. (Empty-token enforcement is handled at // the top of Run so the worker fails before registering.) @@ -172,9 +184,7 @@ func Run(ctx *cliContext.Context, cfg *Config) error { StartEphemeralRootsCleanup(shutdownCtx, ephemeralRoots, ephemeralCapacity, 0, 0) // The readiness gate is created here but only armed once NATS is up and the // backend supervisor exists, below, because the gate probes both. - // Until then /readyz reports ready, which is correct: reaching this line - // means the worker has already registered with the frontend, so it is - // mid-startup rather than broken. + // The historical unarmed /readyz success is not launcher readiness evidence. readiness := &nodes.WorkerReadiness{} httpServer, err := nodes.StartFileTransferServerWithCapacity(httpAddr, stagingDir, cfg.ModelsPath, dataDir, cfg.RegistrationToken, config.DefaultMaxUploadSize, readiness, ephemeralCapacity, ml.BackendLogs()) if err != nil { @@ -204,7 +214,10 @@ func Run(ctx *cliContext.Context, cfg *Config) error { continue } body := cfg.heartbeatBody() - if err := regClient.Heartbeat(shutdownCtx, nodeID, body); err != nil { + + err := regClient.Heartbeat(shutdownCtx, nodeID, body) + status.Heartbeat(err) + if err != nil { xlog.Warn("Heartbeat failed", "error", err) } } @@ -239,30 +252,21 @@ func Run(ctx *cliContext.Context, cfg *Config) error { maxPort: cfg.effectiveMaxPort(basePort), } - // Arm the readiness gate now that the worker can actually receive work. - // NATS is already connected at this point, so a worker that is up but cut - // off from the bus reports 503 instead of a meaningless 200 (#10987). - // - // Readiness also covers the data path: a worker whose NATS link is fine - // but whose backend processes have died is up and useless, and the - // scheduler cannot tell the difference from the bus alone. - readiness.Set(nodes.CompositeReadiness( - nodes.NATSReadiness(natsClient), - nodes.BackendDataPathReadiness(supervisor, func(addr string) error { - conn, err := net.DialTimeout("tcp", addr, 2*time.Second) - if err != nil { - return err - } - return conn.Close() - }), - )) - // A previous worker that was killed left its backends running. Kill them // now, before this worker hands out ports or reports an incarnation. supervisor.ledger = newProcessLedger(filepath.Join(dataDir, "worker-processes.json")) - if n := supervisor.ledger.sweepStale(); n > 0 { + supervisor.ledger.generation = status.Generation() + n, sweepErr := supervisor.ledger.sweepStale() + if n > 0 { xlog.Warn("Killed backend process groups left behind by a previous worker", "count", n) } + if sweepErr != nil { + // Starting while an earlier backend may still run would double-book its + // port and GPU memory. Nothing unverified is killed; an operator or the + // launcher resolves the record. + nodes.ShutdownFileTransferServer(httpServer) + return fmt.Errorf("refusing to start: %w", sweepErr) + } // The watchdog stops load operations the controller no longer renews. go supervisor.runOperationWatchdog(shutdownCtx) @@ -281,6 +285,19 @@ func Run(ctx *cliContext.Context, cfg *Config) error { } } + // Arm the readiness gate now that the worker can actually receive work. + // NATS is already connected at this point, so a worker that is up but cut + // off from the bus reports 503 instead of a meaningless 200 (#10987). + // + // Readiness also covers the data path: a worker whose NATS link is fine + // but whose backend processes have died is up and useless, and the + // scheduler cannot tell the difference from the bus alone. + + if shutdownCtx.Err() != nil { + nodes.ShutdownFileTransferServer(httpServer) + return workerstatus.ErrUnavailable + } + armWorkerStatus(status, readiness, natsClient, supervisor) xlog.Info("Worker ready, waiting for backend.install events") // Exit on an OS signal or on an internal fatal condition (e.g. NATS // credentials became unrenewable), so the worker restarts and re-acquires @@ -293,6 +310,7 @@ func Run(ctx *cliContext.Context, cfg *Config) error { xlog.Error("Internal shutdown requested", "error", runErr) } + status.Update(func(s *workerstatus.Snapshot) { s.Failure = "stopping"; s.Armed = false }) xlog.Info("Shutting down worker") shutdownCancel() // stop heartbeat loop immediately regClient.GracefulDeregister(nodeID) diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index 111287ce5551..9696e0784752 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -1496,3 +1496,66 @@ Further enhancements, surfaced from a survey of SGLang, vLLM production-stack, R - **Prefill/decode disaggregation routing** ([#10068](https://github.com/mudler/LocalAI/issues/10068)): route prefill and decode to separate pools with KV transfer. - **Per-user fairness (VTC)** ([#10069](https://github.com/mudler/LocalAI/issues/10069)): balance per-user token usage against pod load. - **Minor tuning + MCP parity** ([#10070](https://github.com/mudler/LocalAI/issues/10070)): per-model TTL override, probabilistic LRU updates, and MCP scheduling-config tool parity. + +### Desktop launcher preparation + +The desktop launcher now validates a separate generic-worker launch profile, but +**worker startup remains disabled** until runtime status and process-ownership +integration is available. Use the existing CLI workflow above for deployments; +changing the launcher's saved mode does not enable joining. See +[launcher configuration](/features/launcher-configuration/) for the profile, +authentication/TLS validation and log-security boundaries. This preparation does +not add NATS discovery, P2P, another transport, or worker-side TLS termination. + +### Optional launcher-local worker evidence + +Generic workers support the opt-in, owner-only schema-1 status snapshot described +in [Launcher configuration](/features/launcher-configuration/#private-worker-status-contract-runtime-support). +It is disabled unless both launcher status environment values are supplied and +adds no network endpoint. Existing unmanaged invocation remains unchanged. +Heartbeat acknowledgements now require HTTP 2xx; a rejected heartbeat is no longer +counted as success. Registration alone, static NATS credentials alone, and an +unarmed `/readyz` response are not launcher readiness evidence. + +Launcher-managed workers now observe approval through the read-only, +registration-token-protected `GET /api/node/:id/status` endpoint every ten seconds, +even with non-expiring or static NATS credentials. A successful heartbeat alone +does not prove approval. Managed credential refresh uses the `renew_node_id` +registration request field to avoid restoring offline nodes or changing telemetry. +Controllers lacking this support cannot provide launcher-ready evidence. See the +[approval and process freshness contract](/features/launcher-configuration/#approval-and-process-freshness). + +Renewal failures downgrade launcher status even if approval, NATS, and heartbeat +remain healthy. The snapshot records `renewal_failure` separately from retained +credentials and current approval: `request_failed`, `pending`, or +`missing_credentials`. Only a successful credential renewal clears this evidence; +a successful approval query or heartbeat cannot clear it. A retained JWT can still +work on the current connection while renewal needs attention. + +For dynamic credential-provider connections, repeated authentication errors after +JWT expiry do not terminate NATS reconnect attempts. A successful renewal can +restore the connection to a continuously running broker. Authentication and +subscription permissions remain enforced, and readiness still requires fresh +approval, renewal, NATS, readiness checks, and heartbeat evidence. Static and +unauthenticated connection retry semantics are unchanged; the credential manager's +existing bounded renewal retry policy still applies. + +#### Backend process ownership + +A worker records every backend process group it starts in +`worker-processes.json` in its data directory (owner-only file, atomic replace). +Each record holds the group leader's process ID, its start time and, for +launcher-managed workers, the launch generation. The start time comes from +`/proc` where it exists and from `ps` elsewhere, such as macOS, so a recycled +process ID is not mistaken for a backend. + +- The worker does not run a backend whose record it cannot write. The start + fails and the new process is stopped. +- A record is released only when the whole process group is gone. A group whose + leader exited but still has members is killed, and stays recorded if it + cannot be emptied within five seconds. +- At startup the worker kills the groups an earlier worker left behind. If a + record cannot be resolved (the identity of a live group cannot be verified, + a group survives the kill, or the file is unreadable) the worker refuses to + start, kills nothing unverified, and keeps the record. Remove the file only + after you confirm that no backend process of the earlier worker is running. diff --git a/docs/content/features/launcher-configuration.md b/docs/content/features/launcher-configuration.md new file mode 100644 index 000000000000..c38c7d97ca2b --- /dev/null +++ b/docs/content/features/launcher-configuration.md @@ -0,0 +1,283 @@ +--- +title: Launcher configuration +--- + +The desktop launcher stores settings in `~/.localai/launcher.json`. Existing +flat settings migrate automatically to schema version 2. The launcher continues +to run the standalone server with the same paths, address, and environment. + +## Settings layout + +Version 2 separates settings into these objects: + +- `schema_version`: `2`. +- `mode`: `standalone` or `worker`. Files without a mode use `standalone`. +- `shared`: `models_path`, `backends_path`, `log_level`, `auto_start_server`, + and `show_welcome`. +- `standalone`: `address` and `environment_vars`. +- `worker.connection`: `frontend_url`, `registration_token`, + `advertised_address`, `node_name`, and optional integer `grpc_max_port`. +- `worker.nats`: `url`, `tls_ca`, `tls_cert`, `tls_key`, `jwt`, and `user_seed`. +- `worker.environment_vars`: separate advanced runtime overrides. +- `legacy`: preserves `start_on_boot` when present in an older file. + +Selecting `worker` never starts a standalone server as a fallback. The settings +window described in [Settings window](#settings-window) edits both profiles. + +Legacy environment settings stay in the standalone profile and are not copied +to worker settings. Saving unrelated settings retains remembered worker values. +Empty worker settings may be stored. Unknown modes and unsupported schema +versions produce a settings error instead of launching a guessed role. + +`auto_start_server: false` disables automatic startup; `true` enables it. +Absent or `null` retains the default of enabled. The obsolete `auto_start` key +is ignored. `legacy.start_on_boot` is inert, including when true: it does not +override an explicit opt-out or install an OS service, login item, or boot task. + +## Settings window + +The window begins with a mode selector: **Run locally** or **Join a distributed +server**. Joining contributes this machine's inference capacity to an existing +deployment. It does not create one. The two modes are mutually exclusive and each +keeps its draft when you switch; paths and log level are shared. + +The join form takes the frontend URL, the registration token, the advertised +address and an optional node name. **Advanced connection settings** holds the +NATS URL, the NATS CA, client certificate and key files, the maximum gRPC port, +optional NATS JWT and user seed overrides, runtime environment overrides +(`KEY=VALUE` per line), and the plaintext acknowledgement. The window shows the +reverse HTTP port and the gRPC port range the frontend must reach. + +Secret fields are masked and never filled in from the saved file. A blank secret +keeps the saved value when you save other settings; type a new value to replace +it, or tick **Clear the saved ...** to remove it. Plain `http://` or `nats://` +connections need the acknowledgement checkbox, which lasts for the launcher +session and is never saved. + +**Save Configuration** persists both profiles and the selected mode. An invalid +draft of the inactive profile does not block saving; an invalid active worker +profile blocks only the worker start. When a runtime is running and the change +affects it (a role switch, or any shared or active-profile setting), **Save** +offers **Stop and apply** or **Cancel**. Cancel changes nothing. Stop and apply +saves only after the runtime and its backends are confirmed gone, and does not +start anything. Start, save and update actions are refused while this runs. + +**Start selected mode when launcher opens** keeps the default of enabled for +standalone profiles. A worker starts at launch only when you chose it. In worker +mode the tray offers **Start worker**, **Stop worker** and **Open deployment UI**; +the last opens the saved frontend URL without credentials, and the browser +handles its own login. The registration token is not a browser session. + +## File security and recovery + +The launcher requires an owned regular configuration file and tightens existing +permissions to `0600` **before** reading its contents. Saves write and flush an +owner-only temporary file in the same directory, then atomically replace the +configuration. Failed saves leave the active settings unchanged and block +automatic startup until settings can be saved securely again. + +Symlink files, hard-linked files, foreign ownership, and symlink configuration +directories are rejected. `~/.localai` must be owned by your account and not +writable by other users. New configuration directories use `0700`; existing +model and backend directory permissions are not recursively modified. + +If saving or loading fails, check ownership, free disk space and directory +permissions. Remove unsafe links only after checking their targets; restore a +regular `launcher.json` from a trusted backup if needed. Do not resolve errors +by granting world-write access. Configuration errors do not include saved +values. Configuration and environment dumps have been removed from launcher +logging. + +{{% notice warning %}} +Remembered registration tokens and optional NATS JWT/seed values are **plaintext** +inside the protected file, not encrypted at rest. File permissions do not protect +against the same user, root, backups, or child-process inspection. Protect backups +accordingly. The settings window masks these fields; see below. +{{% /notice %}} + +## Launch validation and logs + +The launch builder validates the saved worker profile before any start. A worker +starts only through the lifecycle controller described in +[Lifecycle and ownership](#lifecycle-and-ownership), which adds the runtime +compatibility, status and descendant-ownership gates. The settings window for +choosing a mode is not part of this change: a worker starts only when the saved +mode is `worker` and the profile validates, and worker errors never fall back to +standalone. + +The worker profile requires an explicit frontend URL (prefer +`https://controller.example/prefix`), one NATS endpoint (prefer +`tls://nats.example:4222`), a registration token, and an advertised address such +as `worker.example:50051` or `[2001:db8::10]:50051`. URL userinfo, queries and +fragments are forbidden; NATS paths are not supported. Frontend path prefixes +are retained. Plain HTTP/NATS requires explicit protected-network acknowledgement +at the builder boundary; no launcher control enables it yet. + +The base port must be 2–65535. Reverse HTTP uses base minus one; gRPC allocates +from base through `grpc_max_port` (default 65535). Loopback addresses only work +for colocated peers. Advertised addresses do not restrict wildcard binding or +prove remote reachability. Reverse HTTP/gRPC requires a protected network; this +is not end-to-end encryption. + +Authentication is always required. The registration token is passed only in the +child environment, never argv. Normally registration supplies and renews NATS +JWT/seed credentials after approval. Optional static overrides must be a matching +valid user JWT and seed pair; static mode has no automatic renewal. Possession +of credentials is not evidence of approval or readiness. NATS TLS file paths +must be readable regular files, and client certificate/key must be paired. +Relative CA, client certificate and key paths resolve against the launcher's +working directory when the launch description is built, not the child's data +directory. The builder validates and snapshots those absolute paths without +changing saved settings. Prefer absolute paths when configuring TLS files. +NATS TLS settings require `tls://`; they do not configure frontend HTTPS trust, +which uses the OS trust store. + +Advanced worker environment settings cannot override role, authentication, +connection, address, path or launcher-observability settings, including supported +aliases. Inherited reserved values are removed. Non-reserved accelerator options +are retained. Standalone arguments and environment precedence remain unchanged. +The builder snapshots values without modifying the launcher's environment. + +Child output is filtered before reaching the UI, in-memory buffer or log file. +Credential-bearing, control-character, oversized and incomplete lines are omitted +rather than retained raw. Configured credentials, issued JWT/seed forms, +authorization headers and credential-bearing URLs are covered. Lines are limited +to 16 KiB, UI/memory buffers to 100 KB, and each session log to 1 MiB (additional +file output is dropped). New log files are owner-only (`0600`) in an owned `0700` +logs directory; unsafe directories are rejected. Existing historical log files +are not rewritten or deleted. Review and remove old logs yourself if they may +contain credentials; filtering is not encryption or protection from your account +or root. + +## Private worker status contract (runtime support) + +The launcher starts a worker only after the lifecycle controller has confirmed +that no process of an earlier run is still owned (see +[Lifecycle and ownership](#lifecycle-and-ownership)). This runtime support does +not enable the settings UI or start a worker automatically. + +Compatible runtimes implement `local-ai worker-status-capability`, which prints +`localai-worker-status-v1` and exits without starting a worker. The launcher +compatibility helper uses a five-second, bounded-output probe, not a guessed +release-number threshold. An incompatible runtime requires an upgrade; there is +no standalone fallback or log-based readiness inference. + +Launcher-managed workers opt in through child-only environment variables: + +- `LOCALAI_LAUNCHER_STATUS_PATH`: absolute `status.json` path inside a fresh, + owner-only mode-0700 run directory. +- `LOCALAI_LAUNCHER_GENERATION`: unique per-launch identity, matched by the reader. + +Both values are reserved against inherited/profile overrides. Without them, +workers do not create status files. The file is a local observation channel, not +an HTTP API, transport, process ledger, or command queue. It is atomically +replaced using an owner-only mode-0600 temporary file, every two seconds and on +observed transitions. Unsafe ownership, permissions, symlinks, hard links, +malformed data, unknown schema versions, and snapshots over 16 KiB are refused. +A publication failure invalidates old evidence and requests worker shutdown. + +Schema 1 includes generation, increasing sequence, publication time, +registration outcome/time, credential availability (booleans, never secrets), +NATS connectivity and server-required authentication, confirmed lifecycle subscriptions, armed local checks, +last successful heartbeat acknowledgement, heartbeat interval, and categorical +failure. Static credentials do not imply approval. Registration pending and +approved-but-missing-credentials remain distinct. Non-2xx heartbeats are failures. + +The reader measures sequence advancement with a monotonic clock; rereading a +file cannot refresh it. Ten seconds without advancement means status unavailable. +Initial registration has a 30-second progress deadline, except a fresh explicit +pending response. Ready requires approval, credentials, NATS, subscriptions, +armed passing checks and a heartbeat within three intervals (normally 30 seconds). +Early `/readyz` success and process existence are not proof. + +**Ready — waiting for work** means the worker can accept work under these checks. +It does not certify reverse network reachability, model availability, VRAM, or +successful inference. Full authenticated frontend/inference and native macOS +launcher acceptance remain separate validation requirements. + +#### Approval and process freshness + +Status schema 1 also carries the worker PID. The future lifecycle controller must +supply the PID of its owned child to the reader, together with the fresh random +per-launch generation; it must never learn the expected PID from the snapshot. +This binding is not a replacement for portable descendant ownership or PID-reuse +checks during crash recovery. Worker startup in the launcher remains disabled. + +Each new sequence must have a publication timestamp less than ten seconds old, +not in the future, and not older than the last accepted publication. Approval and +heartbeat timestamps cannot be later than publication. Once admitted, remaining +freshness is measured with the reader's monotonic clock, including the initial +publication age. Neither rereading nor a delayed unseen sequence grants a fresh +lease. Approval must be nonzero and less than 30 seconds old. + +Both static and registration-issued credential paths query controller approval +at ten-second intervals, independently of heartbeat and JWT renewal, including +non-expiring JWTs. `GET /api/node/:id/status` uses the existing worker +registration-token authentication boundary and returns only `id` and `status`. +It does not mint credentials, update heartbeat, or restore offline nodes. An old +controller without this endpoint fails closed. Like existing self-service routes, +the shared registration token is not per-node authorization. + +Managed credential renewal uses `POST /api/node/register` with `renew_node_id` +and `token`, rather than restarting registration. This observes the existing row +without restoring its status or updating telemetry; credentials are minted only +for a healthy node. Periodic approval never changes the credential manager, and +renewal responses cannot overwrite the independently observed approval evidence. +The optional `renewal_failure` field records `request_failed`, `pending`, or +`missing_credentials` independently of retained credentials. Any renewal failure +prevents ready status until a successful credential renewal clears it. Heartbeat +and approval success cannot clear a renewal failure. +Dynamic credential-provider NATS connections keep retrying authentication if the +JWT expires during a renewal outage, including repeated identical broker errors. +A later successful renewal can therefore reconnect to the still-running broker +without restarting it or the worker. This does not bypass authentication or make +the worker ready during the outage: all readiness evidence must become current +again. Static credentials retain NATS's default terminal authentication-error +behavior. The credential manager's existing bounded renewal retries are unchanged. +A launcher-managed worker also keeps a durable record of its backend process +groups, tagged with the launch generation, and refuses to start while an earlier +group cannot be verified or cleaned up. See +[Backend process ownership](/features/distributed-mode/#backend-process-ownership). +This worker-only surface is not an admin action or a UI capability. +The approval route is not registered when the controller has no registration +token, and renewal-only requests are refused in that configuration. Missing or +incorrect bearer tokens are rejected before registry access. These worker +self-service operations do not use admin feature permissions or MCP tools. + +## Lifecycle and ownership + +One lifecycle controller owns the launcher's runtime child. It allows one +operation at a time: while a runtime is running, starting or stopping, another +start or stop is refused. Work tagged with an older run is discarded. + +A per-user lock file (`~/.localai/launcher.lock`, owner-only, not inherited by +the runtime) stops a second launcher from starting an overlapping role. The +operating system drops it when the holder exits or crashes. + +**Start.** The launcher first reads the worker's backend process ledger +(`worker-processes.json`, next to the models directory under `data/`). It kills +and confirms every group that provably belongs to an earlier run, and drops +records whose group is gone or whose process id now belongs to something else. +A record it cannot resolve (identity unverifiable, group that survives the +kill, unreadable ledger) blocks the start and nothing unverified is signalled. +A worker additionally needs a runtime that answers +`worker-status-capability`; an older runtime gives an upgrade-required error and +never falls back to log parsing. + +**Stop.** The launcher sends `SIGTERM` and allows 30 seconds for draining, +deregistration and backend cleanup. It then kills the runtime's process group, +allows 5 more seconds, and kills every recorded backend group. It reports +**Stopped** only after it has verified that the runtime group and all recorded +backend groups are empty. Otherwise it shows **Failed: shutdown incomplete**, +keeps the lock, blocks every start, and leaves **Stop** available to retry. +An unexpected runtime exit runs the same cleanup, because a parent exit does +not prove its backends stopped. Quit uses the same stop path and does not +release the lock when shutdown is incomplete. + +A forced stop can interrupt active inference. A failure to deregister from the +frontend is a warning and does not decide whether another role may start. +Local ownership does. + +Worker state shown by the launcher comes from the private status snapshot, not +from log text. Process liveness alone shows **Connection problem: status +unavailable**, never **Ready**. diff --git a/pkg/workerledger/group_unix.go b/pkg/workerledger/group_unix.go new file mode 100644 index 000000000000..a5daf9e08474 --- /dev/null +++ b/pkg/workerledger/group_unix.go @@ -0,0 +1,19 @@ +//go:build !windows + +package workerledger + +import ( + "errors" + "syscall" +) + +// killProcessGroup sends SIGKILL to the process group led by pid. +func killProcessGroup(pid int) error { return syscall.Kill(-pid, syscall.SIGKILL) } + +// GroupAlive reports whether any process, a zombie included, is still a member +// of the group with id pid. EPERM means the group exists but is not ours to +// signal, which still counts as alive. +func GroupAlive(pid int) bool { + err := syscall.Kill(-pid, 0) + return err == nil || errors.Is(err, syscall.EPERM) +} diff --git a/pkg/workerledger/group_windows.go b/pkg/workerledger/group_windows.go new file mode 100644 index 000000000000..d66344d196be --- /dev/null +++ b/pkg/workerledger/group_windows.go @@ -0,0 +1,12 @@ +//go:build windows + +package workerledger + +import "errors" + +// killProcessGroup is not supported on Windows: backends there are not started +// in process groups, so no entry is ever swept. +func killProcessGroup(int) error { return errors.New("process groups are not supported on windows") } + +// GroupAlive is always false: there are no process groups to own on Windows. +func GroupAlive(int) bool { return false } diff --git a/pkg/workerledger/identity.go b/pkg/workerledger/identity.go new file mode 100644 index 000000000000..925a1aa8d6c3 --- /dev/null +++ b/pkg/workerledger/identity.go @@ -0,0 +1,199 @@ +// SPDX-License-Identifier: MIT + +// Package workerledger holds the process identity rules shared by a worker, +// which records the backend process groups it owns, and the launcher that +// manages it and must find those groups again after a crash. +// +// A record is the group leader's pid, its start time and the launch generation. +// The start time keeps a recycled pid from being mistaken for a backend. +package workerledger + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" +) + +// Entry is one recorded backend process group. +type Entry struct { + Key string `json:"key"` + PID int `json:"pid"` + StartTime string `json:"start_time"` + Generation string `json:"generation,omitempty"` +} + +// Ownership states of a recorded group. +type Ownership int + +const ( + // Gone: no live process group has the recorded id. + Gone Ownership = iota + // Foreign: the recorded pid now leads an unrelated group. + Foreign + // Owned: a live group whose identity matches the record. + Owned + // Uncertain: a live group that cannot be proven to be ours. + Uncertain +) + +// ReapTimeout bounds how long a verified group is given to disappear after it +// is killed. It is a variable only so tests can shorten it. +var ReapTimeout = 5 * time.Second + +// KillGroup is the signal sender; tests replace it to model a kill that fails. +var KillGroup = killProcessGroup + +// ProcRoot is where the kernel process files live. It is a variable so tests +// can model a system without it, such as macOS. +var ProcRoot = "/proc" + +// ReadStartTime returns a process's start time as the system reports it, or "" +// when it cannot be read (the process is gone, or neither /proc nor ps works). +// Values from /proc and from ps are never compared with each other: one system +// uses one source for the lifetime of a ledger. +var ReadStartTime = portableStartTime + +// ReadFile loads the entries of a ledger file. A missing file is an empty +// ledger; an unreadable or corrupt one is an error, because ownership of the +// earlier processes is then unknown. +func ReadFile(path string) ([]Entry, error) { + data, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("reading the process ledger: %w", err) + } + var entries []Entry + if err := json.Unmarshal(data, &entries); err != nil { + return nil, fmt.Errorf("the process ledger is unreadable, so ownership of earlier backends is unknown: %w", err) + } + return entries, nil +} + +// WriteFile replaces path with entries through a synced, owner-only temporary +// file so a crash leaves either the old or the new content. +func WriteFile(path string, entries []Entry) error { + if entries == nil { + entries = []Entry{} + } + data, err := json.Marshal(entries) + if err == nil { + err = os.MkdirAll(filepath.Dir(path), 0o750) + } + if err != nil { + return err + } + tmp := path + ".tmp" + f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600) + if err != nil { + return err + } + if _, err = f.Write(data); err == nil { + err = f.Sync() + } + if cerr := f.Close(); err == nil { + err = cerr + } + if err == nil { + err = os.Rename(tmp, path) + } + if err != nil { + _ = os.Remove(tmp) + } + return err +} + +// Classify decides whose a recorded group is. A group id cannot be reused while +// any member lives, so a missing leader with a live group is ours; a live leader +// with a different start time is a recycled pid. +func Classify(e Entry) Ownership { + if e.PID <= 0 || !GroupAlive(e.PID) { + return Gone + } + if e.StartTime == "" { + return Uncertain + } + now := ReadStartTime(e.PID) + switch { + case now == "" && StartTimesReadable(): + return Owned // the leader exited; its group lives on + case now == "": + return Uncertain + case now != e.StartTime: + return Foreign + } + return Owned +} + +// Reap kills a verified group and waits until it is empty. +func Reap(e Entry) error { + if err := KillGroup(e.PID); err != nil && GroupAlive(e.PID) { + return fmt.Errorf("signalling process group %d: %w", e.PID, err) + } + deadline := time.Now().Add(ReapTimeout) + for GroupAlive(e.PID) { + if time.Now().After(deadline) { + return fmt.Errorf("process group %d still has live members after %s", e.PID, ReapTimeout) + } + time.Sleep(25 * time.Millisecond) + } + return nil +} + +// StartTimesReadable reports whether this system can tell a process's start +// time, through /proc or ps. +func StartTimesReadable() bool { return ReadStartTime(os.Getpid()) != "" } + +func portableStartTime(pid int) string { + if _, err := os.Stat(filepath.Join(ProcRoot, "self", "stat")); err == nil { + return procStartTime(pid) + } + return psStartTime(pid) +} + +func procStartTime(pid int) string { + data, err := os.ReadFile(filepath.Join(ProcRoot, strconv.Itoa(pid), "stat")) + if err != nil { + return "" + } + // The command name is in parentheses and may hold spaces; the fields after + // the last ')' are space separated and start at field 3. Field 22 is the + // start time. + s := string(data) + i := strings.LastIndex(s, ")") + if i < 0 { + return "" + } + fields := strings.Fields(s[i+1:]) + if len(fields) < 20 { + return "" + } + return fields[19] +} + +// psStartTime asks ps for the process start time (one second granularity), +// which is how macOS and the BSDs expose it. +func psStartTime(pid int) string { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, "ps", "-o", "lstart=", "-p", strconv.Itoa(pid)) + // A fixed locale keeps the text identical for every process that reads it. + cmd.Env = []string{"LC_ALL=C", "LANG=C", "PATH=/usr/bin:/bin"} + out, err := cmd.Output() + if err != nil { + return "" + } + fields := strings.Fields(string(out)) + if len(fields) == 0 { + return "" + } + return "ps:" + strings.Join(fields, " ") +} diff --git a/pkg/workerledger/identity_test.go b/pkg/workerledger/identity_test.go new file mode 100644 index 000000000000..37dd90ebf151 --- /dev/null +++ b/pkg/workerledger/identity_test.go @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: MIT +package workerledger_test + +import ( + "os" + "path/filepath" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/pkg/workerledger" +) + +func TestWorkerLedger(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Worker ledger") +} + +var _ = Describe("Process ledger file", func() { + It("distinguishes a missing ledger from a corrupt one", func() { + dir := GinkgoT().TempDir() + entries, err := workerledger.ReadFile(filepath.Join(dir, "none.json")) + Expect(err).ToNot(HaveOccurred()) + Expect(entries).To(BeNil()) + bad := filepath.Join(dir, "bad.json") + Expect(os.WriteFile(bad, []byte("{"), 0o600)).To(Succeed()) + _, err = workerledger.ReadFile(bad) + Expect(err).To(HaveOccurred(), "ownership of earlier backends is unknown") + }) + + It("writes owner-only, atomically, and round-trips", func() { + path := filepath.Join(GinkgoT().TempDir(), "sub", "ledger.json") + want := []workerledger.Entry{{Key: "m#0", PID: 42, StartTime: "7", Generation: "g"}} + Expect(workerledger.WriteFile(path, want)).To(Succeed()) + info, err := os.Stat(path) + Expect(err).ToNot(HaveOccurred()) + Expect(info.Mode().Perm()).To(Equal(os.FileMode(0o600))) + got, err := workerledger.ReadFile(path) + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(Equal(want)) + _, err = os.Stat(path + ".tmp") + Expect(os.IsNotExist(err)).To(BeTrue(), "no temporary file is left behind") + }) + + It("classifies a missing group as gone", func() { + Expect(workerledger.Classify(workerledger.Entry{PID: 0})).To(Equal(workerledger.Gone)) + Expect(workerledger.Classify(workerledger.Entry{PID: 1 << 30, StartTime: "1"})).To(Equal(workerledger.Gone)) + }) +}) diff --git a/pkg/workerstatus/capability.go b/pkg/workerstatus/capability.go new file mode 100644 index 000000000000..a97bf919e341 --- /dev/null +++ b/pkg/workerstatus/capability.go @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: MIT +package workerstatus + +import "fmt" + +// CapabilityCommand has no connection flags and never starts a worker. +type CapabilityCommand struct{} + +func (*CapabilityCommand) Run() error { _, err := fmt.Println(Capability); return err } diff --git a/pkg/workerstatus/publisher.go b/pkg/workerstatus/publisher.go new file mode 100644 index 000000000000..41a9addd4c01 --- /dev/null +++ b/pkg/workerstatus/publisher.go @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: MIT +package workerstatus + +import ( + "context" + "os" + "sync" + "time" +) + +// Publisher serializes transitions and periodic snapshots. A persistence failure +// is terminal: remove old evidence and notify the worker to shut down. Readers +// also expire a snapshot whose sequence no longer advances. +type Publisher struct { + mu sync.Mutex + path string + state Snapshot + failed bool + cancel context.CancelFunc + probe func() (connected, authenticated, subscriptions, ready bool) +} + +func FromEnvironment(ctx context.Context, cancel context.CancelFunc) (*Publisher, error) { + path, generation := os.Getenv(PathEnv), os.Getenv(GenerationEnv) + if path == "" && generation == "" { + return nil, nil + } + return NewPublisher(ctx, path, generation, cancel) +} + +// Generation returns the launch generation this publisher reports, or "" for a +// nil publisher (a worker not managed by a launcher). +func (p *Publisher) Generation() string { + if p == nil { + return "" + } + p.mu.Lock() + defer p.mu.Unlock() + return p.state.Generation +} + +func NewPublisher(ctx context.Context, path, generation string, cancel context.CancelFunc) (*Publisher, error) { + p := &Publisher{path: path, cancel: cancel, state: Snapshot{PID: os.Getpid(), Schema: Schema, Generation: generation, Registration: "starting", HeartbeatInterval: 10 * time.Second}} + // A fresh launch must never adopt a previous session's file. + if err := requireNoSnapshot(path); err != nil { + return nil, err + } + if err := p.publishLocked(); err != nil { + return nil, err + } + go func() { + ticker := time.NewTicker(PollInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + p.Update(nil) + } + } + }() + return p, nil +} +func (p *Publisher) publishLocked() error { + if p.failed { + return ErrUnavailable + } + p.state.Sequence++ + p.state.Time = time.Now() + if err := Write(p.path, p.state); err != nil { + p.failed = true + // Never follow a replaced parent/snapshot to invalidate it. + if dir, e := directory(p.path); e == nil { + _ = removeSnapshot(dir) + dir.Close() + } + if p.cancel != nil { + p.cancel() + } + return ErrUnavailable + } + return nil +} +func (p *Publisher) Update(fn func(*Snapshot)) { + if p == nil { + return + } + p.mu.Lock() + defer p.mu.Unlock() + if p.failed { + return + } + if fn != nil { + fn(&p.state) + } + if p.probe != nil { + p.state.Connected, p.state.Authenticated, p.state.Subscriptions, p.state.LocalReady = p.probe() + } + _ = p.publishLocked() +} +func (p *Publisher) Arm(probe func() (bool, bool, bool, bool)) { + if p == nil { + return + } + p.mu.Lock() + defer p.mu.Unlock() + p.probe = probe + p.state.Armed = true + p.state.Connected, p.state.Authenticated, p.state.Subscriptions, p.state.LocalReady = probe() + _ = p.publishLocked() +} +func (p *Publisher) Registration(status string, credentials bool, err error) { + p.Update(func(s *Snapshot) { + s.RegistrationTime = time.Now() + s.Credentials = credentials + if err != nil { + s.Registration = "error" + return + } + switch status { + case "healthy", "pending": + s.Registration = status + default: + s.Registration = "unknown" + } + }) +} +func (p *Publisher) Heartbeat(err error) { + p.Update(func(s *Snapshot) { + if err != nil { + s.Failure = "heartbeat" + } else { + s.Heartbeat = time.Now() + s.Failure = "" + } + }) +} diff --git a/pkg/workerstatus/replace_race_test.go b/pkg/workerstatus/replace_race_test.go new file mode 100644 index 000000000000..f7150749e00b --- /dev/null +++ b/pkg/workerstatus/replace_race_test.go @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: MIT +package workerstatus + +import ( + "sync" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// A reader that opened the snapshot just before it was replaced holds an inode +// with no links left. That file was complete and owner-only, so it is valid +// evidence; only an extra hard link (more than one) is a reason to refuse it. +var _ = Describe("Snapshot replacement", func() { + It("stays readable for a reader that opened the previous file", func() { + for round := 0; round < 200; round++ { + path, generation, err := NewSession(GinkgoT().TempDir()) + Expect(err).ToNot(HaveOccurred()) + s := Snapshot{PID: 1, Schema: Schema, Generation: generation, Sequence: 1, Time: time.Now(), Registration: "starting", HeartbeatInterval: 10 * time.Second} + Expect(Write(path, s)).To(Succeed()) + var wg sync.WaitGroup + stop := make(chan struct{}) + wg.Go(func() { + defer GinkgoRecover() + for { + select { + case <-stop: + return + default: + s.Sequence++ + Expect(Write(path, s)).To(Succeed()) + } + } + }) + for i := 0; i < 40; i++ { + _, err := Read(path, generation) + Expect(err).ToNot(HaveOccurred(), "round %d read %d", round, i) + } + close(stop) + wg.Wait() + } + }) +}) diff --git a/pkg/workerstatus/review_test.go b/pkg/workerstatus/review_test.go new file mode 100644 index 000000000000..f8a1ed802eff --- /dev/null +++ b/pkg/workerstatus/review_test.go @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: MIT +package workerstatus + +import ( + "context" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "os" + "time" +) + +var _ = Describe("Worker status review regressions", func() { + It("rejects expired approval even with fresh heartbeat and publication", func() { + now := time.Now() + for _, approval := range []time.Time{{}, now.Add(-31 * time.Second)} { + s := Snapshot{PID: os.Getpid(), Schema: Schema, Generation: "g", Sequence: 1, Time: now, Registration: "healthy", RegistrationTime: approval, Credentials: true, Authenticated: true, Connected: true, Subscriptions: true, Armed: true, LocalReady: true, Heartbeat: now, HeartbeatInterval: 10 * time.Second} + tracker := Tracker{ExpectedPID: os.Getpid(), Generation: "g", Started: now} + Expect(tracker.Classify(s, now)).To(Equal("registration")) + } + }) + It("does not grant unseen stale publications a new lease", func() { + now := time.Now() + s := Snapshot{PID: os.Getpid(), Schema: Schema, Generation: "g", Sequence: 1, Time: now.Add(-time.Hour), Registration: "healthy", RegistrationTime: now.Add(-time.Hour), Credentials: true, Authenticated: true, Connected: true, Subscriptions: true, Armed: true, LocalReady: true, Heartbeat: now.Add(-time.Hour), HeartbeatInterval: 10 * time.Second} + tracker := Tracker{ExpectedPID: os.Getpid(), Generation: "g", Started: now} + Expect(tracker.Classify(s, now)).To(Equal("status_unavailable")) + }) +}) + +var _ = Describe("Worker status admission", func() { + It("rejects wrong identities, future timestamps and stale progress after a pause", func() { + now := time.Now() + s := Snapshot{PID: os.Getpid(), Schema: Schema, Generation: "g", Sequence: 1, Time: now, Registration: "starting", HeartbeatInterval: time.Second} + tracker := Tracker{ExpectedPID: os.Getpid(), Generation: "g", Started: now} + bad := s + bad.PID++ + Expect(tracker.Classify(bad, now)).To(Equal("status_unavailable")) + bad = s + bad.Time = now.Add(time.Second) + Expect(tracker.Classify(bad, now)).To(Equal("status_unavailable")) + bad = s + bad.RegistrationTime = now.Add(time.Second) + Expect(tracker.Classify(bad, now)).To(Equal("status_unavailable")) + Expect(tracker.Classify(s, now)).To(Equal("starting")) + s.Sequence++ + Expect(tracker.Classify(s, now.Add(time.Minute))).To(Equal("status_unavailable")) + }) + It("activates only complete inherited opt-in and records its actual PID", func() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + GinkgoT().Setenv(PathEnv, "") + GinkgoT().Setenv(GenerationEnv, "") + p, err := FromEnvironment(ctx, cancel) + Expect(err).NotTo(HaveOccurred()) + Expect(p).To(BeNil()) + path, generation, err := NewSession(GinkgoT().TempDir()) + Expect(err).NotTo(HaveOccurred()) + GinkgoT().Setenv(PathEnv, path) + _, err = FromEnvironment(ctx, cancel) + Expect(err).To(HaveOccurred()) + GinkgoT().Setenv(GenerationEnv, generation) + p, err = FromEnvironment(ctx, cancel) + Expect(err).NotTo(HaveOccurred()) + Expect(p).NotTo(BeNil()) + s, err := Read(path, generation) + Expect(err).NotTo(HaveOccurred()) + Expect(s.PID).To(Equal(os.Getpid())) + }) +}) diff --git a/pkg/workerstatus/status.go b/pkg/workerstatus/status.go new file mode 100644 index 000000000000..1605ab1a4cc0 --- /dev/null +++ b/pkg/workerstatus/status.go @@ -0,0 +1,279 @@ +// SPDX-License-Identifier: MIT +// Package workerstatus defines the private, opt-in launcher observation channel. +// It carries evidence, never credentials or process-control instructions. +package workerstatus + +import ( + "bytes" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "io" + "os" + "path/filepath" + "time" + + "golang.org/x/sys/unix" +) + +const Schema = 1 +const Capability = "localai-worker-status-v1" +const PathEnv = "LOCALAI_LAUNCHER_STATUS_PATH" +const GenerationEnv = "LOCALAI_LAUNCHER_GENERATION" +const MaxSize = 16 * 1024 +const PollInterval = 2 * time.Second +const ApprovalInterval = 10 * time.Second +const ApprovalMaxAge = 30 * time.Second + +var ErrUnavailable = errors.New("worker status unavailable") + +type Snapshot struct { + PID int `json:"pid"` + Schema int `json:"schema_version"` + Generation string `json:"generation"` + Sequence uint64 `json:"sequence"` + Time time.Time `json:"time"` + Registration string `json:"registration"` + RegistrationTime time.Time `json:"registration_time"` + RenewalFailure string `json:"renewal_failure,omitempty"` + Credentials bool `json:"credentials"` + Authenticated bool `json:"nats_authenticated"` + Connected bool `json:"nats_connected"` + Subscriptions bool `json:"lifecycle_subscriptions"` + Armed bool `json:"readiness_armed"` + LocalReady bool `json:"local_ready"` + Heartbeat time.Time `json:"heartbeat_ack"` + HeartbeatInterval time.Duration `json:"heartbeat_interval_ns"` + Failure string `json:"failure"` +} + +func (s Snapshot) valid() bool { + if s.Schema != Schema || len(s.Generation) == 0 || len(s.Generation) > 128 || s.Sequence == 0 || s.Time.IsZero() || s.HeartbeatInterval <= 0 || s.HeartbeatInterval > time.Duration(1<<63-1)/3 { + return false + } + switch s.Registration { + case "starting", "pending", "healthy", "error", "unknown": + default: + return false + } + switch s.RenewalFailure { + case "", "request_failed", "pending", "missing_credentials": + default: + return false + } + switch s.Failure { + case "", "registration", "credentials", "nats", "subscriptions", "readiness", "heartbeat", "stopping": + default: + return false + } + return true +} + +// NewSession creates a distinct 0700 directory even for consecutive starts. +// The caller retains it until owned-process cleanup completes. +func NewSession(parent string) (path, generation string, err error) { + dir, e := os.MkdirTemp(parent, "worker-run-") + if e != nil { + return "", "", ErrUnavailable + } + b := make([]byte, 32) + if _, e = rand.Read(b); e != nil { + os.Remove(dir) + return "", "", ErrUnavailable + } + return filepath.Join(dir, "status.json"), hex.EncodeToString(b), nil +} + +func directory(path string) (*os.File, error) { + if !filepath.IsAbs(path) || filepath.Base(path) != "status.json" { + return nil, ErrUnavailable + } + fd, err := unix.Open(filepath.Dir(path), unix.O_RDONLY|unix.O_DIRECTORY|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0) + if err != nil { + return nil, ErrUnavailable + } + f := os.NewFile(uintptr(fd), "worker status directory") + var st unix.Stat_t + if unix.Fstat(fd, &st) != nil || int64(st.Uid) != int64(os.Geteuid()) || st.Mode&07777 != 0700 { + f.Close() + return nil, ErrUnavailable + } + return f, nil +} + +// A replaced snapshot has no links left (Nlink 0) and is still our complete +// file; only an extra hard link is refused. +func openSnapshot(dir *os.File) (*os.File, error) { + fd, err := unix.Openat(int(dir.Fd()), "status.json", unix.O_RDONLY|unix.O_NOFOLLOW|unix.O_NONBLOCK|unix.O_CLOEXEC, 0) + if err != nil { + return nil, err + } + f := os.NewFile(uintptr(fd), "worker status") + var st unix.Stat_t + if unix.Fstat(fd, &st) != nil || int64(st.Uid) != int64(os.Geteuid()) || st.Mode&unix.S_IFMT != unix.S_IFREG || st.Mode&07777 != 0600 || st.Nlink > 1 || st.Size > MaxSize { + f.Close() + return nil, ErrUnavailable + } + return f, nil +} +func Write(path string, s Snapshot) error { + if !s.valid() { + return ErrUnavailable + } + data, err := json.Marshal(s) + if err != nil || len(data) > MaxSize { + return ErrUnavailable + } + dir, err := directory(path) + if err != nil { + return ErrUnavailable + } + defer dir.Close() + old, err := openSnapshot(dir) + if err != nil && !errors.Is(err, unix.ENOENT) { + return ErrUnavailable + } + if old != nil { + old.Close() + } + b := make([]byte, 16) + if _, err = rand.Read(b); err != nil { + return ErrUnavailable + } + name := ".status-" + hex.EncodeToString(b) + fd, err := unix.Openat(int(dir.Fd()), name, unix.O_WRONLY|unix.O_CREAT|unix.O_EXCL|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0600) + if err != nil { + return ErrUnavailable + } + defer unix.Unlinkat(int(dir.Fd()), name, 0) + f := os.NewFile(uintptr(fd), "temporary worker status") + defer f.Close() + if err = f.Chmod(0600); err != nil { + return ErrUnavailable + } + if _, err = f.Write(data); err != nil { + return ErrUnavailable + } + if f.Sync() != nil { + return ErrUnavailable + } + if unix.Renameat(int(dir.Fd()), name, int(dir.Fd()), "status.json") != nil { + return ErrUnavailable + } + if dir.Sync() != nil { + _ = unix.Unlinkat(int(dir.Fd()), "status.json", 0) + return ErrUnavailable + } + return nil +} +func Read(path, generation string) (Snapshot, error) { + var s Snapshot + dir, err := directory(path) + if err != nil { + return s, ErrUnavailable + } + defer dir.Close() + f, err := openSnapshot(dir) + if err != nil { + return s, ErrUnavailable + } + defer f.Close() + data, err := io.ReadAll(io.LimitReader(f, MaxSize+1)) + if err != nil || len(data) > MaxSize { + return s, ErrUnavailable + } + d := json.NewDecoder(bytes.NewReader(data)) + d.DisallowUnknownFields() + if d.Decode(&s) != nil || d.Decode(new(any)) != io.EOF || !s.valid() || s.Generation != generation { + return Snapshot{}, ErrUnavailable + } + return s, nil +} + +// Tracker measures sequence progress using the caller's monotonic clock. Polling +// an unchanged file cannot renew its lease. Never share it across generations. +type Tracker struct { + Generation string + // ExpectedPID must come from the owned child, not from the snapshot. The + // per-launch generation prevents PID reuse across launches. + ExpectedPID int + Started time.Time + sequence uint64 + advanced time.Time + publication time.Time + age time.Duration + last Snapshot +} + +func (t *Tracker) Classify(s Snapshot, now time.Time) string { + if !s.valid() || s.Generation != t.Generation || s.Sequence < t.sequence || t.ExpectedPID <= 0 || s.PID != t.ExpectedPID { + return "status_unavailable" + } + if s.Sequence > t.sequence { + // Wall time is used only at sequence admission. Thereafter use the + // local monotonic clock; polling must not grant a new evidence lease. + age := now.Sub(s.Time) + if age < 0 || age >= 10*time.Second || (!t.publication.IsZero() && s.Time.Before(t.publication)) || s.RegistrationTime.After(s.Time) || s.Heartbeat.After(s.Time) { + return "status_unavailable" + } + t.sequence = s.Sequence + t.advanced = now + t.publication = s.Time + t.age = age + t.last = s + } + if s != t.last || now.Before(t.advanced) || t.age+now.Sub(t.advanced) >= 10*time.Second { + return "status_unavailable" + } + if s.Failure != "" { + return s.Failure + } + approvalCurrent := !s.RegistrationTime.IsZero() && s.Time.Sub(s.RegistrationTime)+t.age+now.Sub(t.advanced) < ApprovalMaxAge + if s.Registration == "pending" && approvalCurrent { + return "pending" + } + if s.Registration == "starting" && now.Sub(t.Started) < 30*time.Second { + return "starting" + } + if s.Registration != "healthy" || !approvalCurrent { + return "registration" + } + if !s.Credentials || s.RenewalFailure != "" { + return "credentials" + } + if !s.Connected { + return "nats" + } + if !s.Authenticated { + return "credentials" + } + if !s.Subscriptions { + return "subscriptions" + } + if !s.Armed || !s.LocalReady { + return "readiness" + } + if s.Heartbeat.IsZero() || s.Heartbeat.After(s.Time) || s.Time.Sub(s.Heartbeat)+t.age+now.Sub(t.advanced) > 3*s.HeartbeatInterval { + return "heartbeat" + } + return "ready" +} + +// requireNoSnapshot succeeds only when the validated session directory holds no +// entry named status.json. The lookup goes through the pinned directory +// descriptor and does not follow a symbolic link. +func requireNoSnapshot(path string) error { + dir, err := directory(path) + if err != nil { + return err + } + defer func() { _ = dir.Close() }() + var st unix.Stat_t + if err := unix.Fstatat(int(dir.Fd()), "status.json", &st, unix.AT_SYMLINK_NOFOLLOW); !errors.Is(err, unix.ENOENT) { + return ErrUnavailable + } + return nil +} + +func removeSnapshot(dir *os.File) error { return unix.Unlinkat(int(dir.Fd()), "status.json", 0) } diff --git a/pkg/workerstatus/status_test.go b/pkg/workerstatus/status_test.go new file mode 100644 index 000000000000..d3b03c0b1641 --- /dev/null +++ b/pkg/workerstatus/status_test.go @@ -0,0 +1,196 @@ +// SPDX-License-Identifier: MIT +package workerstatus + +import ( + "context" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "os" + "path/filepath" + "sync" + "testing" + "time" +) + +func TestStatus(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "Worker status") } + +var _ = Describe("Worker status", func() { + It("publishes private atomic snapshots and rejects old sessions and unsafe files", func() { + root := GinkgoT().TempDir() + path, generation, err := NewSession(root) + Expect(err).NotTo(HaveOccurred()) + s := Snapshot{PID: os.Getpid(), Schema: 1, Generation: generation, Sequence: 1, Time: time.Now(), Registration: "starting", HeartbeatInterval: 10 * time.Second} + Expect(Write(path, s)).To(Succeed()) + got, err := Read(path, generation) + Expect(err).NotTo(HaveOccurred()) + Expect(got.Sequence).To(Equal(uint64(1))) + _, err = Read(path, "old") + Expect(err).To(HaveOccurred()) + st, err := os.Stat(path) + Expect(err).NotTo(HaveOccurred()) + Expect(st.Mode().Perm()).To(Equal(os.FileMode(0600))) + st, err = os.Stat(filepath.Dir(path)) + Expect(err).NotTo(HaveOccurred()) + Expect(st.Mode().Perm()).To(Equal(os.FileMode(0700))) + Expect(os.Chmod(path, 0644)).To(Succeed()) + _, err = Read(path, generation) + Expect(err).To(HaveOccurred()) + Expect(Write(path, s)).NotTo(Succeed()) + Expect(os.Remove(path)).To(Succeed()) + Expect(os.Symlink("missing", path)).To(Succeed()) + Expect(Write(path, s)).NotTo(Succeed()) + _, err = Read(path, generation) + Expect(err).To(HaveOccurred()) + }) + It("requires all evidence and sequence progress, not repeated reads or early readyz", func() { + now := time.Now() + t := Tracker{ExpectedPID: os.Getpid(), Generation: "session", Started: now} + s := Snapshot{PID: os.Getpid(), Schema: 1, Generation: "session", Sequence: 1, Time: now, Registration: "healthy", RegistrationTime: now, HeartbeatInterval: 10 * time.Second, LocalReady: true} + Expect(t.Classify(s, now)).NotTo(Equal("ready")) + s.Credentials = true + s.Connected = true + s.Authenticated = true + s.Subscriptions = true + s.Armed = true + s.Heartbeat = now + s.Sequence++ + Expect(t.Classify(s, now)).To(Equal("ready")) + Expect(t.Classify(s, now.Add(10*time.Second))).To(Equal("status_unavailable")) + s.Sequence++ + s.Time = now.Add(11 * time.Second) + s.Heartbeat = now.Add(-31 * time.Second) + Expect(t.Classify(s, now.Add(11*time.Second))).To(Equal("heartbeat")) + s.Sequence++ + s.Registration = "pending" + s.RegistrationTime = now.Add(35 * time.Second) + s.Time = s.RegistrationTime + Expect(t.Classify(s, now.Add(35*time.Second))).To(Equal("pending")) + s.Generation = "old" + Expect(t.Classify(s, now)).To(Equal("status_unavailable")) + }) +}) + +var _ = Describe("Worker status security", func() { + It("rejects malformed, oversized, future-schema, linked and unsafe-directory evidence", func() { + path, generation, err := NewSession(GinkgoT().TempDir()) + Expect(err).NotTo(HaveOccurred()) + for _, data := range []string{"{", `{"schema_version":99}`, string(make([]byte, MaxSize+1)), `{} {}`} { + Expect(os.WriteFile(path, []byte(data), 0600)).To(Succeed()) + _, err = Read(path, generation) + Expect(err).To(HaveOccurred()) + } + Expect(os.Remove(path)).To(Succeed()) + target := filepath.Join(filepath.Dir(path), "other") + Expect(os.WriteFile(target, []byte("{}"), 0600)).To(Succeed()) + Expect(os.Link(target, path)).To(Succeed()) + _, err = Read(path, generation) + Expect(err).To(HaveOccurred()) + Expect(os.Remove(path)).To(Succeed()) + Expect(os.Chmod(filepath.Dir(path), 0755)).To(Succeed()) + _, err = Read(path, generation) + Expect(err).To(HaveOccurred()) + }) + It("fails closed on publication failure and never adopts an existing session", func() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + path, generation, err := NewSession(GinkgoT().TempDir()) + Expect(err).NotTo(HaveOccurred()) + p, err := NewPublisher(ctx, path, generation, cancel) + Expect(err).NotTo(HaveOccurred()) + _, err = NewPublisher(ctx, path, generation, cancel) + Expect(err).To(HaveOccurred()) + p.Registration("healthy", true, nil) + p.Arm(func() (bool, bool, bool, bool) { return true, true, true, true }) + p.Heartbeat(nil) + s, err := Read(path, generation) + Expect(err).NotTo(HaveOccurred()) + Expect(s.Armed).To(BeTrue()) + Expect(os.Chmod(path, 0644)).To(Succeed()) + p.Update(nil) + Eventually(ctx.Done()).Should(BeClosed()) + _, err = Read(path, generation) + Expect(err).To(HaveOccurred()) + p.Update(nil) + Expect(os.IsNotExist(func() error { _, e := os.Stat(path); return e }())).To(BeTrue()) + }) + It("refuses a new publisher over a dangling symlink or an untrusted directory", func() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + path, generation, err := NewSession(GinkgoT().TempDir()) + Expect(err).NotTo(HaveOccurred()) + Expect(os.Symlink(filepath.Join(filepath.Dir(path), "missing-target"), path)).To(Succeed()) + _, err = NewPublisher(ctx, path, generation, cancel) + Expect(err).To(HaveOccurred()) + Expect(os.Remove(path)).To(Succeed()) + Expect(os.Chmod(filepath.Dir(path), 0755)).To(Succeed()) + _, err = NewPublisher(ctx, path, generation, cancel) + Expect(err).To(HaveOccurred()) + _, err = NewPublisher(ctx, "status.json", generation, cancel) + Expect(err).To(HaveOccurred()) + }) + It("preserves explicit approval independently of credentials and expires pending evidence", func() { + now := time.Now() + s := Snapshot{PID: os.Getpid(), Schema: 1, Generation: "test", Sequence: 1, Time: now, Registration: "unknown", Credentials: true, Authenticated: true, Connected: true, Subscriptions: true, Armed: true, LocalReady: true, Heartbeat: now, HeartbeatInterval: 10 * time.Second} + tracker := Tracker{ExpectedPID: os.Getpid(), Generation: "test", Started: now} + Expect(tracker.Classify(s, now)).To(Equal("registration")) + s.Sequence++ + s.Registration = "pending" + s.RegistrationTime = now.Add(-31 * time.Second) + Expect(tracker.Classify(s, now)).To(Equal("registration")) + s.Sequence++ + s.Registration = "starting" + s.Time = now.Add(31 * time.Second) + Expect(tracker.Classify(s, now.Add(31*time.Second))).To(Equal("registration")) + s.Sequence++ + s.Registration = "healthy" + s.Credentials = false + s.RegistrationTime = s.Time + Expect(tracker.Classify(s, s.Time)).To(Equal("credentials")) + s.Sequence-- + Expect(tracker.Classify(s, now)).To(Equal("status_unavailable")) + }) +}) + +var _ = Describe("Worker status atomicity", func() { + It("serializes concurrent transitions and keeps complete snapshots", func() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + path, generation, err := NewSession(GinkgoT().TempDir()) + Expect(err).NotTo(HaveOccurred()) + p, err := NewPublisher(ctx, path, generation, cancel) + Expect(err).NotTo(HaveOccurred()) + var wg sync.WaitGroup + for i := 0; i < 4; i++ { + wg.Go(func() { + defer GinkgoRecover() + for j := 0; j < 20; j++ { + p.Update(nil) + s, e := Read(path, generation) + Expect(e).NotTo(HaveOccurred()) + Expect(s.Generation).To(Equal(generation)) + } + }) + } + wg.Wait() + s, err := Read(path, generation) + Expect(err).NotTo(HaveOccurred()) + Expect(s.Sequence).To(BeNumerically(">=", 81)) + }) + It("rejects a symlink run directory and owner mismatch where supported", func() { + root := GinkgoT().TempDir() + path, generation, err := NewSession(root) + Expect(err).NotTo(HaveOccurred()) + link := filepath.Join(root, "link") + Expect(os.Symlink(filepath.Dir(path), link)).To(Succeed()) + _, err = Read(filepath.Join(link, "status.json"), generation) + Expect(err).To(HaveOccurred()) + if os.Geteuid() == 0 { + s := Snapshot{PID: os.Getpid(), Schema: 1, Generation: generation, Sequence: 1, Time: time.Now(), Registration: "starting", HeartbeatInterval: time.Second} + Expect(Write(path, s)).To(Succeed()) + Expect(os.Chown(path, 65534, 65534)).To(Succeed()) + _, err = Read(path, generation) + Expect(err).To(HaveOccurred()) + Expect(Write(path, s)).NotTo(Succeed()) + } + }) +}) diff --git a/swagger/docs.go b/swagger/docs.go index 2880cf1f45e1..aa6e53241317 100644 --- a/swagger/docs.go +++ b/swagger/docs.go @@ -1607,6 +1607,115 @@ const docTemplate = `{ } } }, + "/api/node/register": { + "post": { + "description": "Supply renew_node_id and the configured registration token to renew credentials without changing approval or telemetry. Renewal requires authentication and only healthy nodes receive credentials.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "nodes" + ], + "summary": "Register a worker or renew its credentials", + "parameters": [ + { + "description": "Registration or credential renewal", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/localai.RegisterNodeRequest" + } + } + ], + "responses": { + "200": { + "description": "Node ID, status, and credentials when available", + "schema": { + "type": "object", + "additionalProperties": true + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/schema.ErrorResponse" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/schema.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/schema.ErrorResponse" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/schema.ErrorResponse" + } + } + } + } + }, + "/api/node/{id}/status": { + "get": { + "description": "Read approval without changing registration, heartbeat, or credentials. Requires the configured registration token when authentication is enabled.", + "produces": [ + "application/json" + ], + "tags": [ + "nodes" + ], + "summary": "Observe worker approval", + "parameters": [ + { + "type": "string", + "description": "Node ID", + "name": "id", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Bearer registration token", + "name": "Authorization", + "in": "header" + } + ], + "responses": { + "200": { + "description": "Node ID and current status", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/schema.ErrorResponse" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/schema.ErrorResponse" + } + } + } + } + }, "/api/nodes/models": { "get": { "tags": [ @@ -5510,6 +5619,92 @@ const docTemplate = `{ } } }, + "localai.RegisterNodeRequest": { + "type": "object", + "properties": { + "address": { + "type": "string" + }, + "available_disk": { + "type": "integer" + }, + "available_ram": { + "type": "integer" + }, + "available_vram": { + "type": "integer" + }, + "capability": { + "description": "Capability is the worker's own meta-backend capability string. The\ncontroller stores it so backend discovery can reflect what the cluster\ncan run rather than what the (typically GPU-less) controller can run.", + "type": "string" + }, + "commit": { + "description": "Commit is the git commit hash the worker binary was built from.", + "type": "string" + }, + "cpu_load_1": { + "type": "number" + }, + "cpu_logical_cores": { + "type": "integer" + }, + "cpu_usage_percent": { + "type": "number" + }, + "gpu_compute_capability": { + "description": "GPUComputeCapability is the worker GPU's compute capability (\"major.minor\",\ne.g. \"12.1\" for GB10). Used by the router for per-arch option tuning.", + "type": "string" + }, + "gpu_vendor": { + "type": "string" + }, + "http_address": { + "type": "string" + }, + "labels": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "max_replicas_per_model": { + "description": "MaxReplicasPerModel is the per-node cap on replicas of any single model.\nWorkers older than this field omit it; we coerce 0 → 1 below to preserve\nhistorical single-replica behavior.", + "type": "integer" + }, + "name": { + "type": "string" + }, + "node_type": { + "description": "\"backend\" (default) or \"agent\"", + "type": "string" + }, + "renew_node_id": { + "description": "RenewNodeID renews credentials without restoring status or mutating registration.", + "type": "string" + }, + "token": { + "type": "string" + }, + "total_disk": { + "description": "TotalDisk / AvailableDisk describe the filesystem backing the worker's\nMODELS directory (where staged weights land), not the root filesystem.\nOmitted by workers that predate the fields; the scheduler treats\ntotal_disk == 0 as \"unknown\" and leaves such a node in rotation.", + "type": "integer" + }, + "total_ram": { + "type": "integer" + }, + "total_vram": { + "type": "integer" + }, + "version": { + "description": "Version is the LocalAI build version reported by the worker at\nregistration. Empty for workers registered before this field existed.", + "type": "string" + }, + "vram_budget": { + "description": "VRAMBudget is the worker's operator-set VRAM cap (\"80%\" or \"12GB\"). The\nregistry resolves and enforces it against the raw reported VRAM.", + "type": "string" + } + } + }, "localai.TTSModelVoices": { "type": "object", "properties": { diff --git a/swagger/swagger.json b/swagger/swagger.json index b5825d5fe1f9..823bce7bc942 100644 --- a/swagger/swagger.json +++ b/swagger/swagger.json @@ -1604,6 +1604,115 @@ } } }, + "/api/node/register": { + "post": { + "description": "Supply renew_node_id and the configured registration token to renew credentials without changing approval or telemetry. Renewal requires authentication and only healthy nodes receive credentials.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "nodes" + ], + "summary": "Register a worker or renew its credentials", + "parameters": [ + { + "description": "Registration or credential renewal", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/localai.RegisterNodeRequest" + } + } + ], + "responses": { + "200": { + "description": "Node ID, status, and credentials when available", + "schema": { + "type": "object", + "additionalProperties": true + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/schema.ErrorResponse" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/schema.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/schema.ErrorResponse" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/schema.ErrorResponse" + } + } + } + } + }, + "/api/node/{id}/status": { + "get": { + "description": "Read approval without changing registration, heartbeat, or credentials. Requires the configured registration token when authentication is enabled.", + "produces": [ + "application/json" + ], + "tags": [ + "nodes" + ], + "summary": "Observe worker approval", + "parameters": [ + { + "type": "string", + "description": "Node ID", + "name": "id", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Bearer registration token", + "name": "Authorization", + "in": "header" + } + ], + "responses": { + "200": { + "description": "Node ID and current status", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/schema.ErrorResponse" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/schema.ErrorResponse" + } + } + } + } + }, "/api/nodes/models": { "get": { "tags": [ @@ -5507,6 +5616,92 @@ } } }, + "localai.RegisterNodeRequest": { + "type": "object", + "properties": { + "address": { + "type": "string" + }, + "available_disk": { + "type": "integer" + }, + "available_ram": { + "type": "integer" + }, + "available_vram": { + "type": "integer" + }, + "capability": { + "description": "Capability is the worker's own meta-backend capability string. The\ncontroller stores it so backend discovery can reflect what the cluster\ncan run rather than what the (typically GPU-less) controller can run.", + "type": "string" + }, + "commit": { + "description": "Commit is the git commit hash the worker binary was built from.", + "type": "string" + }, + "cpu_load_1": { + "type": "number" + }, + "cpu_logical_cores": { + "type": "integer" + }, + "cpu_usage_percent": { + "type": "number" + }, + "gpu_compute_capability": { + "description": "GPUComputeCapability is the worker GPU's compute capability (\"major.minor\",\ne.g. \"12.1\" for GB10). Used by the router for per-arch option tuning.", + "type": "string" + }, + "gpu_vendor": { + "type": "string" + }, + "http_address": { + "type": "string" + }, + "labels": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "max_replicas_per_model": { + "description": "MaxReplicasPerModel is the per-node cap on replicas of any single model.\nWorkers older than this field omit it; we coerce 0 → 1 below to preserve\nhistorical single-replica behavior.", + "type": "integer" + }, + "name": { + "type": "string" + }, + "node_type": { + "description": "\"backend\" (default) or \"agent\"", + "type": "string" + }, + "renew_node_id": { + "description": "RenewNodeID renews credentials without restoring status or mutating registration.", + "type": "string" + }, + "token": { + "type": "string" + }, + "total_disk": { + "description": "TotalDisk / AvailableDisk describe the filesystem backing the worker's\nMODELS directory (where staged weights land), not the root filesystem.\nOmitted by workers that predate the fields; the scheduler treats\ntotal_disk == 0 as \"unknown\" and leaves such a node in rotation.", + "type": "integer" + }, + "total_ram": { + "type": "integer" + }, + "total_vram": { + "type": "integer" + }, + "version": { + "description": "Version is the LocalAI build version reported by the worker at\nregistration. Empty for workers registered before this field existed.", + "type": "string" + }, + "vram_budget": { + "description": "VRAMBudget is the worker's operator-set VRAM cap (\"80%\" or \"12GB\"). The\nregistry resolves and enforces it against the raw reported VRAM.", + "type": "string" + } + } + }, "localai.TTSModelVoices": { "type": "object", "properties": { diff --git a/swagger/swagger.yaml b/swagger/swagger.yaml index 623af08fe872..dacbd9ce06a8 100644 --- a/swagger/swagger.yaml +++ b/swagger/swagger.yaml @@ -619,6 +619,83 @@ definitions: origin: type: string type: object + localai.RegisterNodeRequest: + properties: + address: + type: string + available_disk: + type: integer + available_ram: + type: integer + available_vram: + type: integer + capability: + description: |- + Capability is the worker's own meta-backend capability string. The + controller stores it so backend discovery can reflect what the cluster + can run rather than what the (typically GPU-less) controller can run. + type: string + commit: + description: Commit is the git commit hash the worker binary was built from. + type: string + cpu_load_1: + type: number + cpu_logical_cores: + type: integer + cpu_usage_percent: + type: number + gpu_compute_capability: + description: |- + GPUComputeCapability is the worker GPU's compute capability ("major.minor", + e.g. "12.1" for GB10). Used by the router for per-arch option tuning. + type: string + gpu_vendor: + type: string + http_address: + type: string + labels: + additionalProperties: + type: string + type: object + max_replicas_per_model: + description: |- + MaxReplicasPerModel is the per-node cap on replicas of any single model. + Workers older than this field omit it; we coerce 0 → 1 below to preserve + historical single-replica behavior. + type: integer + name: + type: string + node_type: + description: '"backend" (default) or "agent"' + type: string + renew_node_id: + description: RenewNodeID renews credentials without restoring status or mutating + registration. + type: string + token: + type: string + total_disk: + description: |- + TotalDisk / AvailableDisk describe the filesystem backing the worker's + MODELS directory (where staged weights land), not the root filesystem. + Omitted by workers that predate the fields; the scheduler treats + total_disk == 0 as "unknown" and leaves such a node in rotation. + type: integer + total_ram: + type: integer + total_vram: + type: integer + version: + description: |- + Version is the LocalAI build version reported by the worker at + registration. Empty for workers registered before this field existed. + type: string + vram_budget: + description: |- + VRAMBudget is the worker's operator-set VRAM cap ("80%" or "12GB"). The + registry resolves and enforces it against the raw reported VRAM. + type: string + type: object localai.TTSModelVoices: properties: model: @@ -4824,6 +4901,81 @@ paths: summary: Issue a motion WebSocket ticket (30 seconds, single use) tags: - motion + /api/node/{id}/status: + get: + description: Read approval without changing registration, heartbeat, or credentials. + Requires the configured registration token when authentication is enabled. + parameters: + - description: Node ID + in: path + name: id + required: true + type: string + - description: Bearer registration token + in: header + name: Authorization + type: string + produces: + - application/json + responses: + "200": + description: Node ID and current status + schema: + additionalProperties: + type: string + type: object + "401": + description: Unauthorized + schema: + $ref: '#/definitions/schema.ErrorResponse' + "404": + description: Not Found + schema: + $ref: '#/definitions/schema.ErrorResponse' + summary: Observe worker approval + tags: + - nodes + /api/node/register: + post: + consumes: + - application/json + description: Supply renew_node_id and the configured registration token to renew + credentials without changing approval or telemetry. Renewal requires authentication + and only healthy nodes receive credentials. + parameters: + - description: Registration or credential renewal + in: body + name: request + required: true + schema: + $ref: '#/definitions/localai.RegisterNodeRequest' + produces: + - application/json + responses: + "200": + description: Node ID, status, and credentials when available + schema: + additionalProperties: true + type: object + "400": + description: Bad Request + schema: + $ref: '#/definitions/schema.ErrorResponse' + "401": + description: Unauthorized + schema: + $ref: '#/definitions/schema.ErrorResponse' + "403": + description: Forbidden + schema: + $ref: '#/definitions/schema.ErrorResponse' + "404": + description: Not Found + schema: + $ref: '#/definitions/schema.ErrorResponse' + summary: Register a worker or renew its credentials + tags: + - nodes /api/nodes/{id}/max-replicas-per-model: delete: parameters: