diff --git a/addons/fine-tuning/ui/src/FineTune.jsx b/addons/fine-tuning/ui/src/FineTune.jsx index d4f3b207dbdc..364402a8d548 100644 --- a/addons/fine-tuning/ui/src/FineTune.jsx +++ b/addons/fine-tuning/ui/src/FineTune.jsx @@ -146,7 +146,7 @@ export function createFineTune(ctx, api, shared) { // events while the job is active; a finished job is read from the job record. // `seed` starts the view with an event and checkpoints already read; tests use // it to draw a state the stream would reach. - function JobView({ job, onStop, onReuse, onResume, onTerminal, exportCheckpoint, seed }) { + function JobView({ job, onStop, onReuse, onResume, onTerminal, exportCheckpoint, seed, modelExports = true }) { const isAdmin = viewer.isAdmin() const [events, setEvents] = useState([]) const [latest, setLatest] = useState(seed?.latest ?? null) @@ -339,7 +339,7 @@ export function createFineTune(ctx, api, shared) {
{isAdmin && ( - + )}
@@ -355,7 +355,7 @@ export function createFineTune(ctx, api, shared) { } // What to do with a finished job: export it as a model, then use it. - function ExportForm({ job, prefilledCheckpoint, seed }) { + function ExportForm({ job, prefilledCheckpoint, seed, modelExports = true }) { const [checkpoints, setCheckpoints] = useState(seed?.checkpoints || []) const [exportFormat, setExportFormat] = useState('lora') const [quantMethod, setQuantMethod] = useState('q4_k_m') @@ -484,10 +484,11 @@ export function createFineTune(ctx, api, shared) { setModelName(e.target.value)} placeholder="my-finetuned-model" />
-
+ {!modelExports &&

{t('export.blocked')}

} {message && (!exportedModelName || exportFailed) && (

{exporting && } {message} @@ -501,7 +502,7 @@ export function createFineTune(ctx, api, shared) { // The wrapper of the export controls. Registering a model is for // administrators; anyone else sees why the controls are missing. - function ExportPanel({ job, prefilledCheckpoint, seed }) { + function ExportPanel({ job, prefilledCheckpoint, seed, modelExports = true }) { if (!job || !TERMINAL.includes(job.status)) return null if (!viewer.isAdmin()) { return ( @@ -511,7 +512,19 @@ export function createFineTune(ctx, api, shared) { ) } - return + return + } + + // Whether this instance may write an exported model. Only a clear false turns + // the controls off: a failed or slow answer leaves them on, as the server does. + function useModelExports() { + const [allowed, setAllowed] = useState(true) + useEffect(() => { + let off = false + api.fineTune.status().then(r => { if (!off && r && r.model_exports === false) setAllowed(false) }).catch(() => {}) + return () => { off = true } + }, []) + return allowed } // The machine facts for the checks: the one request useResources makes, as @@ -538,6 +551,7 @@ export function createFineTune(ctx, api, shared) { function FineTuneBody({ path }) { const isAdmin = viewer.isAdmin() const resources = useResources(10000) + const modelExports = useModelExports() const facts = useMemo(() => machineFacts(resources), [resources]) const [jobs, setJobs] = useState([]) @@ -951,8 +965,9 @@ export function createFineTune(ctx, api, shared) { onResume={handleResumeFromCheckpoint} onTerminal={loadJobs} exportCheckpoint={setExportCheckpoint} + modelExports={modelExports} /> - + ) : (

diff --git a/addons/fine-tuning/ui/src/api.js b/addons/fine-tuning/ui/src/api.js index d6d081809612..795cee902764 100644 --- a/addons/fine-tuning/ui/src/api.js +++ b/addons/fine-tuning/ui/src/api.js @@ -9,6 +9,8 @@ export function createApi(ctx) { return { fineTune: { + // {model_exports: bool}: false when this instance cannot share the models path of the owner. + status: () => json('/api/fine-tuning/status'), listBackends: () => json('/api/fine-tuning/backends'), startJob: (data) => post('/api/fine-tuning/jobs', data), listJobs: () => json('/api/fine-tuning/jobs'), diff --git a/addons/fine-tuning/ui/src/i18n/en.js b/addons/fine-tuning/ui/src/i18n/en.js index 9a69bb7123af..34b977f86d1b 100644 --- a/addons/fine-tuning/ui/src/i18n/en.js +++ b/addons/fine-tuning/ui/src/i18n/en.js @@ -244,6 +244,7 @@ export default { "checkpoints.resumeTitle": "Resume training from this checkpoint", "checkpoints.export": "Export", "checkpoints.exportTitle": "Export this checkpoint", + "export.blocked": "Export is off on this server because it does not share the models folder of the add-on host.", "export.title": "Export as a model", "export.lede": "Exporting turns the result into a model LocalAI can load.", "export.checkpoint": "Checkpoint", diff --git a/addons/fine-tuning/ui/test/page.test.mjs b/addons/fine-tuning/ui/test/page.test.mjs index 29554797f3c4..b376d754f222 100644 --- a/addons/fine-tuning/ui/test/page.test.mjs +++ b/addons/fine-tuning/ui/test/page.test.mjs @@ -44,6 +44,7 @@ test('page: the API helper uses the core fine-tuning paths and bodies', async () json: async (p, i) => { seen.push([p, i]); return {} }, } const api = createApi(ctx) + await api.fineTune.status() await api.fineTune.listBackends() await api.fineTune.listJobs() await api.fineTune.getJob('a b') @@ -56,13 +57,13 @@ test('page: the API helper uses the core fine-tuning paths and bodies', async () await api.resources.get() const paths = seen.map(s => `${s[1]?.method || 'GET'} ${s[0]}`) assert.deepEqual(paths, [ - 'GET /api/fine-tuning/backends', 'GET /api/fine-tuning/jobs', 'GET /api/fine-tuning/jobs/a%20b', + 'GET /api/fine-tuning/status', 'GET /api/fine-tuning/backends', 'GET /api/fine-tuning/jobs', 'GET /api/fine-tuning/jobs/a%20b', 'POST /api/fine-tuning/jobs', 'POST /api/fine-tuning/jobs/j1/stop?save_checkpoint=true', 'POST /api/fine-tuning/jobs/j1/stop?save_checkpoint=false', 'DELETE /api/fine-tuning/jobs/j1', 'GET /api/fine-tuning/jobs/j1/checkpoints', 'POST /api/fine-tuning/jobs/j1/export', 'GET /api/resources', ]) - assert.equal(seen[3][1].body, JSON.stringify({ model: 'm' })) - assert.equal(seen[3][1].headers['Content-Type'], 'application/json') + assert.equal(seen[4][1].body, JSON.stringify({ model: 'm' })) + assert.equal(seen[4][1].headers['Content-Type'], 'application/json') }) test('page: the dataset upload is a FormData body with no Content-Type header', async () => { @@ -222,7 +223,7 @@ test('i18n: en.js repeats the host strings and adds only the keys the add-on own if (k in tools) assert.equal(v, tools[k], k) else own.push(k) } - assert.deepEqual(own.sort(), ['export.adminOnly.text', 'export.adminOnly.title', 'run.unknownPercent']) + assert.deepEqual(own.sort(), ['export.adminOnly.text', 'export.adminOnly.title', 'export.blocked', 'run.unknownPercent']) assert.equal(Object.keys(en).some(k => k.startsWith('quantize.')), false) }) @@ -238,3 +239,23 @@ test('source: no file imports a core UI file or the router', () => { assert.doesNotMatch(text, /\bEventSource\b/, f) } }) + +test('export: the export controls are off with the reason when the models probe fails', async () => { + const done = job({ status: 'completed' }) + const seed = { checkpoints: [{ path: '/srv/c1', step: 10, loss: 0.5, epoch: 1 }] } + const { mod, sdk } = await load() + const { ExportPanel, JobView } = (await mod.__partsFor(sdk)).parts + const blocked = renderToString(React.createElement(ExportPanel, { job: done, seed, modelExports: false })) + assert.match(blocked, /data-testid="export-blocked"/) + assert.match(blocked, new RegExp(en['export.blocked'].slice(0, 30))) + assert.match(blocked, /]*disabled=""[^>]*>[^]*?<\/button>/) + const open = renderToString(React.createElement(ExportPanel, { job: done, seed })) + assert.doesNotMatch(open, /data-testid="export-blocked"/) + const props = { job: done, seed, onStop() {}, onReuse() {}, onResume() {}, onTerminal() {}, exportCheckpoint() {} } + const off = renderToString(React.createElement(JobView, { ...props, modelExports: false })) + const on = renderToString(React.createElement(JobView, props)) + const exportBtn = (html) => html.match(/]*title="[^"]*"[^>]*> setModelName(e.target.value)} />
- {t('quantize.download')}
+ {!modelExports &&

{t('quantize.importBlocked')}

} )} @@ -148,7 +149,7 @@ export function createQuantize(ctx, api, shared) { // the result. The server streams progress events while the job is active; a // finished job is read from the job record. `seed` starts the view with an // event already read; tests use it to draw a state the stream would reach. - function JobView({ job, onStop, onReuse, onTerminal, onRefresh, seed }) { + function JobView({ job, onStop, onReuse, onTerminal, onRefresh, seed, modelExports = true }) { const [latest, setLatest] = useState(seed?.latest ?? null) const [log, setLog] = useState([]) const previousRef = useRef(null) @@ -253,11 +254,23 @@ export function createQuantize(ctx, api, shared) { )} {(active || log.length > 0) && } - + ) } + // Whether this instance may write an imported model. Only a clear false turns + // the controls off: a failed or slow answer leaves them on, as the server does. + function useModelExports() { + const [allowed, setAllowed] = useState(true) + useEffect(() => { + let off = false + api.quantize.status().then(r => { if (!off && r && r.model_exports === false) setAllowed(false) }).catch(() => {}) + return () => { off = true } + }, []) + return allowed + } + // The machine facts for the checks: the one request the page makes, as a // poll that waits while the tab is hidden and catches up on return. function useResources(interval) { @@ -282,6 +295,7 @@ export function createQuantize(ctx, api, shared) { function QuantizeBody({ path }) { const isAdmin = viewer.isAdmin() const resources = useResources(10000) + const modelExports = useModelExports() const facts = useMemo(() => machineFacts(resources), [resources]) // A deep link names the model: ?model= or the first path segment. @@ -435,7 +449,7 @@ export function createQuantize(ctx, api, shared) { )} {selectedJob ? ( - + ) : (
diff --git a/addons/quantization/ui/src/api.js b/addons/quantization/ui/src/api.js index baaf1d9a8386..455739d027d6 100644 --- a/addons/quantization/ui/src/api.js +++ b/addons/quantization/ui/src/api.js @@ -9,6 +9,8 @@ export function createApi(ctx) { return { quantize: { + // {model_exports: bool}: false when this instance cannot share the models path of the owner. + status: () => json('/api/quantization/status'), listBackends: () => json('/api/quantization/backends'), startJob: (data) => post('/api/quantization/jobs', data), listJobs: () => json('/api/quantization/jobs'), diff --git a/addons/quantization/ui/src/i18n/en.js b/addons/quantization/ui/src/i18n/en.js index 18afe26faf35..f089fe8a94fb 100644 --- a/addons/quantization/ui/src/i18n/en.js +++ b/addons/quantization/ui/src/i18n/en.js @@ -146,6 +146,7 @@ export default { "quantize.file": "File", "quantize.importName": "Model name", "quantize.importNamePlaceholder": "Generated if empty", + "quantize.importBlocked": "Import is off on this server because it does not share the models folder of the add-on host.", "quantize.import": "Import to LocalAI", "quantize.importing": "Importing...", "quantize.importFailed": "Import failed", diff --git a/addons/quantization/ui/test/page.test.mjs b/addons/quantization/ui/test/page.test.mjs index ec0a8c8851e6..3f4031efa920 100644 --- a/addons/quantization/ui/test/page.test.mjs +++ b/addons/quantization/ui/test/page.test.mjs @@ -48,6 +48,7 @@ test('page: the API helper uses the core quantization paths and bodies', async ( const seen = [] const ctx = { core: { fetch: async (p, i) => { seen.push([p, i]); return { ok: true } } }, json: async (p, i) => { seen.push([p, i]); return {} } } const api = createApi(ctx) + await api.quantize.status() await api.quantize.listBackends() await api.quantize.listJobs() await api.quantize.getJob('a b') @@ -58,13 +59,13 @@ test('page: the API helper uses the core quantization paths and bodies', async ( await api.resources.get() const paths = seen.map(s => `${s[1]?.method || 'GET'} ${s[0]}`) assert.deepEqual(paths, [ - 'GET /api/quantization/backends', 'GET /api/quantization/jobs', 'GET /api/quantization/jobs/a%20b', + 'GET /api/quantization/status', 'GET /api/quantization/backends', 'GET /api/quantization/jobs', 'GET /api/quantization/jobs/a%20b', 'POST /api/quantization/jobs', 'POST /api/quantization/jobs/j1/stop', 'DELETE /api/quantization/jobs/j1', 'POST /api/quantization/jobs/j1/import', 'GET /api/resources', ]) - assert.equal(seen[3][1].body, JSON.stringify({ model: 'm' })) - assert.equal(seen[3][1].headers['Content-Type'], 'application/json') - assert.equal(seen[6][1].body, JSON.stringify({ name: 'n' })) + assert.equal(seen[4][1].body, JSON.stringify({ model: 'm' })) + assert.equal(seen[4][1].headers['Content-Type'], 'application/json') + assert.equal(seen[7][1].body, JSON.stringify({ name: 'n' })) }) test('page: a failed call throws the server message', async () => { @@ -194,7 +195,7 @@ test('i18n: en.js repeats the host strings and adds only the keys the add-on own if (k in tools) assert.equal(v, tools[k], k) else own.push(k) } - assert.deepEqual(own.sort(), ['run.unknownPercent']) + assert.deepEqual(own.sort(), ['quantize.importBlocked', 'run.unknownPercent']) assert.equal(Object.keys(en).some(k => k.startsWith('rewards.') || k.startsWith('checkpoints.') || k.startsWith('chart.')), false) }) @@ -210,3 +211,16 @@ test('source: no file imports a core UI file or the router', () => { assert.doesNotMatch(text, /\bEventSource\b/, f) } }) + +test('import: the import control is off with the reason when the models probe fails', async () => { + const { mod, sdk } = await load() + const { ResultPanel } = (await mod.__partsFor(sdk)).parts + const done = job({ status: 'completed', output_file: 'model-q4.gguf' }) + const blocked = renderToString(React.createElement(ResultPanel, { job: done, onRefresh: async () => {}, modelExports: false })) + assert.match(blocked, /]*disabled=""[^>]*data-testid="quantize-import"/) + assert.match(blocked, /data-testid="quantize-import-blocked"/) + assert.match(blocked, new RegExp(en['quantize.importBlocked'].slice(0, 30))) + const open = renderToString(React.createElement(ResultPanel, { job: done, onRefresh: async () => {} })) + assert.doesNotMatch(open, /quantize-import-blocked/) + assert.doesNotMatch(open, /]*disabled=""[^>]*data-testid="quantize-import"/) +}) diff --git a/core/application/addon_gateway.go b/core/application/addon_gateway.go index 1f3e4fc6e48d..481c243387ee 100644 --- a/core/application/addon_gateway.go +++ b/core/application/addon_gateway.go @@ -15,8 +15,10 @@ import ( mcpTools "github.com/mudler/LocalAI/core/http/endpoints/mcp" "github.com/mudler/LocalAI/core/services/addons" "github.com/mudler/LocalAI/core/services/addons/artifacts" + "github.com/mudler/LocalAI/core/services/addons/backendlib" "github.com/mudler/LocalAI/core/services/addons/gateway" "github.com/mudler/LocalAI/core/services/addons/hostops" + "github.com/mudler/LocalAI/core/services/addons/modelsprobe" "github.com/mudler/LocalAI/core/services/addons/policy" "github.com/mudler/LocalAI/core/services/addons/resources" addonruntime "github.com/mudler/LocalAI/core/services/addons/runtime" @@ -93,6 +95,15 @@ func (a *Application) RegisterAddonEngineModels(ex hostops.Exporter, im hostops. if a.addonHostOps == nil { return nil } + // The owner writes the exports, so it refuses them while its models path + // does not hold its probe. + ok := a.ownerProbeCurrent + if ex != nil { + ex = guardedExporter{Exporter: ex, ok: ok} + } + if im != nil { + im = guardedImporter{Importer: im, ok: ok} + } return hostops.RegisterEngineModels(a.addonHostOps.Broker, ex, im) } @@ -209,6 +220,14 @@ func (a *Application) startAddonGateway() error { if cfg.InstanceID != cfg.OwnerID { g.Forward = carrier a.addonGateway = g + // The frontend installs backends on its own node and takes the + // resolved plan from the owner. + a.backendLibrary = &backendlib.Installer{ + Remote: remoteBackends{a}, SystemState: a.applicationConfig.SystemState, ModelLoader: a.modelLoader, + MaxBytes: cfg.BackendMaxBytes, RequireIntegrity: a.applicationConfig.RequireBackendIntegrity, + } + go a.runLibraryRefresh(a.applicationConfig.Context) + go a.runModelsProbe(a.applicationConfig.Context) return nil } // Only the owner holds the policy store. Other frontends reach it through the @@ -301,6 +320,19 @@ func (a *Application) startAddonGateway() error { } manage := resources.New(resourceOpts) g.Resources = manage + // The owner serves the library calls of the frontends. A node with no + // source store (no catalog) has nothing to serve. + if a.addonSources != nil { + g.Libraries = a.newLibraryOwner() + g.OS, g.Arch = cfg.Host.OS, cfg.Host.Arch + } + g.ModelsProbe = func() (modelsprobe.Probe, bool) { + p := a.modelsProbe.own.Load() + if p == nil { + return modelsprobe.Probe{}, false + } + return *p, true + } loop := policy.NewReconcileLoop(p, exists, a.addonReconcileTick) loop.Resume = manage.Resume loop.Report = func(r policy.ReconcileReport, err error) { diff --git a/core/application/addons_feature.go b/core/application/addons_feature.go new file mode 100644 index 000000000000..582c86e3ef61 --- /dev/null +++ b/core/application/addons_feature.go @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +package application + +import ( + "context" + "sync/atomic" + "time" +) + +// addonsReachableTTL is how long a frontend keeps its answer about the owner. +// It keeps /api/features cheap: the owner is not dialled for every request. +const addonsReachableTTL = 10 * time.Second + +// addonsProbe is one cached answer of a frontend about the owner. +type addonsProbe struct { + reachable bool + at time.Time +} + +// addonsProbeState is per-instance state: two frontends can disagree for up to +// addonsReachableTTL after the owner goes up or down (mode 4 of +// .agents/distributed-state.md). Two requests that probe at the same time both +// store an answer; the last one wins, which is harmless. +type addonsProbeState struct { + last atomic.Pointer[addonsProbe] + // now and ttl are set by tests only. + now func() time.Time + ttl time.Duration +} + +func (s *addonsProbeState) clock() time.Time { + if s.now != nil { + return s.now() + } + return time.Now() +} + +func (s *addonsProbeState) lifetime() time.Duration { + if s.ttl > 0 { + return s.ttl + } + return addonsReachableTTL +} + +// AddonsEnabled reports the instance switch alone, so a refusal can tell a +// disabled feature from an owner that does not answer. +func (a *Application) AddonsEnabled() bool { return a.applicationConfig.Addons.Enabled } + +// AddonsFeature is the one place that decides whether the add-on surfaces are +// on for this instance. The routes and the UI read it; nothing else checks +// Addons.Enabled for this purpose. +// +// A combined or owner process answers true when add-ons are enabled and it +// holds the owner manager. A frontend answers true when add-ons are enabled +// and the owner answered the carrier's audience call, cached for ten seconds. +func (a *Application) AddonsFeature(ctx context.Context) bool { + cfg := a.applicationConfig.Addons + if !cfg.Enabled || a.addonManager == nil { + return false + } + if cfg.InstanceID == cfg.OwnerID { + return true + } + g := a.addonGateway + if g == nil || g.Forward == nil { + return false + } + s := &a.addonsProbe + if p := s.last.Load(); p != nil && s.clock().Sub(p.at) < s.lifetime() { + return p.reachable + } + ok := g.Forward.Reachable(ctx) == nil + s.last.Store(&addonsProbe{reachable: ok, at: s.clock()}) + return ok +} diff --git a/core/application/addons_models_probe.go b/core/application/addons_models_probe.go new file mode 100644 index 000000000000..2e3ee21ab843 --- /dev/null +++ b/core/application/addons_models_probe.go @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: MIT +package application + +import ( + "context" + "path/filepath" + "sync/atomic" + "time" + + "github.com/mudler/LocalAI/core/schema" + "github.com/mudler/LocalAI/core/services/addons/hostops" + "github.com/mudler/LocalAI/core/services/addons/modelsprobe" + "github.com/mudler/xlog" +) + +// Model export states. Only a mismatch turns exports off: an owner that does +// not answer, or that is older than this check, leaves them on. +const ( + ModelExportsShared = "shared" + ModelExportsMismatch = "mismatch" + ModelExportsUnknown = "unknown" +) + +// modelsProbeInterval is how often a frontend looks at the probe again. It +// also covers delayed visibility on a network filesystem. +const modelsProbeInterval = 60 * time.Second + +// modelsProbeState is per-instance state (mode 4 of +// .agents/distributed-state.md): each process checks its own view of the +// storage, and that view is what the state describes. +type modelsProbeState struct { + // own is the probe this process wrote; only the owner sets it. + own atomic.Pointer[modelsprobe.Probe] + // state is the last frontend result; the zero value reads as unknown. + state atomic.Pointer[string] + // interval is set by tests only. + interval time.Duration +} + +func (a *Application) modelsPath() string { + if a.applicationConfig == nil || a.applicationConfig.SystemState == nil { + return "" + } + return a.applicationConfig.SystemState.Model.ModelsPath +} + +// writeModelsProbe is called by the owner at start. A failure does not stop +// the owner: it keeps running and refuses exports (the owner guard). +func (a *Application) writeModelsProbe() { + dir := a.modelsPath() + if dir == "" { + xlog.Error("addons: no models path is set, so the shared models path cannot be checked; model exports are refused") + return + } + p, err := modelsprobe.Write(dir) + if err != nil { + xlog.Error("addons: the models path probe could not be written; model exports are refused", "path", dir, "error", err) + return + } + a.modelsProbe.own.Store(&p) +} + +// ownerProbeCurrent reports whether the probe the owner wrote is still in its +// models path with the same nonce. +func (a *Application) ownerProbeCurrent() bool { + own := a.modelsProbe.own.Load() + if own == nil { + return false + } + got, err := modelsprobe.Read(a.modelsPath()) + return err == nil && modelsprobe.Same(*own, got) +} + +// checkModelsProbe is one frontend check against the owner. +func (a *Application) checkModelsProbe(ctx context.Context) { + g := a.addonGateway + if g == nil || g.Forward == nil { + return + } + next, reason := ModelExportsShared, "" + remote, err := g.Forward.ModelsProbe(ctx) + switch { + case err != nil: + next, reason = ModelExportsUnknown, "the add-on owner did not give its probe" + default: + local, rerr := modelsprobe.Read(a.modelsPath()) + if rerr != nil { + next, reason = ModelExportsMismatch, "the probe file of the owner is not visible in this models path" + } else if abs, _ := filepath.Abs(a.modelsPath()); !modelsprobe.Same(remote, local) || abs != remote.ModelsPath { + // The file records the owner's path, so the path of this instance + // is compared on its own: a symlink to the same storage fails. + next, reason = ModelExportsMismatch, "the probe file in this models path or its path differs from the owner's" + } + } + prev := ModelExportsUnknown + if p := a.modelsProbe.state.Load(); p != nil { + prev = *p + } + a.modelsProbe.state.Store(&next) + if next == prev { + return + } + switch next { + case ModelExportsMismatch: + xlog.Error("addons: this instance does not share the models path of the add-on owner; model exports are blocked here", + "reason", reason, "models_path", a.modelsPath(), "owner_models_path", remote.ModelsPath) + case ModelExportsUnknown: + xlog.Warn("addons: the shared models path could not be checked; model exports stay on", "reason", reason) + } +} + +func (a *Application) runModelsProbe(ctx context.Context) { + iv := a.modelsProbe.interval + if iv <= 0 { + iv = modelsProbeInterval + } + t := time.NewTicker(iv) + defer t.Stop() + for { + a.checkModelsProbe(ctx) + select { + case <-ctx.Done(): + return + case <-t.C: + } + } +} + +// AddonModelExports is the one reader of the shared models path state: +// "shared", "mismatch" or "unknown". An owner or combined process answers from +// its own probe. Without the add-on carrier there is nothing to check and the +// answer is "shared". +func (a *Application) AddonModelExports(_ context.Context) string { + cfg := a.applicationConfig.Addons + if !cfg.Enabled || cfg.Transport == nil { + return ModelExportsShared + } + if cfg.InstanceID == cfg.OwnerID { + if a.ownerProbeCurrent() { + return ModelExportsShared + } + return ModelExportsMismatch + } + if p := a.modelsProbe.state.Load(); p != nil { + return *p + } + return ModelExportsUnknown +} + +// ModelExportsAllowed is what the export routes and /api/features ask. +func (a *Application) ModelExportsAllowed(ctx context.Context) bool { + return a.AddonModelExports(ctx) != ModelExportsMismatch +} + +// guardedExporter refuses an export when the probe of the owner's models path +// is missing. Reads stay open. +type guardedExporter struct { + hostops.Exporter + ok func() bool +} + +func (g guardedExporter) ExportModel(ctx context.Context, userID, jobID string, req schema.ExportRequest) (string, error) { + if !g.ok() { + return "", modelsprobe.ErrNotShared + } + return g.Exporter.ExportModel(ctx, userID, jobID, req) +} + +type guardedImporter struct { + hostops.Importer + ok func() bool +} + +func (g guardedImporter) ImportModel(ctx context.Context, userID, jobID string, req schema.QuantizationImportRequest) (string, error) { + if !g.ok() { + return "", modelsprobe.ErrNotShared + } + return g.Importer.ImportModel(ctx, userID, jobID, req) +} diff --git a/core/application/addons_split_test.go b/core/application/addons_split_test.go new file mode 100644 index 000000000000..b67302749c05 --- /dev/null +++ b/core/application/addons_split_test.go @@ -0,0 +1,398 @@ +// SPDX-License-Identifier: MIT +package application + +import ( + "archive/tar" + "bytes" + "context" + "crypto/sha256" + "fmt" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "time" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/http/auth" + "github.com/mudler/LocalAI/core/library" + "github.com/mudler/LocalAI/core/schema" + "github.com/mudler/LocalAI/core/services/addons" + "github.com/mudler/LocalAI/core/services/addons/catalog" + "github.com/mudler/LocalAI/core/services/addons/gateway" + "github.com/mudler/LocalAI/core/services/addons/hostops" + "github.com/mudler/LocalAI/core/services/addons/modelsprobe" + "github.com/mudler/LocalAI/pkg/addons/contract" + "github.com/mudler/LocalAI/pkg/system" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// The owner and the frontends are separate Application values with separate +// data paths. They share only the carrier key set and the owner address, as two +// processes of a split deployment would. +var _ = Describe("split owner and frontends", func() { + var ( + keys string + ownerData string + ownerAddress string + ownerPort string + sourceURL string + stops map[*Application]func() + nextModels string + actor = contract.Actor{UserID: auth.AddonLocalOwnerID, CredentialID: auth.AddonLocalOwnerID, Mode: auth.AddonModeLocalOwner} + ) + + BeforeEach(func() { + requireAddonRuntime() + stops = map[*Application]func(){} + nextModels = "" + keys = filepath.Join(realTempDir(), "keys") + Expect(gateway.GenerateCarrierKeys(keys, []string{"127.0.0.1"})).To(Succeed()) + ownerData = realTempDir() + // The port is chosen once and reused, like a stable service address. + l, e := net.Listen("tcp", "127.0.0.1:0") + Expect(e).NotTo(HaveOccurred()) + ownerPort = strconv.Itoa(l.Addr().(*net.TCPAddr).Port) + Expect(l.Close()).To(Succeed()) + ownerAddress = "https://127.0.0.1:" + ownerPort + + manifest, e := os.ReadFile("../../tests/addons/fixtures/packages/manifest.json") + Expect(e).NotTo(HaveOccurred()) + var b bytes.Buffer + tw := tar.NewWriter(&b) + for name, data := range map[string][]byte{"manifest.json": manifest, "bin/sample": []byte("binary")} { + Expect(tw.WriteHeader(&tar.Header{Name: name, Mode: 0755, Size: int64(len(data))})).To(Succeed()) + _, e = tw.Write(data) + Expect(e).NotTo(HaveOccurred()) + } + Expect(tw.Close()).To(Succeed()) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/package.tar" { + _, _ = w.Write(b.Bytes()) + return + } + _ = catalog.WriteIndex(w, []catalog.Release{{ID: "sample", Name: "Sample", Version: "1.2.3", OS: "linux", Arch: "amd64", Artifact: sourceURL + "/package.tar", Digest: fmt.Sprintf("sha256:%x", sha256.Sum256(b.Bytes()))}}) + })) + DeferCleanup(server.Close) + sourceURL = server.URL + root := filepath.Join(ownerData, "addons") + DeferCleanup(func() { + _ = filepath.Walk(root, func(p string, i os.FileInfo, e error) error { + if e == nil && i.IsDir() { + _ = os.Chmod(p, 0700) + } + return nil + }) + }) + Expect(os.MkdirAll(root, 0700)).To(Succeed()) + sources, e := catalog.NewSources(filepath.Join(root, "sources.json"), func(context.Context, contract.Actor) error { return nil }, catalog.CredentialResolver{}) + Expect(e).NotTo(HaveOccurred()) + Expect(sources.Save(context.Background(), actor, catalog.Source{ID: "test", Name: "Test", URL: sourceURL, Enabled: true, AllowLoopbackHTTP: true})).To(Succeed()) + }) + + // start builds one application the way local-ai run does for a role and + // starts its add-on side. + start := func(role, dataPath string) (*Application, error) { + transport, e := gateway.LoadCarrierConfig(keys, role, ownerAddress) + Expect(e).NotTo(HaveOccurred()) + ctx, cancel := context.WithCancel(context.Background()) + a := &Application{applicationConfig: &config.ApplicationConfig{Context: ctx, DataPath: dataPath}} + if nextModels != "" { + a.applicationConfig.SystemState = &system.SystemState{Model: system.Model{ModelsPath: nextModels}} + } + c := config.AddonConfig{ + Enabled: true, + InstanceID: transport.InstanceID, OwnerID: gateway.OwnerIdentity, OwnerAddress: ownerAddress, + StorageID: "local", Root: filepath.Join(dataPath, "addons"), Transport: transport, OwnershipMode: auth.AddonModeLocalOwner, + Host: contract.HostCompatibility{Host: 1, API: 1, Protocol: 1, SDK: 1, DataFormat: 1, OS: "linux", Arch: "amd64"}, + } + if role == "frontend" { + c.Root = "" + } + if role == "owner" { + c.ListenAddress = "127.0.0.1:" + ownerPort + } + a.applicationConfig.Addons = c + e = a.startAddons() + stop := func() { + cancel() + _ = a.Shutdown() + } + if e != nil { + stop() + return nil, e + } + stops[a] = stop + DeferCleanup(stop) + return a, nil + } + startOK := func(role, dataPath string) *Application { + a, e := start(role, dataPath) + ExpectWithOffset(1, e).NotTo(HaveOccurred()) + return a + } + installed := func(a *Application) string { + out, e := a.AddonGateway().Dispatch(context.Background(), actor, contract.Request{OperationID: "list", Action: "addons-installed", Arguments: []byte(`{}`)}) + ExpectWithOffset(1, e).NotTo(HaveOccurred()) + return string(out.Payload) + } + install := func(a *Application) { + _, e := a.AddonGateway().Dispatch(context.Background(), actor, contract.Request{OperationID: "install", AddonID: "sample", Action: "install", Arguments: []byte(`{"sourceId":"test","version":"1.2.3"}`)}) + ExpectWithOffset(1, e).NotTo(HaveOccurred()) + } + + It("lets a frontend list apps and read add-on status through the owner", func() { + startOK("owner", ownerData) + frontendData := realTempDir() + frontend := startOK("frontend", frontendData) + install(frontend) + _, e := frontend.AddonGateway().Apps(context.Background(), actor) + Expect(e).NotTo(HaveOccurred()) + Expect(installed(frontend)).To(ContainSubstring("sample")) + // The frontend wrote nothing: the state is on the owner. + entries, e := os.ReadDir(frontendData) + Expect(e).NotTo(HaveOccurred()) + Expect(entries).To(BeEmpty()) + Expect(filepath.Join(ownerData, "addons", "journal.json")).To(BeARegularFile()) + }) + + It("serves two frontends that share one key set", func() { + startOK("owner", ownerData) + first := startOK("frontend", realTempDir()) + second := startOK("frontend", realTempDir()) + install(first) + Expect(installed(first)).To(ContainSubstring("sample")) + Expect(installed(second)).To(ContainSubstring("sample")) + }) + + It("starts a frontend while the owner is down and answers unavailable", func() { + frontend := startOK("frontend", realTempDir()) + _, e := frontend.AddonGateway().Apps(context.Background(), actor) + Expect(e).To(MatchError(gateway.ErrUnavailable)) + Expect(gateway.StatusFor(e)).To(Equal(http.StatusServiceUnavailable)) + _, e = frontend.AddonGateway().Dispatch(context.Background(), actor, contract.Request{OperationID: "list", Action: "addons-installed", Arguments: []byte(`{}`)}) + Expect(e).To(MatchError(gateway.ErrUnavailable)) + Expect(gateway.StatusFor(e)).To(Equal(http.StatusServiceUnavailable)) + // The same frontend works once the owner is up. + startOK("owner", ownerData) + _, e = frontend.AddonGateway().Apps(context.Background(), actor) + Expect(e).NotTo(HaveOccurred()) + }) + + It("reports the addons feature of a frontend from the reachability of its owner, cached", func() { + ctx := context.Background() + now := time.Now() + frontend := startOK("frontend", realTempDir()) + frontend.addonsProbe.now = func() time.Time { return now } + Expect(frontend.AddonsFeature(ctx)).To(BeFalse()) + // The owner starts, but the frontend keeps its cached answer. + owner := startOK("owner", ownerData) + Expect(owner.AddonsFeature(ctx)).To(BeTrue()) + now = now.Add(5 * time.Second) + Expect(frontend.AddonsFeature(ctx)).To(BeFalse()) + // Once the cache expires the frontend asks again. + now = now.Add(6 * time.Second) + Expect(frontend.AddonsFeature(ctx)).To(BeTrue()) + }) + + It("refuses a second owner on the same root", func() { + startOK("owner", ownerData) + _, e := start("owner", ownerData) + Expect(e).To(MatchError(addons.ErrOwnerBusy)) + }) + + It("serves the installed add-on after the owner restarts on the same root and address", func() { + owner := startOK("owner", ownerData) + frontend := startOK("frontend", realTempDir()) + install(frontend) + Expect(installed(frontend)).To(ContainSubstring("sample")) + // A process that ends cancels its context; that releases the owner's + // stores. The release runs on its own goroutine. + stops[owner]() + _, e := frontend.AddonGateway().Dispatch(context.Background(), actor, contract.Request{OperationID: "list", Action: "addons-installed", Arguments: []byte(`{}`)}) + Expect(e).To(MatchError(gateway.ErrUnavailable)) + Eventually(func() error { + _, e := start("owner", ownerData) + return e + }).Should(Succeed()) + // The audience is new, and the frontend's next call fetches it. + Expect(installed(frontend)).To(ContainSubstring("sample")) + }) + Describe("shared models path", func() { + startWith := func(role, data, models string) *Application { + nextModels = models + defer func() { nextModels = "" }() + return startOK(role, data) + } + + It("reports shared when the frontend sees the owner's probe", func() { + models := realTempDir() + owner := startWith("owner", ownerData, models) + frontend := startWith("frontend", realTempDir(), models) + frontend.checkModelsProbe(context.Background()) + Expect(frontend.AddonModelExports(context.Background())).To(Equal(ModelExportsShared)) + Expect(owner.AddonModelExports(context.Background())).To(Equal(ModelExportsShared)) + p, e := modelsprobe.Read(models) + Expect(e).NotTo(HaveOccurred()) + Expect(p.Nonce).To(Equal(owner.modelsProbe.own.Load().Nonce)) + }) + + It("reports a mismatch for a different directory and blocks exports there", func() { + startWith("owner", ownerData, realTempDir()) + frontend := startWith("frontend", realTempDir(), realTempDir()) + frontend.checkModelsProbe(context.Background()) + Expect(frontend.AddonModelExports(context.Background())).To(Equal(ModelExportsMismatch)) + Expect(frontend.ModelExportsAllowed(context.Background())).To(BeFalse()) + }) + + It("reports a mismatch when the same storage is reached through another path", func() { + models := realTempDir() + startWith("owner", ownerData, models) + link := filepath.Join(realTempDir(), "models") + Expect(os.Symlink(models, link)).To(Succeed()) + frontend := startWith("frontend", realTempDir(), link) + frontend.checkModelsProbe(context.Background()) + Expect(frontend.AddonModelExports(context.Background())).To(Equal(ModelExportsMismatch)) + }) + + It("follows a new nonce after the owner restarts", func() { + models := realTempDir() + owner := startWith("owner", ownerData, models) + frontend := startWith("frontend", realTempDir(), models) + frontend.checkModelsProbe(context.Background()) + Expect(frontend.AddonModelExports(context.Background())).To(Equal(ModelExportsShared)) + first := owner.modelsProbe.own.Load().Nonce + stops[owner]() + var again *Application + Eventually(func() error { + var e error + nextModels = models + again, e = start("owner", ownerData) + return e + }).Should(Succeed()) + Expect(again.modelsProbe.own.Load().Nonce).NotTo(Equal(first)) + frontend.checkModelsProbe(context.Background()) + Expect(frontend.AddonModelExports(context.Background())).To(Equal(ModelExportsShared)) + }) + + It("reports unknown, and keeps exports on, while the owner is down", func() { + frontend := startWith("frontend", realTempDir(), realTempDir()) + frontend.checkModelsProbe(context.Background()) + Expect(frontend.AddonModelExports(context.Background())).To(Equal(ModelExportsUnknown)) + Expect(frontend.ModelExportsAllowed(context.Background())).To(BeTrue()) + }) + + It("reports unknown for an owner without the probe endpoint", func() { + owner := startWith("owner", ownerData, realTempDir()) + owner.addonGateway.ModelsProbe = nil + frontend := startWith("frontend", realTempDir(), realTempDir()) + frontend.checkModelsProbe(context.Background()) + Expect(frontend.AddonModelExports(context.Background())).To(Equal(ModelExportsUnknown)) + }) + + It("refuses an export and an import on the owner when its probe is missing", func() { + models := realTempDir() + owner := startWith("owner", ownerData, models) + ex := &countingModels{} + gx := guardedExporter{Exporter: ex, ok: owner.ownerProbeCurrent} + gi := guardedImporter{Importer: ex, ok: owner.ownerProbeCurrent} + _, e := gx.ExportModel(context.Background(), "u", "j", schema.ExportRequest{}) + Expect(e).NotTo(HaveOccurred()) + Expect(os.Remove(filepath.Join(models, modelsprobe.FileName))).To(Succeed()) + _, e = gx.ExportModel(context.Background(), "u", "j", schema.ExportRequest{}) + Expect(e).To(MatchError(modelsprobe.ErrNotShared)) + _, e = gi.ImportModel(context.Background(), "u", "j", schema.QuantizationImportRequest{}) + Expect(e).To(MatchError(modelsprobe.ErrNotShared)) + Expect(ex.calls).To(Equal(1)) + Expect(owner.AddonModelExports(context.Background())).To(Equal(ModelExportsMismatch)) + }) + + It("keeps the owner running and refuses exports when it cannot write the probe", func() { + owner := startWith("owner", ownerData, filepath.Join(realTempDir(), "missing")) + Expect(owner.modelsProbe.own.Load()).To(BeNil()) + Expect(owner.ownerProbeCurrent()).To(BeFalse()) + Expect(owner.AddonsFeature(context.Background())).To(BeTrue()) + }) + }) + + Describe("library operations", func() { + libCtx := func() context.Context { return library.WithActor(context.Background(), actor) } + + It("lists the sources through the owner and keeps the frontend's data path empty", func() { + startOK("owner", ownerData) + frontendData := realTempDir() + frontend := startOK("frontend", frontendData) + l, e := frontend.LibraryAdmin().List(libCtx(), "") + Expect(e).NotTo(HaveOccurred()) + var ids []string + for _, in := range l.Sources { + ids = append(ids, in.ID) + } + Expect(ids).To(ContainElement("test")) + // A version the caller holds gives the cheap answer. + again, e := frontend.LibraryAdmin().List(libCtx(), l.Version) + Expect(e).NotTo(HaveOccurred()) + Expect(again.Unchanged).To(BeTrue()) + entries, e := os.ReadDir(frontendData) + Expect(e).NotTo(HaveOccurred()) + Expect(entries).To(BeEmpty()) + }) + + It("refuses a frontend on another OS or architecture", func() { + startOK("owner", ownerData) + frontend := startOK("frontend", realTempDir()) + frontend.applicationConfig.Addons.Host.Arch = "arm64" + _, e := frontend.LibraryAdmin().List(libCtx(), "") + Expect(e).To(MatchError(gateway.ErrUnavailable)) + Expect(gateway.UnavailableReason(e)).To(Equal(gateway.ReasonPlatformMismatch)) + Expect(frontend.RefreshLibraries(context.Background())).To(MatchError(gateway.ErrUnavailable)) + }) + + It("refuses library calls to an owner that does not serve them", func() { + owner := startOK("owner", ownerData) + owner.addonGateway.Libraries = nil + frontend := startOK("frontend", realTempDir()) + _, e := frontend.LibraryAdmin().List(libCtx(), "") + Expect(gateway.UnavailableReason(e)).To(Equal(gateway.ReasonLibraryUnsupported)) + }) + + It("checks the administrator right on the owner whatever the frontend did", func() { + owner := startOK("owner", ownerData) + stranger := contract.Actor{UserID: "someone", CredentialID: "key:1", Mode: auth.AddonModeLocalOwner} + _, e := owner.addonGateway.Libraries.Library(context.Background(), stranger, gateway.LibraryCall{Op: gateway.LibraryList}) + Expect(e).To(MatchError(gateway.ErrForbidden)) + _, e = owner.addonGateway.Libraries.Library(context.Background(), actor, gateway.LibraryCall{Op: gateway.LibraryList}) + Expect(e).NotTo(HaveOccurred()) + }) + + It("keeps a frontend with the official libraries off from creating an add-ons folder", func() { + startOK("owner", ownerData) + frontendData := realTempDir() + frontend := startOK("frontend", frontendData) + frontend.applicationConfig.OfficialLibraries = false + Expect(frontend.ApplyLibraries()).To(Succeed()) + applyOfficialLibrariesAtStartup(frontend.applicationConfig) + entries, e := os.ReadDir(frontendData) + Expect(e).NotTo(HaveOccurred()) + Expect(entries).To(BeEmpty()) + }) + }) +}) + +// countingModels is an exporter and importer that counts the calls it gets. +type countingModels struct{ calls int } + +func (c *countingModels) Checkpoints(context.Context, string, string) ([]hostops.Checkpoint, error) { + return nil, nil +} +func (c *countingModels) ExportModel(context.Context, string, string, schema.ExportRequest) (string, error) { + c.calls++ + return "m", nil +} +func (c *countingModels) ImportModel(context.Context, string, string, schema.QuantizationImportRequest) (string, error) { + c.calls++ + return "m", nil +} diff --git a/core/application/addons_test.go b/core/application/addons_test.go index 56cba7c22cb7..a18d195a625b 100644 --- a/core/application/addons_test.go +++ b/core/application/addons_test.go @@ -8,8 +8,10 @@ import ( "crypto/sha256" "fmt" "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/http/auth" "github.com/mudler/LocalAI/core/services/addons" "github.com/mudler/LocalAI/core/services/addons/catalog" + "github.com/mudler/LocalAI/core/services/addons/gateway" "github.com/mudler/LocalAI/pkg/addons/contract" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -260,6 +262,36 @@ var _ = Describe("addon bootstrap", func() { }) }) +var _ = Describe("addon frontend role", func() { + It("creates nothing under the add-ons folder, refuses owner calls and forwards", func() { + keys := filepath.Join(realTempDir(), "keys") + Expect(gateway.GenerateCarrierKeys(keys, []string{"127.0.0.1"})).To(Succeed()) + // Nothing listens on this port, so a forwarded call fails at the dial. + const ownerAddress = "https://127.0.0.1:1" + transport, e := gateway.LoadCarrierConfig(keys, "frontend", ownerAddress) + Expect(e).NotTo(HaveOccurred()) + dataPath := realTempDir() + a := &Application{applicationConfig: &config.ApplicationConfig{Context: context.Background(), DataPath: dataPath}} + a.applicationConfig.Addons = config.AddonConfig{ + Enabled: true, + InstanceID: gateway.FrontendIdentity, OwnerID: gateway.OwnerIdentity, OwnerAddress: ownerAddress, + StorageID: "local", Root: filepath.Join(dataPath, "addons"), Transport: transport, OwnershipMode: "local-owner", + } + Expect(a.startAddons()).To(Succeed()) + DeferCleanup(a.Shutdown) + Expect(filepath.Join(dataPath, "addons")).NotTo(BeADirectory()) + _, e = a.AddonManager().Status(context.Background(), "sample") + Expect(e).To(MatchError(addons.ErrNotOwner)) + Expect(a.AddonGateway()).NotTo(BeNil()) + Expect(a.AddonGateway().Forward).NotTo(BeNil()) + _, e = a.AddonGateway().Apps(context.Background(), contract.Actor{UserID: auth.AddonLocalOwnerID, CredentialID: auth.AddonLocalOwnerID, Mode: auth.AddonModeLocalOwner}) + Expect(e).To(MatchError(gateway.ErrUnavailable)) + entries, e := os.ReadDir(dataPath) + Expect(e).NotTo(HaveOccurred()) + Expect(entries).To(BeEmpty()) + }) +}) + var _ = Describe("addon gateway bootstrap", func() { It("keeps public gateway unavailable without explicit transport bootstrap", func() { a := &Application{applicationConfig: &config.ApplicationConfig{Context: context.Background()}} diff --git a/core/application/application.go b/core/application/application.go index 49789599303d..747624b20863 100644 --- a/core/application/application.go +++ b/core/application/application.go @@ -82,9 +82,14 @@ type Application struct { addonGateway *gateway.Gateway addonServer *http.Server addonSources *catalog.Sources - backendLibrary *backendlib.Installer - addonManager *addons.Manager - addonHostOps *addonruntime.HostOps + // addonAdminCheck is the owner's administrator check, the one the manager gets. + addonAdminCheck func(context.Context, contract.Actor) error + librarySnap librarySnapshot + backendLibrary *backendlib.Installer + addonManager *addons.Manager + addonHostOps *addonruntime.HostOps + addonsProbe addonsProbeState + modelsProbe modelsProbeState // libraryStore is the library sources file when no add-on owner runs here. libraryStore *catalog.Sources @@ -870,6 +875,7 @@ func (a *Application) startAddons() error { } } } + a.addonAdminCheck = c.AdminCheck owner := addons.OwnerConfig{InstanceID: c.InstanceID, OwnerID: c.OwnerID, OwnerAddress: c.OwnerAddress, StorageID: c.StorageID, Root: c.Root, SharedStorage: c.SharedStorage} var cat catalog.Catalog var installer *packages.Installer @@ -946,6 +952,7 @@ func (a *Application) startAddons() error { } if c.InstanceID == c.OwnerID { limits := contract.DefaultLimits() + a.writeModelsProbe() a.addonHostOps = a.newAddonHostOps(m, limits) supervisor, err := addonruntime.NewSupervisor(addonruntime.Config{StateRoot: filepath.Join(c.Root, "runtime"), DataRoot: filepath.Join(c.Root, "data"), Limits: limits, ReadyTimeout: limits.ReadTimeout, StableHealth: limits.BackoffMax, HostOps: a.addonHostOps}) if err != nil { @@ -983,3 +990,19 @@ type libraryInstallNoProgress struct{ in *backendlib.Installer } func (l libraryInstallNoProgress) Install(ctx context.Context, sourceID, backend, version string) error { return l.in.Install(ctx, sourceID, backend, version, backendlib.Progress{}) } + +func (l libraryInstallNoProgress) Updates(ctx context.Context) ([]localaitools.BackendLibraryUpdate, error) { + found, err := l.in.Updates(ctx) + if err != nil { + return nil, err + } + out := make([]localaitools.BackendLibraryUpdate, 0, len(found)) + for _, u := range found { + out = append(out, localaitools.BackendLibraryUpdate{ + Name: u.Name, SourceID: u.SourceID, + InstalledVersion: u.InstalledVersion, InstalledDigest: u.InstalledDigest, + Version: u.Version, Digest: u.Digest, Signed: u.Signed, + }) + } + return out, nil +} diff --git a/core/application/library_frontend_test.go b/core/application/library_frontend_test.go new file mode 100644 index 000000000000..5849e95a8a0d --- /dev/null +++ b/core/application/library_frontend_test.go @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: MIT +package application + +import ( + "os" + "path/filepath" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/services/addons/gateway" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("library sources on a frontend", func() { + frontend := func(dataPath string, enabled bool) *Application { + cfg := &config.ApplicationConfig{DataPath: dataPath, OfficialLibraries: false} + cfg.Addons = config.AddonConfig{Frontend: true, Enabled: enabled} + return &Application{applicationConfig: cfg} + } + + It("refuses the store and writes nothing under the add-ons folder", func() { + dir := realTempDir() + a := frontend(dir, false) + _, err := a.LibrarySources() + Expect(err).To(MatchError(gateway.ErrUnavailable)) + Expect(a.ApplyLibraries()).To(Succeed()) + Expect(filepath.Join(dir, "addons")).NotTo(BeADirectory()) + }) + + It("creates no sources.json at startup with the official libraries off", func() { + dir := realTempDir() + a := frontend(dir, true) + a.applicationConfig.Addons.InstanceID = gateway.FrontendIdentity + a.applicationConfig.Addons.OwnerID = gateway.OwnerIdentity + applyOfficialLibrariesAtStartup(a.applicationConfig) + entries, err := os.ReadDir(dir) + Expect(err).NotTo(HaveOccurred()) + Expect(entries).To(BeEmpty()) + }) + + It("still opens the store on the combined role", func() { + dir := realTempDir() + cfg := &config.ApplicationConfig{DataPath: dir, OfficialLibraries: true} + a := &Application{applicationConfig: cfg} + _, err := a.LibrarySources() + Expect(err).NotTo(HaveOccurred()) + }) +}) diff --git a/core/application/library_owner.go b/core/application/library_owner.go new file mode 100644 index 000000000000..bfee8eb9e37b --- /dev/null +++ b/core/application/library_owner.go @@ -0,0 +1,180 @@ +// SPDX-License-Identifier: MIT +package application + +import ( + "context" + "encoding/json" + "errors" + + "github.com/mudler/LocalAI/core/library" + "github.com/mudler/LocalAI/core/services/addons/backendlib" + "github.com/mudler/LocalAI/core/services/addons/catalog" + "github.com/mudler/LocalAI/core/services/addons/gateway" + "github.com/mudler/LocalAI/pkg/addons/contract" +) + +// The error types of the backend verbs that the frontend maps back to the +// errors of backendlib. The source verbs use the types of the gateway. +const ( + libraryBackendInvalid = "backend_invalid" + libraryBackendNoCapability = "backend_no_capability" + libraryBackendIntegrity = "backend_integrity" + libraryCatalogUnavailable = "catalog_unavailable" +) + +// libraryOwner serves the library calls of the carrier on the owner. It is the +// owner's only writer of the library sources: every change goes through the +// same LocalAdmin as the owner's own HTTP routes, and the same administrator +// check as the owner's other administrator-only routes runs before it. +type libraryOwner struct { + admin library.Admin + adminOnly func(context.Context, contract.Actor) error + installer func() *backendlib.Installer +} + +// newLibraryOwner builds the owner's side. The sources are the owner's store +// and ApplyLibraries brings the owner's own galleries in line after a change. +func (a *Application) newLibraryOwner() *libraryOwner { + return &libraryOwner{ + admin: library.LocalAdmin{ + Store: a.LibrarySources, + Official: func() bool { return a.applicationConfig.OfficialLibraries }, + Applied: a.ApplyLibraries, + }, + adminOnly: a.addonAdminCheck, + installer: a.BackendLibrary, + } +} + +// Library runs one verb. The administrator check comes first; a missing check +// denies everything. +func (o *libraryOwner) Library(ctx context.Context, actor contract.Actor, call gateway.LibraryCall) (json.RawMessage, error) { + // A system call has no actor and is the one call that skips the + // administrator check. It is read-only and the carrier lets it through for + // backend-releases alone; any other call, with or without an actor, takes + // the check below. + if call.System { + if call.Op != gateway.LibraryBackendReleases { + return nil, gateway.ErrForbidden + } + return o.marshal(ctx, call) + } + if o.adminOnly == nil { + return nil, gateway.ErrForbidden + } + if err := o.adminOnly(ctx, actor); err != nil { + return nil, gateway.ClassifyCheck(err) + } + return o.marshal(ctx, call) +} + +func (o *libraryOwner) marshal(ctx context.Context, call gateway.LibraryCall) (json.RawMessage, error) { + v, err := o.run(ctx, call) + if err != nil { + return nil, ownerLibraryError(err) + } + raw, err := json.Marshal(v) + if err != nil { + return nil, gateway.ErrUnavailable + } + return raw, nil +} + +func (o *libraryOwner) run(ctx context.Context, c gateway.LibraryCall) (any, error) { + switch c.Op { + case gateway.LibraryList: + return o.admin.List(ctx, c.Known) + case gateway.LibraryAdd: + return o.admin.Add(ctx, library.AddRequest{Kind: c.Kind, URL: c.URL, ID: c.ID, Name: c.Name, Format: c.Format, Mirrors: c.Mirrors}) + case gateway.LibraryUpdate: + return o.admin.SetMirrors(ctx, c.ID, c.Mirrors) + case gateway.LibraryEnable: + return o.admin.SetEnabled(ctx, c.ID, true) + case gateway.LibraryDisable: + return o.admin.SetEnabled(ctx, c.ID, false) + case gateway.LibraryRemove: + return o.admin.Remove(ctx, c.ID) + case gateway.LibraryBackendResolve: + in := o.installer() + if in == nil { + return nil, gateway.ErrUnavailable + } + if c.SourceID == "" || c.Backend == "" || c.Version == "" { + return nil, &library.BadRequestError{Message: "sourceId, backend and version are required"} + } + return in.Resolve(ctx, c.SourceID, c.Backend, c.Version, c.Capability) + case gateway.LibraryBackendReleases: + in := o.installer() + if in == nil { + return nil, gateway.ErrUnavailable + } + out := map[string][]catalog.Release{} + for _, ref := range c.Refs { + // A backend whose library is off or does not list it has nothing to + // compare, as in the local update check. + rel, err := in.Catalog.KindReleases(ctx, catalog.KindBackend, ref.SourceID, ref.Name) + if err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + continue + } + out[backendlib.RefKey(ref.SourceID, ref.Name)] = rel + } + return out, nil + } + return nil, gateway.ErrForbidden +} + +// ownerLibraryError turns a failure of a verb into the owner's typed refusal, +// so the frontend answers with the status the owner's own routes give. An +// error that is not a decision of the owner passes through and is typed as +// unavailable by the carrier. +func ownerLibraryError(err error) error { + var bad *library.BadRequestError + switch { + case errors.Is(err, library.ErrStoreUnavailable): + return gateway.ErrUnavailable + case errors.Is(err, gateway.ErrForbidden), errors.Is(err, gateway.ErrUnavailable): + return err + case errors.As(err, &bad): + return &gateway.LibraryError{Type: gateway.LibraryBadRequest, Message: bad.Message} + case errors.Is(err, catalog.ErrNotFound): + return &gateway.LibraryError{Type: gateway.LibraryNotFound, Message: err.Error()} + case errors.Is(err, library.ErrDuplicate): + return &gateway.LibraryError{Type: gateway.LibraryDuplicate, Message: err.Error()} + case errors.Is(err, catalog.ErrExists): + return &gateway.LibraryError{Type: gateway.LibraryExists, Message: err.Error()} + case errors.Is(err, catalog.ErrInvalid): + return &gateway.LibraryError{Type: gateway.LibraryInvalid, Message: err.Error()} + case errors.Is(err, backendlib.ErrNoCapability): + return &gateway.LibraryError{Type: libraryBackendNoCapability, Message: err.Error()} + case errors.Is(err, backendlib.ErrIntegrity): + return &gateway.LibraryError{Type: libraryBackendIntegrity, Message: err.Error()} + case errors.Is(err, backendlib.ErrInvalid): + return &gateway.LibraryError{Type: libraryBackendInvalid, Message: err.Error()} + case errors.Is(err, catalog.ErrUnavailable), errors.Is(err, catalog.ErrRefresh): + return &gateway.LibraryError{Type: libraryCatalogUnavailable, Message: err.Error()} + } + return &gateway.LibraryError{Type: gateway.LibraryInternal, Message: "the library operation failed"} +} + +// Snapshot is the owner's answer to the frontend's poll: the sources that +// decide the galleries and the official-libraries switch, without the fetch +// state, the policies or the credential references. It needs no actor: the +// frontend holds the pinned carrier identity, and the list is what its own +// gallery lists contain. +func (o *libraryOwner) Snapshot(ctx context.Context, known string) (json.RawMessage, error) { + l, err := o.admin.List(ctx, known) + if err != nil { + return nil, gateway.ErrUnavailable + } + for i := range l.Sources { + l.Sources[i].Refresh, l.Sources[i].Policy, l.Sources[i].AuthReference = nil, nil, "" + } + raw, err := json.Marshal(l) + if err != nil { + return nil, gateway.ErrUnavailable + } + return raw, nil +} diff --git a/core/application/library_remote.go b/core/application/library_remote.go new file mode 100644 index 000000000000..82c6f5fb564b --- /dev/null +++ b/core/application/library_remote.go @@ -0,0 +1,288 @@ +// SPDX-License-Identifier: MIT +package application + +import ( + "context" + "encoding/json" + "errors" + "runtime" + "time" + + "github.com/mudler/LocalAI/core/http/auth" + "github.com/mudler/LocalAI/core/library" + "github.com/mudler/LocalAI/core/services/addons/backendlib" + "github.com/mudler/LocalAI/core/services/addons/catalog" + "github.com/mudler/LocalAI/core/services/addons/gateway" + "github.com/mudler/LocalAI/pkg/addons/contract" +) + +// remoteError is a refusal of the owner, mapped back to the sentinel the +// owner's own routes would have seen. The message is the owner's. +type remoteError struct { + sentinel error + message string +} + +func (e *remoteError) Error() string { return e.message } +func (e *remoteError) Is(target error) bool { return target == e.sentinel } + +func fromLibraryError(le *gateway.LibraryError) error { + switch le.Type { + case gateway.LibraryNotFound: + return &remoteError{catalog.ErrNotFound, le.Message} + case gateway.LibraryExists: + return &remoteError{catalog.ErrExists, le.Message} + case gateway.LibraryDuplicate: + return &remoteError{library.ErrDuplicate, le.Message} + case gateway.LibraryInvalid: + return &remoteError{catalog.ErrInvalid, le.Message} + case gateway.LibraryBadRequest: + return &library.BadRequestError{Message: le.Message} + case libraryBackendInvalid: + return &remoteError{backendlib.ErrInvalid, le.Message} + case libraryBackendNoCapability: + return &remoteError{backendlib.ErrNoCapability, le.Message} + case libraryBackendIntegrity: + return &remoteError{backendlib.ErrIntegrity, le.Message} + case libraryCatalogUnavailable: + return &remoteError{catalog.ErrUnavailable, le.Message} + } + return errors.New(le.Message) +} + +// hostPlatform is the OS and architecture this process runs on, as the add-on +// configuration says, else as the runtime says. +func (a *Application) hostPlatform() (string, string) { + h := a.applicationConfig.Addons.Host + os, arch := h.OS, h.Arch + if os == "" { + os = runtime.GOOS + } + if arch == "" { + arch = runtime.GOARCH + } + return os, arch +} + +// ownerCarrier returns the carrier to the owner, or says why there is none. +func (a *Application) ownerCarrier() (*gateway.Carrier, error) { + g := a.addonGateway + if g == nil || g.Forward == nil { + return nil, &gateway.UnavailableError{Reason: gateway.ReasonAddonsOff} + } + return g.Forward, nil +} + +// libraryOwnerReady checks that the owner serves library calls and runs on the +// platform of this instance. The owner resolves entries for its own OS and +// architecture, so a frontend on another one would install the wrong image. +func (a *Application) libraryOwnerReady(ctx context.Context, c *gateway.Carrier) error { + caps, err := c.Capabilities(ctx) + if err != nil { + return err + } + if caps.Protocol < 2 || !caps.Supports("library") { + return &gateway.UnavailableError{Reason: gateway.ReasonLibraryUnsupported} + } + if os, arch := a.hostPlatform(); caps.OS != os || caps.Arch != arch { + return &gateway.UnavailableError{Reason: gateway.ReasonPlatformMismatch} + } + return nil +} + +// libraryActor is the actor of a library call. An HTTP request brings its own +// (library.WithActor). Work with no request, such as the update check of a +// background loop, runs as the one local owner when the deployment chose that +// mode, and otherwise has no authority. +func (a *Application) libraryActor(ctx context.Context) (contract.Actor, error) { + if actor, ok := library.ActorFrom(ctx); ok { + return actor, nil + } + if a.AddonOwnershipMode() == auth.AddonModeLocalOwner { + return contract.Actor{UserID: auth.AddonLocalOwnerID, CredentialID: auth.AddonLocalOwnerID, Mode: auth.AddonModeLocalOwner}, nil + } + return contract.Actor{}, gateway.ErrForbidden +} + +// forwardLibrary sends one library call to the owner. +func (a *Application) forwardLibrary(ctx context.Context, call gateway.LibraryCall) (json.RawMessage, error) { + c, err := a.ownerCarrier() + if err != nil { + return nil, err + } + actor, err := a.libraryActor(ctx) + if err != nil { + return nil, err + } + if err := a.libraryOwnerReady(ctx, c); err != nil { + return nil, err + } + raw, err := c.Library(ctx, actor, call) + var le *gateway.LibraryError + if errors.As(err, &le) { + return nil, fromLibraryError(le) + } + return raw, err +} + +// releasesCall sends a backend-releases call. A call with a request actor is a +// user-facing call and goes the usual way. A call with none is the background +// update check: it uses the read-only system call, which the owner accepts from +// this pinned frontend without an actor, so it works in named-user mode. +func (a *Application) releasesCall(ctx context.Context, call gateway.LibraryCall) (json.RawMessage, error) { + if _, ok := library.ActorFrom(ctx); ok { + return a.forwardLibrary(ctx, call) + } + c, err := a.ownerCarrier() + if err != nil { + return nil, err + } + if err := a.libraryOwnerReady(ctx, c); err != nil { + return nil, err + } + if caps, err := c.Capabilities(ctx); err != nil { + return nil, err + } else if !caps.Supports(gateway.CallLibrarySystem) { + return nil, &gateway.UnavailableError{Reason: gateway.ReasonLibraryUnsupported} + } + raw, err := c.LibrarySystem(ctx, call) + var le *gateway.LibraryError + if errors.As(err, &le) { + return nil, fromLibraryError(le) + } + return raw, err +} + +// remoteLibrary is the library.Admin of a frontend: it keeps no store and +// forwards every call to the owner, which is the only writer. +type remoteLibrary struct{ a *Application } + +func call[T any](r remoteLibrary, ctx context.Context, c gateway.LibraryCall) (T, error) { + var out T + raw, err := r.a.forwardLibrary(ctx, c) + if err != nil { + return out, err + } + if err := json.Unmarshal(raw, &out); err != nil { + return out, gateway.ErrUnavailable + } + return out, nil +} + +// changed refreshes the gallery lists after a change this frontend forwarded, +// so the frontend does not wait for the next poll to follow its own change. +func (r remoteLibrary) changed(ctx context.Context) { + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) + defer cancel() + _ = r.a.RefreshLibraries(ctx) +} + +func (r remoteLibrary) List(ctx context.Context, known string) (library.Listing, error) { + return call[library.Listing](r, ctx, gateway.LibraryCall{Op: gateway.LibraryList, Known: known}) +} + +func (r remoteLibrary) Add(ctx context.Context, req library.AddRequest) (library.Info, error) { + in, err := call[library.Info](r, ctx, gateway.LibraryCall{Op: gateway.LibraryAdd, Kind: req.Kind, URL: req.URL, ID: req.ID, Name: req.Name, Format: req.Format, Mirrors: req.Mirrors}) + if err == nil { + r.changed(ctx) + } + return in, err +} + +func (r remoteLibrary) SetMirrors(ctx context.Context, id string, mirrors []string) (*library.Info, error) { + in, err := call[*library.Info](r, ctx, gateway.LibraryCall{Op: gateway.LibraryUpdate, ID: id, Mirrors: mirrors}) + if err == nil { + r.changed(ctx) + } + return in, err +} + +func (r remoteLibrary) SetEnabled(ctx context.Context, id string, enabled bool) (*library.Info, error) { + op := gateway.LibraryDisable + if enabled { + op = gateway.LibraryEnable + } + in, err := call[*library.Info](r, ctx, gateway.LibraryCall{Op: op, ID: id}) + if err == nil { + r.changed(ctx) + } + return in, err +} + +func (r remoteLibrary) Remove(ctx context.Context, id string) (library.RemoveResult, error) { + res, err := call[library.RemoveResult](r, ctx, gateway.LibraryCall{Op: gateway.LibraryRemove, ID: id}) + if err == nil { + r.changed(ctx) + } + return res, err +} + +// LibraryAdmin returns the administration of the library sources for this +// process: the owner's store on a combined or owner process, and the carrier to +// the owner on a frontend. A frontend with add-ons off has neither, and the +// routes answer unavailable before they get here. +func (a *Application) LibraryAdmin() library.Admin { + if a.AddonFrontend() { + return remoteLibrary{a} + } + return library.LocalAdmin{ + Store: a.LibrarySources, + Official: func() bool { return a.applicationConfig.OfficialLibraries }, + Applied: a.ApplyLibraries, + } +} + +// remoteBackends is the owner of the libraries for the backend installer of a +// frontend: it resolves and pins on the owner, and the frontend installs. +type remoteBackends struct{ a *Application } + +func (r remoteBackends) Resolve(ctx context.Context, sourceID, backend, version, capability string) (backendlib.Plan, error) { + raw, err := r.a.forwardLibrary(ctx, gateway.LibraryCall{Op: gateway.LibraryBackendResolve, SourceID: sourceID, Backend: backend, Version: version, Capability: capability}) + if err != nil { + return backendlib.Plan{}, err + } + var p backendlib.Plan + if json.Unmarshal(raw, &p) != nil { + return backendlib.Plan{}, gateway.ErrUnavailable + } + return p, nil +} + +func (r remoteBackends) Releases(ctx context.Context, refs []backendlib.Ref) (map[string][]catalog.Release, error) { + out := map[string][]catalog.Release{} + // The call takes at most 256 backends. + for len(refs) > 0 { + n := min(len(refs), 256) + c := gateway.LibraryCall{Op: gateway.LibraryBackendReleases} + for _, ref := range refs[:n] { + c.Refs = append(c.Refs, gateway.LibraryRef{SourceID: ref.SourceID, Name: ref.Name}) + } + refs = refs[n:] + raw, err := r.a.releasesCall(ctx, c) + if err != nil { + return nil, err + } + var part map[string][]catalog.Release + if json.Unmarshal(raw, &part) != nil { + return nil, gateway.ErrUnavailable + } + for k, v := range part { + out[k] = v + } + } + return out, nil +} + +// Target is the platform the owner resolves for. forwardLibrary refuses a +// frontend on another platform, so the answer is this instance's own. +func (r remoteBackends) Target(ctx context.Context) (string, string, error) { + c, err := r.a.ownerCarrier() + if err != nil { + return "", "", err + } + if err := r.a.libraryOwnerReady(ctx, c); err != nil { + return "", "", err + } + os, arch := r.a.hostPlatform() + return os, arch, nil +} diff --git a/core/application/library_snapshot.go b/core/application/library_snapshot.go new file mode 100644 index 000000000000..d757c28cbdf2 --- /dev/null +++ b/core/application/library_snapshot.go @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: MIT +package application + +import ( + "context" + "encoding/json" + "sync" + "time" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/gallery" + "github.com/mudler/LocalAI/core/library" + "github.com/mudler/LocalAI/core/services/addons/gateway" + "github.com/mudler/xlog" +) + +// libraryRefreshInterval is how often a frontend asks the owner whether the +// library sources or the official-libraries switch changed. A change on the +// owner reaches the gallery lists of a frontend within this time. +var libraryRefreshInterval = 60 * time.Second + +// libraryRetryInterval is how often a frontend asks until the owner answered +// once. +const libraryRetryInterval = 5 * time.Second + +// librarySnapshot is the per-instance copy of the owner's source list. Two +// frontends can show different gallery lists for up to one refresh interval +// after a change (mode 4 of .agents/distributed-state.md). Nothing is served +// from it: it only decides which galleries this frontend reads. +type librarySnapshot struct { + mu sync.Mutex + version string + loaded bool + stale bool + // The galleries the flags named. Every rebuild starts from them, so a + // source the owner removed leaves the lists again. + baseModels, baseBackends []config.Gallery +} + +// LibraryListsStale reports that a frontend could not reach the owner at its +// last refresh, so its gallery lists may be older than the owner's sources. It +// is false before the first answer, because the lists are then the flags'. +func (a *Application) LibraryListsStale() bool { + a.librarySnap.mu.Lock() + defer a.librarySnap.mu.Unlock() + return a.librarySnap.stale +} + +// RefreshLibraries asks the owner for its source list and rebuilds the model +// and backend galleries and the official-libraries switch when the version +// changed. When the owner does not answer, the last lists stay and are marked +// stale; the lists are never rebuilt from nothing. +func (a *Application) RefreshLibraries(ctx context.Context) error { + s := &a.librarySnap + err := a.fetchLibraries(ctx) + s.mu.Lock() + defer s.mu.Unlock() + if err != nil { + if s.loaded && !s.stale { + xlog.Warn("the library sources of the add-on owner cannot be read; the gallery lists may be stale", "error", err) + } + if s.loaded { + s.stale = true + } + return err + } + s.stale = false + return nil +} + +func (a *Application) fetchLibraries(ctx context.Context) error { + c, err := a.ownerCarrier() + if err != nil { + return err + } + if err := a.libraryOwnerReady(ctx, c); err != nil { + return err + } + s := &a.librarySnap + s.mu.Lock() + known := s.version + s.mu.Unlock() + raw, err := c.LibrarySnapshot(ctx, known) + if err != nil { + return err + } + var l library.Listing + if json.Unmarshal(raw, &l) != nil || l.Version == "" { + return gateway.ErrUnavailable + } + s.mu.Lock() + defer s.mu.Unlock() + if l.Unchanged || (s.loaded && l.Version == s.version) { + return nil + } + cfg := a.applicationConfig + if !s.loaded { + s.baseModels = append([]config.Gallery(nil), cfg.Galleries...) + s.baseBackends = append([]config.Gallery(nil), cfg.BackendGalleries...) + } + prevM, prevB := cfg.Galleries, cfg.BackendGalleries + // The owner's switch is the deployment's value; a frontend never sets it. + cfg.OfficialLibraries = l.OfficialLibraries + cfg.Galleries, cfg.BackendGalleries = library.ReconcileInfos(l.Sources, s.baseModels, s.baseBackends) + gallery.ResetGalleryModelCacheIfChanged(prevM, prevB, cfg) + s.version, s.loaded = l.Version, true + return nil +} + +// runLibraryRefresh keeps the gallery lists of a frontend in line with the +// owner until ctx ends. +func (a *Application) runLibraryRefresh(ctx context.Context) { + for { + _ = a.RefreshLibraries(ctx) + wait := libraryRefreshInterval + a.librarySnap.mu.Lock() + if !a.librarySnap.loaded { + wait = min(wait, libraryRetryInterval) + } + a.librarySnap.mu.Unlock() + select { + case <-ctx.Done(): + return + case <-time.After(wait): + } + } +} diff --git a/core/application/library_sources.go b/core/application/library_sources.go index 4bfb075979a0..b1c7ac6a5b44 100644 --- a/core/application/library_sources.go +++ b/core/application/library_sources.go @@ -2,11 +2,14 @@ package application import ( "context" + "fmt" + "time" "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/gallery" "github.com/mudler/LocalAI/core/library" "github.com/mudler/LocalAI/core/services/addons/catalog" + "github.com/mudler/LocalAI/core/services/addons/gateway" "github.com/mudler/xlog" ) @@ -16,6 +19,9 @@ import ( // otherwise the store file under the data path is opened once. It is the file // `local-ai library` edits. func (a *Application) LibrarySources() (*catalog.Sources, error) { + if a.AddonFrontend() { + return nil, fmt.Errorf("%w: library sources are managed on the add-on owner", gateway.ErrUnavailable) + } if a.addonSources != nil { return a.addonSources, nil } @@ -40,6 +46,20 @@ func (a *Application) LibrarySources() (*catalog.Sources, error) { // It is called after the HTTP routes change a // source and after the official_libraries setting changes. func (a *Application) ApplyLibraries(dropped ...string) error { + if a.AddonFrontend() { + // A frontend keeps no store. It follows the owner's list instead, and + // with add-ons off there is no owner to follow. + if !a.AddonsEnabled() { + return nil + } + parent := a.applicationConfig.Context + if parent == nil { + parent = context.Background() + } + ctx, cancel := context.WithTimeout(parent, 5*time.Second) + defer cancel() + return a.RefreshLibraries(ctx) + } store, err := a.LibrarySources() if err != nil { return err @@ -64,7 +84,7 @@ func (a *Application) ApplyLibraries(dropped ...string) error { // libraries off. The command line default is on, so this only acts on a saved // setting. func applyOfficialLibrariesAtStartup(cfg *config.ApplicationConfig) { - if cfg.OfficialLibraries || cfg.DataPath == "" { + if cfg.OfficialLibraries || cfg.DataPath == "" || cfg.Addons.IsFrontend() { return } store, err := catalog.OpenStore(catalog.StorePath(cfg.DataPath)) @@ -75,3 +95,9 @@ func applyOfficialLibrariesAtStartup(cfg *config.ApplicationConfig) { store.SetOfficial(false) cfg.Galleries, cfg.BackendGalleries = library.Reconcile(store, cfg.Galleries, cfg.BackendGalleries) } + +// AddonFrontend says whether this process is an add-on frontend. It keeps no +// owner state: nothing under the add-ons folder of the data path is opened. +func (a *Application) AddonFrontend() bool { + return a != nil && a.applicationConfig != nil && a.applicationConfig.Addons.IsFrontend() +} diff --git a/core/application/library_system_test.go b/core/application/library_system_test.go new file mode 100644 index 000000000000..8126c01642ac --- /dev/null +++ b/core/application/library_system_test.go @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: MIT +package application + +import ( + "context" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/services/addons/backendlib" + "github.com/mudler/LocalAI/core/services/addons/gateway" + "github.com/mudler/LocalAI/pkg/addons/contract" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("the owner's library system call", func() { + var checks int + owner := func() *libraryOwner { + return &libraryOwner{ + adminOnly: func(context.Context, contract.Actor) error { checks++; return gateway.ErrForbidden }, + installer: func() *backendlib.Installer { return nil }, + } + } + BeforeEach(func() { checks = 0 }) + + It("runs the release check without an actor and without the administrator check", func() { + _, err := owner().Library(context.Background(), contract.Actor{}, gateway.LibraryCall{Op: gateway.LibraryBackendReleases, System: true}) + // The test installer is absent, so the verb itself is unavailable; what + // matters is that the administrator check did not refuse it first. + Expect(err).To(MatchError(gateway.ErrUnavailable)) + Expect(checks).To(BeZero()) + }) + + It("refuses any other verb marked as a system call", func() { + _, err := owner().Library(context.Background(), contract.Actor{}, gateway.LibraryCall{Op: gateway.LibraryRemove, ID: "x", System: true}) + Expect(err).To(MatchError(gateway.ErrForbidden)) + }) + + It("still refuses a user-facing call without an actor", func() { + _, err := owner().Library(context.Background(), contract.Actor{}, gateway.LibraryCall{Op: gateway.LibraryBackendReleases}) + Expect(err).To(MatchError(gateway.ErrForbidden)) + Expect(checks).To(Equal(1)) + }) +}) + +var _ = Describe("the frontend's library actor", func() { + It("gives the background loop no actor in named-user mode", func() { + cfg := &config.ApplicationConfig{} + cfg.Addons = config.AddonConfig{Frontend: true, Enabled: true} + a := &Application{applicationConfig: cfg} + _, err := a.libraryActor(context.Background()) + Expect(err).To(MatchError(gateway.ErrForbidden)) + }) +}) diff --git a/core/cli/run.go b/core/cli/run.go index fabbd71d2357..f546fd9dd9c7 100644 --- a/core/cli/run.go +++ b/core/cli/run.go @@ -16,7 +16,6 @@ import ( cliContext "github.com/mudler/LocalAI/core/cli/context" "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/http" - "github.com/mudler/LocalAI/core/library" "github.com/mudler/LocalAI/core/p2p" "github.com/mudler/LocalAI/internal" "github.com/mudler/LocalAI/pkg/diagnostics" @@ -37,6 +36,10 @@ type RunCMD struct { AddonHostOps string `name:"addon-host-ops" env:"LOCALAI_ADDON_HOST_OPS" default:"fine-tuning=finetune.,quantization=quantize." help:"Add-ons that may call host operations, as a comma list of addon-id=operation-prefix. An add-on that is not listed gets no host operation channel. Empty gives none" group:"addons"` AddonsEnabled bool `name:"addons-enabled" env:"LOCALAI_ADDONS_ENABLED" default:"true" help:"Run the add-on owner, which installs and supervises add-ons. Turn it off with --addons-enabled=false" group:"addons"` AddonsOwnershipMode string `name:"addons-ownership-mode" env:"LOCALAI_ADDONS_OWNERSHIP_MODE" help:"Who may use add-ons: named-user (each account has its own access; needs authentication) or local-owner (every caller shares one owner). Default: local-owner when authentication is off, named-user when it is on" group:"addons"` + AddonsRole string `name:"addons-role" env:"LOCALAI_ADDONS_ROLE" default:"combined" help:"How this process takes part in the add-on owner: combined (this node is its own owner, the default), owner (only the owner, serves the carrier on the network) or frontend (no owner here; every add-on request goes to the owner). owner and frontend need --addons-owner-address and --addons-carrier-dir" group:"addons"` + AddonsOwnerAddress string `name:"addons-owner-address" env:"LOCALAI_ADDONS_OWNER_ADDRESS" help:"Address of the add-on owner carrier, as https://host:port with no path. Use a stable service name, never a pod address. Required for the owner and frontend roles" group:"addons"` + AddonsOwnerListen string `name:"addons-owner-listen" env:"LOCALAI_ADDONS_OWNER_LISTEN" help:"Owner role only: the address the carrier listens on. Empty listens on all interfaces, on the port of --addons-owner-address" group:"addons"` + AddonsCarrierDir string `name:"addons-carrier-dir" env:"LOCALAI_ADDONS_CARRIER_DIR" type:"path" help:"Directory with the carrier key set made by 'local-ai util addons-carrier-keys'. Required for the owner and frontend roles" group:"addons"` AutoUpgradeAddons bool `name:"auto-upgrade-addons" env:"LOCALAI_AUTO_UPGRADE_ADDONS" default:"false" help:"Upgrade installed add-ons on their own when a newer version appears in their library. Off by default; upgrades then stay manual" group:"addons"` AddonsOfficialLibrary bool `name:"addons-official-library" env:"LOCALAI_ADDONS_OFFICIAL_LIBRARY" default:"true" help:"Register the official add-on library on the first start, when the add-on owner has no sources.json yet. A library the admin removes later stays removed. Turn it off with --addons-official-library=false" group:"addons"` AddonsBackendLibrary bool `name:"addons-backend-library" env:"LOCALAI_ADDONS_BACKEND_LIBRARY" default:"true" help:"Register the backend library on the first start, when the add-on owner has no sources.json yet. A build from master reads it as the backend source; a tagged build reads the backend gallery until the release cutover. Turn it off to read the backend gallery only" group:"addons"` @@ -357,7 +360,7 @@ func (r *RunCMD) Run(ctx *cliContext.Context) (result error) { // The galleries the flags named also become library sources, and the // library sources decide which galleries are read. This must stay // after the two options above. - library.Option(r.DataPath, r.AddonsOfficialLibrary, r.AddonsBackendLibrary, r.OfficialLibraries), + libraryOptionFor(r), config.WithVRAMPersistentCache(r.VRAMPersistentCache), config.WithCors(r.CORS), config.WithCorsAllowOrigins(r.CORSAllowOrigins), @@ -671,16 +674,23 @@ func (r *RunCMD) Run(ctx *cliContext.Context) (result error) { // Authentication authEnabled := r.AuthEnabled || r.GitHubClientID != "" || r.OIDCClientID != "" - addonCfg, err := addonConfigFor(r.AddonsEnabled, r.AddonsOwnershipMode, r.DataPath, authEnabled) + addonCfg, err := addonConfigFor(r.AddonsEnabled, r.AddonsOwnershipMode, r.DataPath, authEnabled, addonRoleFlags{Role: r.AddonsRole, OwnerAddress: r.AddonsOwnerAddress, OwnerListen: r.AddonsOwnerListen, CarrierDir: r.AddonsCarrierDir}) if err != nil { cancelRun(nil) return err } + addonCfg.Frontend = r.AddonsRole == addonRoleFrontend + if addonCfg.Enabled && r.AddonsRole == addonRoleFrontend && !authEnabled { + xlog.Warn("This add-on frontend has no account database, so its add-on pins are kept in this process only and differ between frontends") + } if addonCfg.Enabled { addonCfg.OfficialLibrary = r.AddonsOfficialLibrary addonCfg.OfficialBackendLibrary = r.AddonsBackendLibrary addonCfg.BackendMaxBytes = r.AddonsBackendMaxBytes opts = append(opts, config.WithAddons(addonCfg)) + } else if addonCfg.Frontend { + // Add-ons are off, but the role still forbids owner state. + opts = append(opts, config.WithAddons(config.AddonConfig{Frontend: true})) } if authEnabled { opts = append(opts, config.WithAuthEnabled(true)) diff --git a/core/cli/run_addons.go b/core/cli/run_addons.go index 260123946a11..d8189ec4fcc2 100644 --- a/core/cli/run_addons.go +++ b/core/cli/run_addons.go @@ -3,10 +3,13 @@ package cli import ( "fmt" + "net" + "net/url" "path/filepath" "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/http/auth" + "github.com/mudler/LocalAI/core/library" "github.com/mudler/LocalAI/core/services/addons/gateway" "github.com/mudler/xlog" ) @@ -15,17 +18,40 @@ import ( // owner, so the node supervises the add-ons itself. const localAddonOwnerID = "local" -// addonConfigFor derives the add-on owner configuration of a single node from -// the run flags. Only the on/off switch and the ownership mode are flags; the -// rest follows from the node being its own owner. +// The roles of --addons-role. +const ( + addonRoleCombined = "combined" + addonRoleOwner = "owner" + addonRoleFrontend = "frontend" +) + +// addonRoleFlags holds the flags that choose how this process takes part in the +// add-on owner. The zero value is the combined role of a single node. +type addonRoleFlags struct { + Role string + OwnerAddress string + OwnerListen string + CarrierDir string +} + +// addonConfigFor derives the add-on owner configuration from the run flags. In +// the combined role (the default) the node is its own owner and only the +// on/off switch and the ownership mode matter. In the owner and frontend roles +// the owner is its own workload: the carrier keys come from a key directory +// and the frontend never touches the owner's files. // // An empty mode takes the documented default: local-owner when authentication // is off, named-user when it is on. An explicit mode always wins. Named-user // needs the account database, so an explicit named-user without // authentication leaves the owner off with a warning instead of stopping the // server. -func addonConfigFor(enabled bool, mode, dataPath string, authEnabled bool) (config.AddonConfig, error) { +func addonConfigFor(enabled bool, mode, dataPath string, authEnabled bool, roleFlags addonRoleFlags) (config.AddonConfig, error) { if !enabled { + // Add-ons off means no carrier, no keys and no owner address, so a role + // flag cannot matter. A frontend with add-ons off starts without them. + if roleFlags.Role != "" || roleFlags.OwnerAddress != "" || roleFlags.OwnerListen != "" || roleFlags.CarrierDir != "" { + xlog.Info("Add-ons are off: the --addons-role, --addons-owner-address, --addons-owner-listen and --addons-carrier-dir flags are ignored") + } return config.AddonConfig{}, nil } if mode == "" { @@ -50,6 +76,23 @@ func addonConfigFor(enabled bool, mode, dataPath string, authEnabled bool) (conf xlog.Warn("Add-ons are off: named-user ownership needs authentication. Enable authentication, or set --addons-ownership-mode=local-owner to let every caller share the owner's authority") return config.AddonConfig{}, nil } + role := roleFlags.Role + if role == "" { + role = addonRoleCombined + } + switch role { + case addonRoleCombined: + if roleFlags.OwnerAddress != "" { + return config.AddonConfig{}, fmt.Errorf("--addons-owner-address: only for --addons-role=%s or --addons-role=%s, not %s", addonRoleOwner, addonRoleFrontend, addonRoleCombined) + } + if roleFlags.CarrierDir != "" { + return config.AddonConfig{}, fmt.Errorf("--addons-carrier-dir: only for --addons-role=%s or --addons-role=%s, not %s", addonRoleOwner, addonRoleFrontend, addonRoleCombined) + } + case addonRoleOwner, addonRoleFrontend: + return splitAddonConfig(role, mode, dataPath, roleFlags) + default: + return config.AddonConfig{}, fmt.Errorf("--addons-role: %q is not valid; use %q, %q or %q", role, addonRoleCombined, addonRoleOwner, addonRoleFrontend) + } root, err := filepath.Abs(filepath.Join(dataPath, "addons")) if err != nil { return config.AddonConfig{}, err @@ -65,3 +108,69 @@ func addonConfigFor(enabled bool, mode, dataPath string, authEnabled bool) (conf OwnershipMode: mode, }, nil } + +// splitAddonConfig builds the configuration of the owner role or the frontend +// role. Both use the fixed carrier identities, so a root made by a single node +// keeps passing the owner identity check. The frontend never +// creates or opens the owner's files. +func splitAddonConfig(role, mode, dataPath string, f addonRoleFlags) (config.AddonConfig, error) { + if f.OwnerAddress == "" { + return config.AddonConfig{}, fmt.Errorf("--addons-owner-address is required with --addons-role=%s", role) + } + if f.CarrierDir == "" { + return config.AddonConfig{}, fmt.Errorf("--addons-carrier-dir is required with --addons-role=%s", role) + } + port, err := ownerAddressPort(f.OwnerAddress) + if err != nil { + return config.AddonConfig{}, err + } + transport, err := gateway.LoadCarrierConfig(f.CarrierDir, role, f.OwnerAddress) + if err != nil { + return config.AddonConfig{}, fmt.Errorf("--addons-carrier-dir: %w", err) + } + c := config.AddonConfig{ + Enabled: true, + InstanceID: transport.InstanceID, OwnerID: gateway.OwnerIdentity, OwnerAddress: f.OwnerAddress, + StorageID: "local", Transport: transport, OwnershipMode: mode, + } + // A frontend keeps no add-on state, so it has no root: Root stays empty. + if role == addonRoleFrontend { + return c, nil + } + c.Root, err = filepath.Abs(filepath.Join(dataPath, "addons")) + if err != nil { + return config.AddonConfig{}, err + } + c.ListenAddress = f.OwnerListen + if c.ListenAddress == "" { + c.ListenAddress = ":" + port + } + return c, nil +} + +// ownerAddressPort checks that the owner address is https://host:port with no +// path, and returns the port. +func ownerAddressPort(address string) (string, error) { + u, err := url.Parse(address) + if err != nil || u.Scheme != "https" || u.Host == "" || u.User != nil || u.Path != "" || u.RawQuery != "" || u.Fragment != "" { + return "", fmt.Errorf("--addons-owner-address: %q must look like https://host:port, with no path", address) + } + _, port, err := net.SplitHostPort(u.Host) + if err != nil || port == "" { + return "", fmt.Errorf("--addons-owner-address: %q must name a port, like https://host:port", address) + } + return port, nil +} + +// libraryOptionFor returns the startup option that joins the galleries the +// flags named with the library sources. The sources live in the owner's root, +// and the option writes to them, so the frontend role must not apply it: it +// reads only the galleries the flags named. Library sources are managed on the +// owner. +func libraryOptionFor(r *RunCMD) config.AppOption { + if r.AddonsRole == addonRoleFrontend { + xlog.Info("Add-on frontend: library sources are managed on the add-on owner; reading only --galleries and --backend-galleries") + return func(*config.ApplicationConfig) {} + } + return library.Option(r.DataPath, r.AddonsOfficialLibrary, r.AddonsBackendLibrary, r.OfficialLibraries) +} diff --git a/core/cli/run_addons_test.go b/core/cli/run_addons_test.go index 679a447de4a0..ef0c24e566ca 100644 --- a/core/cli/run_addons_test.go +++ b/core/cli/run_addons_test.go @@ -8,6 +8,7 @@ import ( "github.com/alecthomas/kong" cliContext "github.com/mudler/LocalAI/core/cli/context" "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/services/addons/gateway" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -32,7 +33,7 @@ var _ = Describe("Add-on owner flags", Serial, func() { return &command.Run, err } BeforeEach(func() { - for _, key := range []string{"LOCALAI_ADDONS_ENABLED", "LOCALAI_ADDONS_OWNERSHIP_MODE", "LOCALAI_ADDONS_OFFICIAL_LIBRARY"} { + for _, key := range []string{"LOCALAI_ADDONS_ENABLED", "LOCALAI_ADDONS_OWNERSHIP_MODE", "LOCALAI_ADDONS_OFFICIAL_LIBRARY", "LOCALAI_ADDONS_ROLE", "LOCALAI_ADDONS_OWNER_ADDRESS", "LOCALAI_ADDONS_OWNER_LISTEN", "LOCALAI_ADDONS_CARRIER_DIR"} { value, present := os.LookupEnv(key) DeferCleanup(func() { if present { @@ -54,7 +55,7 @@ var _ = Describe("Add-on owner flags", Serial, func() { // The pilot backend library is a default source too. Expect(r.AddonsBackendLibrary).To(BeTrue()) dir := GinkgoT().TempDir() - c, err := addonConfigFor(r.AddonsEnabled, r.AddonsOwnershipMode, dir, true) + c, err := addonConfigFor(r.AddonsEnabled, r.AddonsOwnershipMode, dir, true, addonRoleFlags{}) Expect(err).NotTo(HaveOccurred()) Expect(c.Enabled).To(BeTrue()) Expect(c.InstanceID).To(Equal(c.OwnerID)) @@ -71,7 +72,7 @@ var _ = Describe("Add-on owner flags", Serial, func() { It("defaults to local-owner when authentication is off", func() { r, err := parse() Expect(err).NotTo(HaveOccurred()) - c, err := addonConfigFor(r.AddonsEnabled, r.AddonsOwnershipMode, GinkgoT().TempDir(), false) + c, err := addonConfigFor(r.AddonsEnabled, r.AddonsOwnershipMode, GinkgoT().TempDir(), false, addonRoleFlags{}) Expect(err).NotTo(HaveOccurred()) Expect(c.Enabled).To(BeTrue()) Expect(c.OwnershipMode).To(Equal("local-owner")) @@ -80,7 +81,7 @@ var _ = Describe("Add-on owner flags", Serial, func() { It("lets an explicit named-user win over the auth-off default", func() { r, err := parse("--addons-ownership-mode=named-user") Expect(err).NotTo(HaveOccurred()) - c, err := addonConfigFor(r.AddonsEnabled, r.AddonsOwnershipMode, GinkgoT().TempDir(), false) + c, err := addonConfigFor(r.AddonsEnabled, r.AddonsOwnershipMode, GinkgoT().TempDir(), false, addonRoleFlags{}) Expect(err).NotTo(HaveOccurred()) Expect(c).To(Equal(config.AddonConfig{})) }) @@ -98,7 +99,7 @@ var _ = Describe("Add-on owner flags", Serial, func() { r, err := parse("--addons-enabled=false") Expect(err).NotTo(HaveOccurred()) Expect(r.AddonsEnabled).To(BeFalse()) - c, err := addonConfigFor(r.AddonsEnabled, r.AddonsOwnershipMode, GinkgoT().TempDir(), true) + c, err := addonConfigFor(r.AddonsEnabled, r.AddonsOwnershipMode, GinkgoT().TempDir(), true, addonRoleFlags{}) Expect(err).NotTo(HaveOccurred()) Expect(c).To(Equal(config.AddonConfig{})) Expect(c.Transport).To(BeNil()) @@ -114,20 +115,149 @@ var _ = Describe("Add-on owner flags", Serial, func() { It("refuses an unknown ownership mode", func() { r, err := parse("--addons-ownership-mode=everyone") Expect(err).NotTo(HaveOccurred()) - _, err = addonConfigFor(r.AddonsEnabled, r.AddonsOwnershipMode, GinkgoT().TempDir(), true) + _, err = addonConfigFor(r.AddonsEnabled, r.AddonsOwnershipMode, GinkgoT().TempDir(), true, addonRoleFlags{}) Expect(err).To(MatchError(ContainSubstring("--addons-ownership-mode"))) }) It("accepts local-owner without authentication", func() { - c, err := addonConfigFor(true, "local-owner", GinkgoT().TempDir(), false) + c, err := addonConfigFor(true, "local-owner", GinkgoT().TempDir(), false, addonRoleFlags{}) Expect(err).NotTo(HaveOccurred()) Expect(c.Enabled).To(BeTrue()) Expect(c.OwnershipMode).To(Equal("local-owner")) }) It("keeps named-user off, without failing, when authentication is off", func() { - c, err := addonConfigFor(true, "named-user", GinkgoT().TempDir(), false) + c, err := addonConfigFor(true, "named-user", GinkgoT().TempDir(), false, addonRoleFlags{}) Expect(err).NotTo(HaveOccurred()) Expect(c.Enabled).To(BeFalse()) }) + Context("roles", func() { + const ownerAddress = "https://owner.example:8443" + var keys string + BeforeEach(func() { + keys = filepath.Join(GinkgoT().TempDir(), "keys") + Expect(gateway.GenerateCarrierKeys(keys, []string{"owner.example"})).To(Succeed()) + }) + resolve := func(dataPath string, args ...string) (config.AddonConfig, error) { + r, err := parse(args...) + Expect(err).NotTo(HaveOccurred()) + return addonConfigFor(r.AddonsEnabled, r.AddonsOwnershipMode, dataPath, true, addonRoleFlags{Role: r.AddonsRole, OwnerAddress: r.AddonsOwnerAddress, OwnerListen: r.AddonsOwnerListen, CarrierDir: r.AddonsCarrierDir}) + } + + It("defaults to the combined role", func() { + r, err := parse() + Expect(err).NotTo(HaveOccurred()) + Expect(r.AddonsRole).To(Equal("combined")) + Expect(r.AddonsOwnerAddress).To(BeEmpty()) + Expect(r.AddonsCarrierDir).To(BeEmpty()) + }) + + It("keeps the combined output of a single node", func() { + dir := GinkgoT().TempDir() + explicit, err := resolve(dir, "--addons-role=combined") + Expect(err).NotTo(HaveOccurred()) + plain, err := addonConfigFor(true, "", dir, true, addonRoleFlags{}) + Expect(err).NotTo(HaveOccurred()) + // The carrier keys are made fresh at every call; the rest is equal. + Expect(explicit.Transport).NotTo(BeNil()) + explicit.Transport, plain.Transport = nil, nil + Expect(explicit).To(Equal(plain)) + Expect(plain.InstanceID).To(Equal("local")) + Expect(plain.OwnerID).To(Equal("local")) + Expect(plain.Root).To(Equal(filepath.Join(dir, "addons"))) + Expect(plain.OwnerAddress).To(Equal("https://" + plain.ListenAddress)) + }) + + It("gives the owner role its own root, identity and listen address", func() { + dir := GinkgoT().TempDir() + c, err := resolve(dir, "--addons-role=owner", "--addons-owner-address="+ownerAddress, "--addons-carrier-dir="+keys) + Expect(err).NotTo(HaveOccurred()) + Expect(c.Enabled).To(BeTrue()) + Expect(c.InstanceID).To(Equal("local")) + Expect(c.OwnerID).To(Equal("local")) + Expect(c.OwnerAddress).To(Equal(ownerAddress)) + Expect(c.Root).To(Equal(filepath.Join(dir, "addons"))) + Expect(c.ListenAddress).To(Equal(":8443")) + Expect(c.StorageID).To(Equal("local")) + Expect(c.Transport).NotTo(BeNil()) + Expect(c.Transport.InstanceID).To(Equal(c.InstanceID)) + Expect(c.Transport.OwnerAddress).To(Equal(ownerAddress)) + Expect(c.Transport.VerificationKeys).To(HaveKey("frontend")) + }) + + It("builds no carrier for a frontend with add-ons off, whatever the role flags say", func() { + c, err := resolve(GinkgoT().TempDir(), "--addons-enabled=false", "--addons-role=frontend") + Expect(err).NotTo(HaveOccurred()) + Expect(c.Enabled).To(BeFalse()) + Expect(c.Transport).To(BeNil()) + Expect(c).To(Equal(config.AddonConfig{})) + // A set flag is ignored, and a key directory that does not exist is + // never read. + c, err = resolve(GinkgoT().TempDir(), "--addons-enabled=false", "--addons-role=frontend", "--addons-owner-address="+ownerAddress, "--addons-carrier-dir=/does/not/exist") + Expect(err).NotTo(HaveOccurred()) + Expect(c).To(Equal(config.AddonConfig{})) + }) + + It("lets the owner listen on another address", func() { + c, err := resolve(GinkgoT().TempDir(), "--addons-role=owner", "--addons-owner-address="+ownerAddress, "--addons-carrier-dir="+keys, "--addons-owner-listen=127.0.0.1:9443") + Expect(err).NotTo(HaveOccurred()) + Expect(c.ListenAddress).To(Equal("127.0.0.1:9443")) + }) + + It("gives the frontend role no listener and creates no addons directory", func() { + dir := GinkgoT().TempDir() + c, err := resolve(dir, "--addons-role=frontend", "--addons-owner-address="+ownerAddress, "--addons-carrier-dir="+keys) + Expect(err).NotTo(HaveOccurred()) + Expect(c.Enabled).To(BeTrue()) + Expect(c.InstanceID).To(Equal("frontend")) + Expect(c.OwnerID).To(Equal("local")) + Expect(c.OwnerAddress).To(Equal(ownerAddress)) + // A frontend has no state root. + Expect(c.Root).To(BeEmpty()) + Expect(c.ListenAddress).To(BeEmpty()) + Expect(c.Transport).NotTo(BeNil()) + Expect(c.Transport.SigningKey).NotTo(BeEmpty()) + Expect(filepath.Join(dir, "addons")).NotTo(BeADirectory()) + entries, err := os.ReadDir(dir) + Expect(err).NotTo(HaveOccurred()) + Expect(entries).To(BeEmpty()) + }) + + It("reads the role flags from the environment", func() { + Expect(os.Setenv("LOCALAI_ADDONS_ROLE", "frontend")).To(Succeed()) + Expect(os.Setenv("LOCALAI_ADDONS_OWNER_ADDRESS", ownerAddress)).To(Succeed()) + Expect(os.Setenv("LOCALAI_ADDONS_CARRIER_DIR", keys)).To(Succeed()) + c, err := resolve(GinkgoT().TempDir()) + Expect(err).NotTo(HaveOccurred()) + Expect(c.InstanceID).To(Equal("frontend")) + }) + + DescribeTable("refuses a bad role setup and names the flag", + func(flag string, args ...string) { + _, err := resolve(GinkgoT().TempDir(), args...) + Expect(err).To(MatchError(ContainSubstring(flag))) + }, + Entry("unknown role", "--addons-role", "--addons-role=worker"), + Entry("owner without an address", "--addons-owner-address", "--addons-role=owner", "--addons-carrier-dir=/keys"), + Entry("owner without a key directory", "--addons-carrier-dir", "--addons-role=owner", "--addons-owner-address="+ownerAddress), + Entry("frontend without an address", "--addons-owner-address", "--addons-role=frontend", "--addons-carrier-dir=/keys"), + Entry("frontend without a key directory", "--addons-carrier-dir", "--addons-role=frontend", "--addons-owner-address="+ownerAddress), + Entry("combined with an address", "--addons-owner-address", "--addons-owner-address="+ownerAddress), + Entry("combined with a key directory", "--addons-carrier-dir", "--addons-carrier-dir=/keys"), + Entry("an address that is not https", "--addons-owner-address", "--addons-role=frontend", "--addons-owner-address=http://owner.example:8443", "--addons-carrier-dir=/keys"), + Entry("an address with a path", "--addons-owner-address", "--addons-role=frontend", "--addons-owner-address=https://owner.example:8443/addons", "--addons-carrier-dir=/keys"), + Entry("an address without a port", "--addons-owner-address", "--addons-role=owner", "--addons-owner-address=https://owner.example", "--addons-carrier-dir=/keys"), + ) + + It("names the key directory when it holds no key set", func() { + _, err := resolve(GinkgoT().TempDir(), "--addons-role=frontend", "--addons-owner-address="+ownerAddress, "--addons-carrier-dir="+GinkgoT().TempDir()) + Expect(err).To(MatchError(ContainSubstring("--addons-carrier-dir"))) + }) + + It("ignores the role when add-ons are off", func() { + c, err := resolve(GinkgoT().TempDir(), "--addons-enabled=false", "--addons-role=worker") + Expect(err).NotTo(HaveOccurred()) + Expect(c).To(Equal(config.AddonConfig{})) + }) + }) }) diff --git a/core/cli/util.go b/core/cli/util.go index 881dc9141577..7f26567261ea 100644 --- a/core/cli/util.go +++ b/core/cli/util.go @@ -21,10 +21,11 @@ import ( ) type UtilCMD struct { - GGUFInfo GGUFInfoCMD `cmd:"" name:"gguf-info" help:"Get information about a GGUF file"` - CreateOCIImage CreateOCIImageCMD `cmd:"" name:"create-oci-image" help:"Create an OCI image from a file or a directory"` - HFScan HFScanCMD `cmd:"" name:"hf-scan" help:"Checks installed models for known security issues. WARNING: this is a best-effort feature and may not catch everything!"` - UsecaseHeuristic UsecaseHeuristicCMD `cmd:"" name:"usecase-heuristic" help:"Checks a specific model config and prints what usecase LocalAI will offer for it."` + GGUFInfo GGUFInfoCMD `cmd:"" name:"gguf-info" help:"Get information about a GGUF file"` + CreateOCIImage CreateOCIImageCMD `cmd:"" name:"create-oci-image" help:"Create an OCI image from a file or a directory"` + HFScan HFScanCMD `cmd:"" name:"hf-scan" help:"Checks installed models for known security issues. WARNING: this is a best-effort feature and may not catch everything!"` + UsecaseHeuristic UsecaseHeuristicCMD `cmd:"" name:"usecase-heuristic" help:"Checks a specific model config and prints what usecase LocalAI will offer for it."` + AddonsCarrier AddonsCarrierKeysCMD `cmd:"" name:"addons-carrier-keys" help:"Generate the key set that a separate add-on owner and its frontends share"` } type GGUFInfoCMD struct { diff --git a/core/cli/util_addons.go b/core/cli/util_addons.go new file mode 100644 index 000000000000..a1eaf2fa2d02 --- /dev/null +++ b/core/cli/util_addons.go @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: MIT +package cli + +import ( + "fmt" + "io" + "os" + + cliContext "github.com/mudler/LocalAI/core/cli/context" + "github.com/mudler/LocalAI/core/services/addons/gateway" +) + +// AddonsCarrierKeysCMD makes the static key set that a split add-on owner and +// its frontends share. +type AddonsCarrierKeysCMD struct { + Out string `required:"" type:"path" name:"out" help:"Directory to write the key set to. It must not hold any of the files yet"` + OwnerHost []string `required:"" sep:"none" name:"owner-host" help:"DNS name or IP address of the add-on owner, as the frontends dial it. Repeat the flag for more than one"` + + // output receives the summary. It is stdout when nil. + output io.Writer +} + +func (c *AddonsCarrierKeysCMD) Run(ctx *cliContext.Context) error { + if err := gateway.GenerateCarrierKeys(c.Out, c.OwnerHost); err != nil { + return fmt.Errorf("--out: %w", err) + } + w := c.output + if w == nil { + w = os.Stdout + } + _, err := fmt.Fprintf(w, `Wrote the add-on carrier key set to %[1]s. + +Mount on the add-on owner (--addons-role=owner), and nowhere else: + ca.pem owner.pem owner.key frontend.pem frontend-sign.pub + +Mount on every frontend (--addons-role=frontend): + ca.pem owner.pem frontend.pem frontend.key frontend-sign.key + +Point --addons-carrier-dir at the directory that holds them. Store the set as a +secret: the frontend files together let a holder act as a frontend. The CA key +was not kept. +`, c.Out) + return err +} diff --git a/core/cli/util_addons_test.go b/core/cli/util_addons_test.go new file mode 100644 index 000000000000..cc2932e716b3 --- /dev/null +++ b/core/cli/util_addons_test.go @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: MIT +package cli + +import ( + "bytes" + "os" + "path/filepath" + + "github.com/alecthomas/kong" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Add-on carrier keys command", func() { + parse := func(args ...string) (*AddonsCarrierKeysCMD, error) { + command := struct { + Keys AddonsCarrierKeysCMD `cmd:"" name:"addons-carrier-keys"` + }{} + p, err := kong.New(&command) + if err != nil { + return nil, err + } + _, err = p.Parse(append([]string{"addons-carrier-keys"}, args...)) + return &command.Keys, err + } + + It("writes the carrier keys and says which files go where", func() { + out := filepath.Join(GinkgoT().TempDir(), "keys") + cmd, err := parse("--out", out, "--owner-host", "owner.example", "--owner-host", "192.0.2.5") + Expect(err).NotTo(HaveOccurred()) + var buf bytes.Buffer + cmd.output = &buf + Expect(cmd.Run(nil)).To(Succeed()) + for _, name := range []string{"ca.pem", "owner.pem", "owner.key", "frontend.pem", "frontend.key", "frontend-sign.key", "frontend-sign.pub"} { + Expect(filepath.Join(out, name)).To(BeARegularFile()) + } + Expect(buf.String()).To(ContainSubstring("--addons-role=owner")) + Expect(buf.String()).To(ContainSubstring("--addons-role=frontend")) + Expect(buf.String()).NotTo(ContainSubstring("BEGIN")) + }) + + It("fails when the directory already holds a key set", func() { + out := filepath.Join(GinkgoT().TempDir(), "keys") + first, err := parse("--out", out, "--owner-host", "owner.example") + Expect(err).NotTo(HaveOccurred()) + first.output = &bytes.Buffer{} + Expect(first.Run(nil)).To(Succeed()) + before, err := os.ReadFile(filepath.Join(out, "ca.pem")) + Expect(err).NotTo(HaveOccurred()) + second, err := parse("--out", out, "--owner-host", "owner.example") + Expect(err).NotTo(HaveOccurred()) + Expect(second.Run(nil)).To(MatchError(ContainSubstring("already exists"))) + after, err := os.ReadFile(filepath.Join(out, "ca.pem")) + Expect(err).NotTo(HaveOccurred()) + Expect(after).To(Equal(before)) + }) + + It("needs --owner-host and --out", func() { + _, err := parse("--out", filepath.Join(GinkgoT().TempDir(), "keys")) + Expect(err).To(MatchError(ContainSubstring("owner-host"))) + _, err = parse("--owner-host", "owner.example") + Expect(err).To(MatchError(ContainSubstring("out"))) + }) +}) diff --git a/core/config/application_config.go b/core/config/application_config.go index 6fba0345ab4e..154a54b355e0 100644 --- a/core/config/application_config.go +++ b/core/config/application_config.go @@ -22,6 +22,10 @@ type AddonConfig struct { Transport *gateway.CarrierConfig `json:"-" yaml:"-"` ListenAddress string + // Frontend is set for the frontend role, also when add-ons are off. A + // frontend keeps no owner state, so it never opens the library store. + Frontend bool + Enabled bool InstanceID, OwnerID, OwnerAddress, StorageID, Root string SharedStorage bool @@ -66,6 +70,12 @@ type AddonConfig struct { ArtifactExpiry time.Duration } +// IsFrontend says whether this process is an add-on frontend: the role was set, +// or the instance is not the owner it forwards to. +func (c AddonConfig) IsFrontend() bool { + return c.Frontend || (c.Enabled && c.OwnerID != "" && c.InstanceID != c.OwnerID) +} + func WithAddons(c AddonConfig) AppOption { return func(o *ApplicationConfig) { o.Addons = c } } diff --git a/core/http/app.go b/core/http/app.go index fa813f72d3a0..319a400adfba 100644 --- a/core/http/app.go +++ b/core/http/app.go @@ -482,9 +482,19 @@ func API(application *application.Application) (*echo.Echo, error) { // Register auth routes (login, callback, API keys, user management) routes.RegisterAuthRoutes(e, application) - routes.RegisterAddonRoutes(e, application) - if err := routes.RegisterAddonNavigationRoutes(e, application); err != nil { - return nil, fmt.Errorf("register the add-on navigation routes: %w", err) + // With add-ons off, no real add-on, fine-tuning or quantization handler is + // registered. One handler per prefix answers the same unavailable response + // whether or not the path exists, so the off state does not reveal the route + // list. With add-ons on, the real routes sit behind the same gate. + addonsOn := application.ApplicationConfig().Addons.Enabled + if addonsOn { + gate := httpMiddleware.RequireAddonsFeature(application) + routes.RegisterAddonRoutes(e, application, gate) + if err := routes.RegisterAddonNavigationRoutes(e, application, gate); err != nil { + return nil, fmt.Errorf("register the add-on navigation routes: %w", err) + } + } else { + routes.RegisterAddonsUnavailable(e, "/api/addons") } // Register routing-module usage endpoints. Unlike /api/auth/usage @@ -521,64 +531,69 @@ func API(application *application.Application) (*echo.Echo, error) { chatHistoryMw := auth.RequireFeature(application.AuthDB(), auth.FeatureChatHistory) routes.RegisterLocalAIRoutes(e, requestExtractor, application.ModelConfigLoader(), application.ModelLoader(), application.ApplicationConfig(), application.GalleryService(), opcache, application.TemplatesEvaluator(), application, adminMiddleware, mcpJobsMw, mcpMw, chatHistoryMw) routes.RegisterAgentPoolRoutes(e, application, agentsMw, skillsMw, collectionsMw) - // Fine-tuning routes - fineTuningMw := auth.RequireFeature(application.AuthDB(), auth.FeatureFineTuning) - // In distributed mode pass the shared NATS client + PostgreSQL store so - // fine-tune jobs stay consistent across replicas (the SyncedMap broadcasts - // mutations and hydrates from the DB); standalone passes nil for both. - var ftNats messaging.MessagingClient - var ftStore *distributed.FineTuneStore - if d := application.Distributed(); d != nil { - ftNats = d.Nats - if d.DistStores != nil && d.DistStores.FineTune != nil { - ftStore = d.DistStores.FineTune + if addonsOn { + // Fine-tuning routes + fineTuningMw := auth.RequireFeature(application.AuthDB(), auth.FeatureFineTuning) + // In distributed mode pass the shared NATS client + PostgreSQL store so + // fine-tune jobs stay consistent across replicas (the SyncedMap broadcasts + // mutations and hydrates from the DB); standalone passes nil for both. + var ftNats messaging.MessagingClient + var ftStore *distributed.FineTuneStore + if d := application.Distributed(); d != nil { + ftNats = d.Nats + if d.DistStores != nil && d.DistStores.FineTune != nil { + ftStore = d.DistStores.FineTune + } } - } - ftService := finetune.NewFineTuneService( - application.ApplicationConfig(), - application.ModelLoader(), - application.ModelConfigLoader(), - ftNats, - ftStore, - ) - routes.RegisterFineTuningRoutes(e, ftService, application.ApplicationConfig(), application, fineTuningMw) - - // Quantization routes - quantizationMw := auth.RequireFeature(application.AuthDB(), auth.FeatureQuantization) - // In distributed mode pass the shared NATS client + PostgreSQL store so - // quantization jobs stay consistent across replicas (the SyncedMap broadcasts - // mutations and hydrates from the DB); standalone passes nil for both. - var quantNats messaging.MessagingClient - var quantStore *distributed.QuantStore - if d := application.Distributed(); d != nil { - quantNats = d.Nats - if d.DistStores != nil && d.DistStores.Quant != nil { - quantStore = d.DistStores.Quant + ftService := finetune.NewFineTuneService( + application.ApplicationConfig(), + application.ModelLoader(), + application.ModelConfigLoader(), + ftNats, + ftStore, + ) + routes.RegisterFineTuningRoutes(e, ftService, application.ApplicationConfig(), application, fineTuningMw) + + // Quantization routes + quantizationMw := auth.RequireFeature(application.AuthDB(), auth.FeatureQuantization) + // In distributed mode pass the shared NATS client + PostgreSQL store so + // quantization jobs stay consistent across replicas (the SyncedMap broadcasts + // mutations and hydrates from the DB); standalone passes nil for both. + var quantNats messaging.MessagingClient + var quantStore *distributed.QuantStore + if d := application.Distributed(); d != nil { + quantNats = d.Nats + if d.DistStores != nil && d.DistStores.Quant != nil { + quantStore = d.DistStores.Quant + } } - } - qService := quantization.NewQuantizationService( - application.ApplicationConfig(), - application.ModelLoader(), - application.ModelConfigLoader(), - quantNats, - quantStore, - ) - routes.RegisterQuantizationRoutes(e, qService, application.ApplicationConfig(), application, quantizationMw) - - // The add-on host operations reach the same services as the routes. Only the - // owner of an add-on host has a broker, so elsewhere this registers nothing. - engineBackends := localai.EngineBackends{AppConfig: application.ApplicationConfig(), Capabilities: routes.ClusterCapabilityProviderFor(application), Installed: routes.ClusterInstalledProviderFor(application)} - if err := application.RegisterAddonEngineJobs(localai.FineTuneJobs{Svc: ftService}, localai.QuantizationJobs{Svc: qService}, engineBackends); err != nil { - return nil, err - } - if err := application.RegisterAddonEngineProgress(localai.FineTuneProgress{Svc: ftService}, localai.QuantizationProgress{Svc: qService}); err != nil { - return nil, err - } - if err := application.RegisterAddonEngineModels(localai.FineTuneModels{Svc: ftService}, localai.QuantizationModels{Svc: qService}); err != nil { - return nil, err - } - if err := application.RegisterAddonEngineDatasets(localai.NewFineTuneDatasets(ftService)); err != nil { - return nil, err + qService := quantization.NewQuantizationService( + application.ApplicationConfig(), + application.ModelLoader(), + application.ModelConfigLoader(), + quantNats, + quantStore, + ) + routes.RegisterQuantizationRoutes(e, qService, application.ApplicationConfig(), application, quantizationMw) + + // The add-on host operations reach the same services as the routes. Only the + // owner of an add-on host has a broker, so elsewhere this registers nothing. + engineBackends := localai.EngineBackends{AppConfig: application.ApplicationConfig(), Capabilities: routes.ClusterCapabilityProviderFor(application), Installed: routes.ClusterInstalledProviderFor(application)} + if err := application.RegisterAddonEngineJobs(localai.FineTuneJobs{Svc: ftService}, localai.QuantizationJobs{Svc: qService}, engineBackends); err != nil { + return nil, err + } + if err := application.RegisterAddonEngineProgress(localai.FineTuneProgress{Svc: ftService}, localai.QuantizationProgress{Svc: qService}); err != nil { + return nil, err + } + if err := application.RegisterAddonEngineModels(localai.FineTuneModels{Svc: ftService}, localai.QuantizationModels{Svc: qService}); err != nil { + return nil, err + } + if err := application.RegisterAddonEngineDatasets(localai.NewFineTuneDatasets(ftService)); err != nil { + return nil, err + } + } else { + routes.RegisterAddonsUnavailable(e, "/api/fine-tuning") + routes.RegisterAddonsUnavailable(e, "/api/quantization") } // Node management routes (distributed mode) diff --git a/core/http/auth/features.go b/core/http/auth/features.go index 459ee1df2295..d4b3d3fbf5b8 100644 --- a/core/http/auth/features.go +++ b/core/http/auth/features.go @@ -183,6 +183,7 @@ var RouteFeatureRegistry = []RouteFeature{ {"POST", "/stores/find", FeatureStores}, // Fine-tuning + {"GET", "/api/fine-tuning/status", FeatureFineTuning}, {"POST", "/api/fine-tuning/jobs", FeatureFineTuning}, {"GET", "/api/fine-tuning/jobs", FeatureFineTuning}, {"GET", "/api/fine-tuning/jobs/:id", FeatureFineTuning}, @@ -208,6 +209,7 @@ var RouteFeatureRegistry = []RouteFeature{ {"DELETE", "/api/conversations/:id", FeatureChatHistory}, // Quantization + {"GET", "/api/quantization/status", FeatureQuantization}, {"POST", "/api/quantization/jobs", FeatureQuantization}, {"GET", "/api/quantization/jobs", FeatureQuantization}, {"GET", "/api/quantization/jobs/:id", FeatureQuantization}, diff --git a/core/http/endpoints/localai/addon_export_status.go b/core/http/endpoints/localai/addon_export_status.go new file mode 100644 index 000000000000..d151f016518c --- /dev/null +++ b/core/http/endpoints/localai/addon_export_status.go @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: MIT +package localai + +import ( + "net/http" + + "github.com/labstack/echo/v4" + httpMiddleware "github.com/mudler/LocalAI/core/http/middleware" +) + +// ModelExportsStatus is the body of the export status routes. +type ModelExportsStatus struct { + // ModelExports is false only when this instance does not see the models + // path of the add-on owner. It is the same value as the model_exports key + // of GET /api/features. + ModelExports bool `json:"model_exports"` +} + +// ModelExportsStatusEndpoint answers whether this instance may write an +// exported or imported model. The add-on UIs read it to disable their export +// controls before a user hits the refusal of the export route. +// +// @Summary Report whether model exports are allowed +// @Description Returns model_exports=false when this instance does not share the models path of the add-on owner. The value is the same as the model_exports key of GET /api/features. +// @Tags fine-tuning, quantization +// @Produce json +// @Success 200 {object} localai.ModelExportsStatus +// @Failure 503 {object} map[string]any "add-ons are off or the owner is not reachable" +// @Router /api/fine-tuning/status [get] +func ModelExportsStatusEndpoint(gate httpMiddleware.ModelExportsGate) echo.HandlerFunc { + return func(c echo.Context) error { + allowed := gate == nil || gate.ModelExportsAllowed(c.Request().Context()) + return c.JSON(http.StatusOK, ModelExportsStatus{ModelExports: allowed}) + } +} + +// ModelExportsStatusQuantizationEndpoint serves the same answer on the +// quantization prefix; it exists only to carry its own swagger route. +// +// @Summary Report whether model imports are allowed +// @Description Returns model_exports=false when this instance does not share the models path of the add-on owner. The value is the same as the model_exports key of GET /api/features. +// @Tags fine-tuning, quantization +// @Produce json +// @Success 200 {object} localai.ModelExportsStatus +// @Failure 503 {object} map[string]any "add-ons are off or the owner is not reachable" +// @Router /api/quantization/status [get] +func ModelExportsStatusQuantizationEndpoint(gate httpMiddleware.ModelExportsGate) echo.HandlerFunc { + return ModelExportsStatusEndpoint(gate) +} diff --git a/core/http/endpoints/localai/addon_export_status_test.go b/core/http/endpoints/localai/addon_export_status_test.go new file mode 100644 index 000000000000..285a25d50aec --- /dev/null +++ b/core/http/endpoints/localai/addon_export_status_test.go @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: MIT +package localai_test + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + + "github.com/labstack/echo/v4" + "github.com/mudler/LocalAI/core/http/endpoints/localai" + httpMiddleware "github.com/mudler/LocalAI/core/http/middleware" + "github.com/mudler/LocalAI/core/services/addons/gateway" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// exportGate stands for the application: the add-on gate and the models path +// check, the two questions the status routes depend on. +type exportGate struct{ addons, exports bool } + +func (g exportGate) AddonsFeature(context.Context) bool { return g.addons } +func (g exportGate) AddonsEnabled() bool { return g.addons } +func (g exportGate) ModelExportsAllowed(context.Context) bool { return g.exports } + +var _ = Describe("Export status routes", func() { + call := func(g exportGate, path string) *httptest.ResponseRecorder { + e := echo.New() + gate := httpMiddleware.RequireAddonsFeature(g) + e.GET("/api/fine-tuning/status", localai.ModelExportsStatusEndpoint(g), gate) + e.GET("/api/quantization/status", localai.ModelExportsStatusQuantizationEndpoint(g), gate) + w := httptest.NewRecorder() + e.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) + return w + } + + for _, path := range []string{"/api/fine-tuning/status", "/api/quantization/status"} { + It("reports false when the models probe fails on "+path, func() { + w := call(exportGate{addons: true, exports: false}, path) + Expect(w.Code).To(Equal(http.StatusOK)) + var got map[string]bool + Expect(json.Unmarshal(w.Body.Bytes(), &got)).To(Succeed()) + Expect(got).To(Equal(map[string]bool{"model_exports": false})) + }) + + It("reports true when the models probe passes on "+path, func() { + w := call(exportGate{addons: true, exports: true}, path) + Expect(w.Code).To(Equal(http.StatusOK)) + var got map[string]bool + Expect(json.Unmarshal(w.Body.Bytes(), &got)).To(Succeed()) + Expect(got).To(Equal(map[string]bool{"model_exports": true})) + }) + + It("answers the typed 503 when add-ons are off on "+path, func() { + w := call(exportGate{addons: false, exports: true}, path) + Expect(w.Code).To(Equal(http.StatusServiceUnavailable)) + Expect(w.Body.String()).To(ContainSubstring(gateway.ReasonAddonsOff)) + Expect(w.Body.String()).To(ContainSubstring(`"code":"unavailable"`)) + }) + } +}) diff --git a/core/http/endpoints/localai/api_instructions.go b/core/http/endpoints/localai/api_instructions.go index e6c1dd217b51..0164b6762487 100644 --- a/core/http/endpoints/localai/api_instructions.go +++ b/core/http/endpoints/localai/api_instructions.go @@ -25,9 +25,10 @@ type instructionDef struct { var instructionDefs = []instructionDef{ {Name: "motion", Description: "Live human pose capture with Protobuf over WebSocket.", Tags: []string{"motion"}, Intro: "Create /api/motion/sessions and negotiate localai.motion.v2 on /poses for binary Protobuf RGB uploads and pose outputs with JSON flow credits. Browsers can POST /api/motion/sessions/{id}/tickets for a single-use 30-second WebSocket ticket. No SONIC controller or offline export."}, - {Name: "addons", Description: "Owner-routed addon lifecycle and response relays", Tags: []string{"Addons"}, Intro: "POST /api/addons/dispatch submits typed lifecycle operations or reads actor-bound operation-status. POST /api/addons/relay serves status and coarse lifecycle progress on the receiving frontend. Named authentication and the addons feature are required; lifecycle operations require administration. Preserve operationId after outcome_unknown and poll status, never submit a new mutation. An add-on that refuses a request as invalid answers 400 invalid_request from every route, never an outage: nothing was applied, so correct the request instead of repeating it. Resource and output routes deny access until host admission and resolution providers are installed. Data and schema mutations are stored proposals: POST /api/addons/proposals previews and stores one (nothing is applied), GET /api/addons/proposals/{id} reads it, GET /api/addons/proposals?state=pending lists the caller's own pending proposals (newest first, at most 32; where a proposal made by a chat tool, an agent or a scheduled job waits for the proposer's own browser session), POST .../cancel withdraws one and POST .../reconcile resolves an unknown outcome by receipt lookup, and when no receipt can be found POST .../resolve lets the resource owner's own browser session abandon it with a reason (same confirmation-surface rules as confirm; nothing is applied again). The owner role finds what to resolve with GET /api/addons/resources/{resourceId}/proposals?state=unresolved (same rules, X-LocalAI-Confirm equals the resource ID): it lists the proposal ID, proposer, effects, digest and age of every unresolved proposal on the resource, never its arguments. A write blocked by an unresolved outcome is denied (403) and, for the owner role only, names it as outcome_blocked with blockedBy. POST .../confirm applies exactly the stored operation once and is accepted only from the proposer's own browser session (same-origin fetch, X-LocalAI-Confirm header equal to the proposal ID, digest of what was shown); API keys, MCP sessions and tool or agent calls can propose but never confirm, and no MCP tool maps to confirm. Sharing and database deletion are never proposals. A read that returns a file result gives private artifact IDs, never addon handles: GET /api/addons/artifacts/{id} returns the transfer state, name, size and expiry, and GET /api/addons/artifacts/{id}/download streams the complete file as an attachment. Only the requesting principal can read or download an artifact, an unknown, foreign or expired ID is the same 403, access to every source resource is checked again at completion and while streaming, and a disabled addon denies downloads until it is enabled again. No MCP tool maps to these routes. The Apps surface: GET /api/addons/apps lists the installed, enabled addons any caller with the addons feature may use, with title, description, icon URL, version, generation, state (running, starting, failed or stopped; an observation separate from enabled), a main page path only when the manifest declares a valid one, and a bounded tool summary; administrators also see disabled addons and per-app controls. GET /api/addons/apps/{id}/icon?generation=N serves the verified icon of a visible app. POST /api/addons/apps/{id}/start and .../stop are administrator-only, take {operationId, generation} with the generation that was listed, deny a stale generation and keep the lifecycle rules (preserve operationId after outcome_unknown). No MCP tool maps to the Apps routes. The listed ui entry of an app gives the same-origin URLs of its UI code: GET /api/addons/{id}/assets/{digest}/{path} streams one declared file (the entry module or a style sheet) of the current installed package, relayed from the owner, with Cache-Control private, max-age=31536000, immutable and nosniff. The digest is the package sha256 in lower-case hex; a file the manifest does not declare, a traversal, another addon, an older package and a disabled or uninstalled addon are all one 403, never cached. A credential in the URL is never accepted; the browser uses its same-origin session. No MCP tool maps to the asset route. Management (administrators only, through POST /api/addons/dispatch): the reads catalog-search, catalog-details, catalog-sources and addons-installed return catalog entries, one resolved entry, the configured libraries with their last fetch state and the installed addons with verified package facts, health, retained generations and storage; catalog-source-save stores or removes a library by reference only (a secret in place of a reference is denied) and catalog-source-refresh fetches one again; upgrade installs a newer release of an installed addon. Removing a library never touches installed addons. The installed view has an update object computed by the owner's update check (state, candidate version and digest, check and last-success time, reasons, redacted error class); an add-on the owner cannot answer for has none, and a failed check is never up-to-date. No MCP tool maps to these actions. UI-only resource management (never tools; no MCP tool maps to any of it): GET /api/addons/resources?addonId=tables lists the resources the caller owns or that were shared with the caller, with the caller's own role, the display name, the owner's display name for a shared one and, for the owner role only, the number of people it is shared with and the revision. POST /api/addons/resources creates one by the person's own action (operationId makes it idempotent; the arguments are the add-on's create tool arguments), POST .../{resourceId}/rename and .../delete rename and delete it (the owner role only; generation and revision must be the listed ones or the call is 409 stale; delete needs confirm equal to the display name, removes the data and every grant and never uninstalls the add-on), GET .../{resourceId}/grants, PUT/DELETE .../grants/{userId} and GET .../{resourceId}/users?q= list, set (viewer or editor, whole resource) and remove grants and search the people it can be shared with (immutable account ID and display name only; local-owner mode answers 409 sharing_disabled). All of them accept only the owner's own browser session with the X-LocalAI-Confirm header equal to the resource ID (the operation ID for a creation); API keys are refused. POST .../{resourceId}/apply changes the data or the schema by the person's own action with no card, through a write or schema tool the add-on declares ({operationId, generation, revision, tool, arguments} with the revision the caller read; editors write rows, only the owner changes the schema; 409 stale when the generation or revision moved; the same operationId with the same request is one change; reads, creations, rename, delete and undeclared tools are refused). GET .../resources/{resourceId}/operations/{operationId}?kind=rename|delete|apply reads the receipt of a rename, a deletion or a change after outcome_unknown, which carries the original operationId. GET/PUT/DELETE /api/addons/conversations/{conversationId}/selection/{addonId} keep the resource a conversation works on for the authenticated user; a chat tool call that names no resourceId uses it, authorized again by the owner on every call, and with none selected the tool answers an error result with code select_resource. Data recovery (administrators only, never tools; no MCP tool maps to any of it): GET /api/addons/{id}/recovery returns the recovery state of an add-on's data (ready, upgrading, restoring, needs-recovery or purging), its backups newest first with size, reason, mode and last verification result, the retention policy and the receipt of the last restore. POST /api/addons/{id}/recovery/backup makes a verified, consistent backup (the add-on is stopped while its SQLite files are copied with VACUUM INTO, admission is blocked and pending proposals are invalidated; nothing is left behind on failure; 503 insufficient_storage when it does not fit), .../verify reads one completely, .../restore restores one (the add-on must not be running, confirm is the add-on ID, mapping names the account that takes the data of a backup made in local-owner mode, the answer carries warnings and no share is given back that was removed since the backup or whose account is gone; 409 needs_recovery, addon_running, backup_corrupt, incompatible_format, identity_changed, mapping_required or downgrade_unsupported), .../delete-backup removes one (never the only verified copy, 409 sole_recovery_copy) and .../prune applies the retention policy. POST /api/addons/{id}/purge removes retained data by explicit scope (data, backups, recoveryCopies) with confirm equal to the add-on ID, for an add-on that is uninstalled or disabled; it is not a database deletion or an uninstall. Every mutation accepts only the administrator's own browser session (same-origin fetch, X-LocalAI-Confirm equal to the operationId); a repeated operationId returns the recorded result and never runs again; after outcome_unknown read the state and keep the operationId. An upgrade of an add-on that has data is backed up, migrated under exclusive ownership and published only when healthy; a failed migration that changes the data format leaves the add-on in needs-recovery with the previous package and the verified backup retained. Updates (never tools; no MCP tool maps to any of it): GET /api/addons/{id}/update reads the owner's update status of an installed add-on for every caller with the addons feature who may see it (administrators any installed add-on, others an enabled one; the same 403 otherwise). The state is unknown, up-to-date, update-available, incompatible (reasons: platform, host-version, api-version, protocol-version, sdk-version, data-format), check-failed (a redacted error class and lastSuccess, the time the library was last read; never shown as up-to-date) or integrity-error (an immutable version was listed again with another digest; never an update). A moving channel (development, a branch, latest) compares resolved digests, so a changed digest with the same version label is an update; an immutable version selects the highest compatible greater version of the same major. The library and the channel the add-on came from never change by themselves. Callers who are not administrators get the state, versions and times only. The route never uses the network. POST /api/addons/{id}/update/check (administrators, own browser session, X-LocalAI-Confirm equal to the operationId) fetches the library again and returns the status; it is rate limited and answers throttled true when it did not fetch, and it applies nothing. POST /api/addons/{id}/upgrade (administrators, own browser session) is the only request that applies an update: it carries operationId, expectedDigest (the installed package the caller saw) and candidateDigest (the candidate the check resolved), and the owner re-resolves, re-checks compatibility and refuses before it records anything with 409 stale_installed, candidate_changed, incompatible, integrity_error or upgrade_conflict (the same transition under another operationId); 503 check_failed when the library cannot be read. A failure keeps the last good generation or enters the recovery state; the previous installation is never deleted. The apps list marks an app update: update-pending from the owner's cached check only and never reaches a library."}, - {Name: "library-sources", Description: "Library sources of add-ons, backends and models", Tags: []string{"library"}, Intro: "Administrators only. GET /api/library/sources lists every source of the add-on, backend and model kinds, shipped defaults and removed defaults (tombstones) included, with kind, id, name, url, mirrors, enabled, default, removed, suspended, a conflict message and the last fetch state. POST /api/library/sources adds a source of your own ({kind: addon|backend|model, url, id, name, format: library|gallery}); the same kind and URL twice is a 409. POST /api/library/sources/{id}/enable and /disable turn one on or off (enabling a removed default brings it back). PUT /api/library/sources/{id} replaces the mirrors of a model or backend source ({mirrors: [..]}); POST also takes mirrors. Each mirror is an http(s), github:, oci:// or file:// address, at most 8. DELETE /api/library/sources/{id} removes a source; a shipped default stays as a tombstone so it does not return at the next start. The routes work on the store that `local-ai library` edits. The runtime setting official_libraries (default on) turns every shipped default off at once: they stay listed with suspended true. No MCP tool maps to these routes."}, + {Name: "addons", Description: "Owner-routed addon lifecycle and response relays", Tags: []string{"Addons"}, Intro: "POST /api/addons/dispatch submits typed lifecycle operations or reads actor-bound operation-status. POST /api/addons/relay serves status and coarse lifecycle progress on the receiving frontend. Named authentication and the addons feature are required; lifecycle operations require administration. Preserve operationId after outcome_unknown and poll status, never submit a new mutation. An add-on that refuses a request as invalid answers 400 invalid_request from every route, never an outage: nothing was applied, so correct the request instead of repeating it. Resource and output routes deny access until host admission and resolution providers are installed. Data and schema mutations are stored proposals: POST /api/addons/proposals previews and stores one (nothing is applied), GET /api/addons/proposals/{id} reads it, GET /api/addons/proposals?state=pending lists the caller's own pending proposals (newest first, at most 32; where a proposal made by a chat tool, an agent or a scheduled job waits for the proposer's own browser session), POST .../cancel withdraws one and POST .../reconcile resolves an unknown outcome by receipt lookup, and when no receipt can be found POST .../resolve lets the resource owner's own browser session abandon it with a reason (same confirmation-surface rules as confirm; nothing is applied again). The owner role finds what to resolve with GET /api/addons/resources/{resourceId}/proposals?state=unresolved (same rules, X-LocalAI-Confirm equals the resource ID): it lists the proposal ID, proposer, effects, digest and age of every unresolved proposal on the resource, never its arguments. A write blocked by an unresolved outcome is denied (403) and, for the owner role only, names it as outcome_blocked with blockedBy. POST .../confirm applies exactly the stored operation once and is accepted only from the proposer's own browser session (same-origin fetch, X-LocalAI-Confirm header equal to the proposal ID, digest of what was shown); API keys, MCP sessions and tool or agent calls can propose but never confirm, and no MCP tool maps to confirm. Sharing and database deletion are never proposals. A read that returns a file result gives private artifact IDs, never addon handles: GET /api/addons/artifacts/{id} returns the transfer state, name, size and expiry, and GET /api/addons/artifacts/{id}/download streams the complete file as an attachment. Only the requesting principal can read or download an artifact, an unknown, foreign or expired ID is the same 403, access to every source resource is checked again at completion and while streaming, and a disabled addon denies downloads until it is enabled again. Only GET /backends/library/updates has an MCP tool, list_backend_library_updates; no MCP tool maps to the other routes. The Apps surface: GET /api/addons/apps lists the installed, enabled addons any caller with the addons feature may use, with title, description, icon URL, version, generation, state (running, starting, failed or stopped; an observation separate from enabled), a main page path only when the manifest declares a valid one, and a bounded tool summary; administrators also see disabled addons and per-app controls. GET /api/addons/apps/{id}/icon?generation=N serves the verified icon of a visible app. POST /api/addons/apps/{id}/start and .../stop are administrator-only, take {operationId, generation} with the generation that was listed, deny a stale generation and keep the lifecycle rules (preserve operationId after outcome_unknown). No MCP tool maps to the Apps routes. The listed ui entry of an app gives the same-origin URLs of its UI code: GET /api/addons/{id}/assets/{digest}/{path} streams one declared file (the entry module or a style sheet) of the current installed package, relayed from the owner, with Cache-Control private, max-age=31536000, immutable and nosniff. The digest is the package sha256 in lower-case hex; a file the manifest does not declare, a traversal, another addon, an older package and a disabled or uninstalled addon are all one 403, never cached. A credential in the URL is never accepted; the browser uses its same-origin session. No MCP tool maps to the asset route. Management (administrators only, through POST /api/addons/dispatch): the reads catalog-search, catalog-details, catalog-sources and addons-installed return catalog entries, one resolved entry, the configured libraries with their last fetch state and the installed addons with verified package facts, health, retained generations and storage; catalog-source-save stores or removes a library by reference only (a secret in place of a reference is denied) and catalog-source-refresh fetches one again; upgrade installs a newer release of an installed addon. Removing a library never touches installed addons. The installed view has an update object computed by the owner's update check (state, candidate version and digest, check and last-success time, reasons, redacted error class); an add-on the owner cannot answer for has none, and a failed check is never up-to-date. No MCP tool maps to these actions. UI-only resource management (never tools; no MCP tool maps to any of it): GET /api/addons/resources?addonId=tables lists the resources the caller owns or that were shared with the caller, with the caller's own role, the display name, the owner's display name for a shared one and, for the owner role only, the number of people it is shared with and the revision. POST /api/addons/resources creates one by the person's own action (operationId makes it idempotent; the arguments are the add-on's create tool arguments), POST .../{resourceId}/rename and .../delete rename and delete it (the owner role only; generation and revision must be the listed ones or the call is 409 stale; delete needs confirm equal to the display name, removes the data and every grant and never uninstalls the add-on), GET .../{resourceId}/grants, PUT/DELETE .../grants/{userId} and GET .../{resourceId}/users?q= list, set (viewer or editor, whole resource) and remove grants and search the people it can be shared with (immutable account ID and display name only; local-owner mode answers 409 sharing_disabled). All of them accept only the owner's own browser session with the X-LocalAI-Confirm header equal to the resource ID (the operation ID for a creation); API keys are refused. POST .../{resourceId}/apply changes the data or the schema by the person's own action with no card, through a write or schema tool the add-on declares ({operationId, generation, revision, tool, arguments} with the revision the caller read; editors write rows, only the owner changes the schema; 409 stale when the generation or revision moved; the same operationId with the same request is one change; reads, creations, rename, delete and undeclared tools are refused). GET .../resources/{resourceId}/operations/{operationId}?kind=rename|delete|apply reads the receipt of a rename, a deletion or a change after outcome_unknown, which carries the original operationId. GET/PUT/DELETE /api/addons/conversations/{conversationId}/selection/{addonId} keep the resource a conversation works on for the authenticated user; a chat tool call that names no resourceId uses it, authorized again by the owner on every call, and with none selected the tool answers an error result with code select_resource. Data recovery (administrators only, never tools; no MCP tool maps to any of it): GET /api/addons/{id}/recovery returns the recovery state of an add-on's data (ready, upgrading, restoring, needs-recovery or purging), its backups newest first with size, reason, mode and last verification result, the retention policy and the receipt of the last restore. POST /api/addons/{id}/recovery/backup makes a verified, consistent backup (the add-on is stopped while its SQLite files are copied with VACUUM INTO, admission is blocked and pending proposals are invalidated; nothing is left behind on failure; 503 insufficient_storage when it does not fit), .../verify reads one completely, .../restore restores one (the add-on must not be running, confirm is the add-on ID, mapping names the account that takes the data of a backup made in local-owner mode, the answer carries warnings and no share is given back that was removed since the backup or whose account is gone; 409 needs_recovery, addon_running, backup_corrupt, incompatible_format, identity_changed, mapping_required or downgrade_unsupported), .../delete-backup removes one (never the only verified copy, 409 sole_recovery_copy) and .../prune applies the retention policy. POST /api/addons/{id}/purge removes retained data by explicit scope (data, backups, recoveryCopies) with confirm equal to the add-on ID, for an add-on that is uninstalled or disabled; it is not a database deletion or an uninstall. Every mutation accepts only the administrator's own browser session (same-origin fetch, X-LocalAI-Confirm equal to the operationId); a repeated operationId returns the recorded result and never runs again; after outcome_unknown read the state and keep the operationId. An upgrade of an add-on that has data is backed up, migrated under exclusive ownership and published only when healthy; a failed migration that changes the data format leaves the add-on in needs-recovery with the previous package and the verified backup retained. Updates (never tools; no MCP tool maps to any of it): GET /api/addons/{id}/update reads the owner's update status of an installed add-on for every caller with the addons feature who may see it (administrators any installed add-on, others an enabled one; the same 403 otherwise). The state is unknown, up-to-date, update-available, incompatible (reasons: platform, host-version, api-version, protocol-version, sdk-version, data-format), check-failed (a redacted error class and lastSuccess, the time the library was last read; never shown as up-to-date) or integrity-error (an immutable version was listed again with another digest; never an update). A moving channel (development, a branch, latest) compares resolved digests, so a changed digest with the same version label is an update; an immutable version selects the highest compatible greater version of the same major. The library and the channel the add-on came from never change by themselves. Callers who are not administrators get the state, versions and times only. The route never uses the network. POST /api/addons/{id}/update/check (administrators, own browser session, X-LocalAI-Confirm equal to the operationId) fetches the library again and returns the status; it is rate limited and answers throttled true when it did not fetch, and it applies nothing. POST /api/addons/{id}/upgrade (administrators, own browser session) is the only request that applies an update: it carries operationId, expectedDigest (the installed package the caller saw) and candidateDigest (the candidate the check resolved), and the owner re-resolves, re-checks compatibility and refuses before it records anything with 409 stale_installed, candidate_changed, incompatible, integrity_error or upgrade_conflict (the same transition under another operationId); 503 check_failed when the library cannot be read. A failure keeps the last good generation or enters the recovery state; the previous installation is never deleted. The apps list marks an app update: update-pending from the owner's cached check only and never reaches a library."}, + {Name: "library-sources", Description: "Library sources of add-ons, backends and models", Tags: []string{"library"}, Intro: "Administrators only. GET /api/library/sources lists every source of the add-on, backend and model kinds, shipped defaults and removed defaults (tombstones) included, with kind, id, name, url, mirrors, enabled, default, removed, suspended, a conflict message and the last fetch state. POST /api/library/sources adds a source of your own ({kind: addon|backend|model, url, id, name, format: library|gallery}); the same kind and URL twice is a 409. POST /api/library/sources/{id}/enable and /disable turn one on or off (enabling a removed default brings it back). PUT /api/library/sources/{id} replaces the mirrors of a model or backend source ({mirrors: [..]}); POST also takes mirrors. Each mirror is an http(s), github:, oci:// or file:// address, at most 8. DELETE /api/library/sources/{id} removes a source; a shipped default stays as a tombstone so it does not return at the next start. The routes work on the store that `local-ai library` edits. On a frontend (--addons-role=frontend) the routes forward to the add-on owner, which is the only writer; they answer 503 when the owner is not reachable or runs on another platform. `GET /backends/library/updates` and `POST /backends/library/install` forward the same way. The runtime setting official_libraries (default on) turns every shipped default off at once: they stay listed with suspended true. No MCP tool maps to these routes."}, {Name: "addon-navigation", Description: "The built-in tiles and the per-user pins of the Add-ons drawer", Tags: []string{"AddonNavigation"}, Intro: "GET /api/addons/builtin lists the page add-ons the caller may open. GET and PUT /api/addons/pins read and replace the caller's ordered pins. All of them need an authenticated caller; none needs the addons feature or the administrator role."}, + {Name: "fine-tuning-quantization", Description: "Export status of the fine-tuning and quantization add-ons", Tags: []string{"fine-tuning", "quantization"}, Intro: "GET /api/fine-tuning/status and GET /api/quantization/status return {\"model_exports\": bool}. It is false only when this instance does not see the models path of the add-on owner, so exporting or importing a model here would write where the owner cannot read; it is the same value as the model_exports key of GET /api/features. The routes need an authenticated caller with the fine-tuning or quantization permission and answer 503 when add-ons are off. No MCP tool maps to them: the value is a read-only switch for the add-on UIs."}, { Name: "chat-inference", Description: "OpenAI-compatible chat completions, text completions, and embeddings", diff --git a/core/http/endpoints/localai/api_instructions_test.go b/core/http/endpoints/localai/api_instructions_test.go index 2738efbb0d40..cb5e29c847cf 100644 --- a/core/http/endpoints/localai/api_instructions_test.go +++ b/core/http/endpoints/localai/api_instructions_test.go @@ -39,7 +39,7 @@ var _ = Describe("API Instructions Endpoints", func() { instructions, ok := resp["instructions"].([]any) Expect(ok).To(BeTrue()) - Expect(instructions).To(HaveLen(26)) + Expect(instructions).To(HaveLen(27)) // Verify each instruction has required fields and correct URL format for _, s := range instructions { diff --git a/core/http/endpoints/localai/backend_library.go b/core/http/endpoints/localai/backend_library.go index 30886a1b85ee..5795bfe8d61e 100644 --- a/core/http/endpoints/localai/backend_library.go +++ b/core/http/endpoints/localai/backend_library.go @@ -58,6 +58,7 @@ const libraryInstallTimeout = time.Hour // @Success 200 {object} schema.BackendResponse "the job; read its status at the status URL" // @Failure 400 {object} schema.ErrorResponse "the request does not name an entry" // @Failure 409 {object} schema.ErrorResponse "the same backend is already being installed" +// @Failure 503 {object} schema.ErrorResponse "on a frontend: the add-on owner is not reachable, does not serve library calls, or runs on another platform; the message says which" // @Router /backends/library/install [post] func InstallBackendFromLibraryEndpoint(installer BackendLibraryInstaller, jobs BackendLibraryJobs) echo.HandlerFunc { var mu sync.Mutex @@ -173,6 +174,7 @@ func libraryReason(err error) string { // @Summary List the installed library backends with a newer build in their library. // @Tags backends // @Success 200 {object} map[string][]backendlib.Update "updates" +// @Failure 503 {object} schema.ErrorResponse "on a frontend: the add-on owner is not reachable, does not serve library calls, or runs on another platform; the message says which" // @Router /backends/library/updates [get] func BackendLibraryUpdatesEndpoint(installer BackendLibraryInstaller) echo.HandlerFunc { return func(c echo.Context) error { diff --git a/core/http/endpoints/localai/library_actor.go b/core/http/endpoints/localai/library_actor.go new file mode 100644 index 000000000000..10b3e3141f9d --- /dev/null +++ b/core/http/endpoints/localai/library_actor.go @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: MIT +package localai + +import ( + "errors" + "net/http" + + "github.com/labstack/echo/v4" + "github.com/mudler/LocalAI/core/application" + "github.com/mudler/LocalAI/core/http/auth" + "github.com/mudler/LocalAI/core/library" + "github.com/mudler/LocalAI/core/services/addons/gateway" +) + +// LibraryActor is the middleware of the library routes on a frontend. It runs +// after the administrator middleware and gives the request the actor that the +// carrier signs for the owner, built and revalidated as the add-on routes do. +// A legacy API key has no named user and is refused, also in local-owner mode: +// the owner cannot tell who changed its sources. +func LibraryActor(app *application.Application) echo.MiddlewareFunc { + return func(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + if u := auth.GetUser(c); u != nil && u.ID == "legacy-api-key" { + return librarySourceError(c, http.StatusUnauthorized, "a named account is required: the legacy API key cannot change the library on a frontend") + } + mode := app.AddonOwnershipMode() + actor, err := auth.AddonActorForMode(c, mode) + if err == nil { + err = auth.AddonRevalidate(app.AuthDB(), mode)(c.Request().Context(), actor) + } + if err != nil { + if errors.Is(err, auth.ErrAddonUnavailable) { + gateway.WriteUnavailable(c.Response(), gateway.ReasonOwnerUnreached) + return nil + } + return librarySourceError(c, http.StatusUnauthorized, "a named account is required") + } + c.SetRequest(c.Request().WithContext(library.WithActor(c.Request().Context(), actor))) + return next(c) + } + } +} diff --git a/core/http/endpoints/localai/library_sources.go b/core/http/endpoints/localai/library_sources.go index 97b84feb0a85..85abf4411956 100644 --- a/core/http/endpoints/localai/library_sources.go +++ b/core/http/endpoints/localai/library_sources.go @@ -3,11 +3,11 @@ package localai import ( "errors" "net/http" - "strings" "github.com/labstack/echo/v4" "github.com/mudler/LocalAI/core/library" "github.com/mudler/LocalAI/core/services/addons/catalog" + "github.com/mudler/LocalAI/core/services/addons/gateway" "github.com/mudler/xlog" ) @@ -16,12 +16,10 @@ import ( // edits, through the same functions, so the command and the routes cannot // drift. The routes are registered behind the administrator middleware. type LibrarySourcesEndpoints struct { - // Store returns the one store of the running server. - Store func() (*catalog.Sources, error) - // Official says whether the official libraries are on. - Official func() bool - // Applied runs after a change, so the server reads the new state. - Applied func(dropped ...string) error + // Admin works on the library sources: the store of this process, or the + // add-on owner's store through the carrier on a frontend. The routes + // answer the same either way. + Admin library.Admin } type LibrarySourcesResponse struct { @@ -47,34 +45,21 @@ func librarySourceError(c echo.Context, code int, message string) error { return c.JSON(code, map[string]any{"error": map[string]any{"code": code, "message": message}}) } -func (e LibrarySourcesEndpoints) open(c echo.Context) (*catalog.Sources, error) { - store, err := e.Store() - if err != nil { - xlog.Warn("library sources are not available", "error", err) - return nil, librarySourceError(c, http.StatusServiceUnavailable, "the library sources are not available") - } - return store, nil -} - -func (e LibrarySourcesEndpoints) applied(dropped ...string) { - if e.Applied == nil { - return - } - if err := e.Applied(dropped...); err != nil { - xlog.Warn("could not apply the library sources", "error", err) - } -} - -func (e LibrarySourcesEndpoints) official() bool { - if e.Official == nil { - return true - } - return e.Official() -} - -// storeError maps a store error to the status the caller sees. +// storeError maps an error of the library administration to the status the +// caller sees. func storeError(c echo.Context, err error, id string) error { + var bad *library.BadRequestError switch { + case errors.As(err, &bad): + return librarySourceError(c, http.StatusBadRequest, bad.Message) + case errors.Is(err, library.ErrStoreUnavailable): + return librarySourceError(c, http.StatusServiceUnavailable, "the library sources are not available") + case errors.Is(err, gateway.ErrForbidden): + return librarySourceError(c, http.StatusForbidden, "this account may not change the library sources") + case errors.Is(err, gateway.ErrUnavailable): + // The add-on owner cannot serve the call. The body says why. + gateway.WriteUnavailable(c.Response(), gateway.UnavailableReason(err)) + return nil case errors.Is(err, catalog.ErrNotFound): return librarySourceError(c, http.StatusNotFound, "no source with id "+id) case errors.Is(err, catalog.ErrExists), errors.Is(err, library.ErrDuplicate): @@ -92,35 +77,18 @@ func storeError(c echo.Context, err error, id string) error { // @Tags library // @Produce json // @Success 200 {object} localai.LibrarySourcesResponse +// @Failure 503 {object} schema.ErrorResponse "on a frontend: the add-on owner is not reachable, does not serve library calls, or runs on another platform; the message says which" // @Router /api/library/sources [get] func (e LibrarySourcesEndpoints) List() echo.HandlerFunc { return func(c echo.Context) error { - store, err := e.open(c) - if store == nil { - return err + l, err := e.Admin.List(c.Request().Context(), "") + if err != nil { + return storeError(c, err, "") } - return c.JSON(http.StatusOK, LibrarySourcesResponse{Sources: library.List(store), OfficialLibraries: e.official()}) + return c.JSON(http.StatusOK, LibrarySourcesResponse{Sources: l.Sources, OfficialLibraries: l.OfficialLibraries}) } } -// cleanMirrors trims the mirrors and checks them as a gallery address is -// checked. An add-on source takes none. -func cleanMirrors(kind string, in []string) ([]string, error) { - var out []string - for _, m := range in { - if m = strings.TrimSpace(m); m != "" { - out = append(out, m) - } - } - if len(out) > 0 && kind == string(catalog.KindAddon) { - return nil, errors.New("an add-on source takes no mirrors") - } - if err := catalog.ValidateMirrors(out); err != nil { - return nil, err - } - return out, nil -} - // Update godoc // @Summary Change the mirrors of a library source // @Description Replaces the mirrors of a model or backend source; an empty list clears them. Each mirror is an http(s), github:, oci:// or file:// address. Administrators only. @@ -129,6 +97,7 @@ func cleanMirrors(kind string, in []string) ([]string, error) { // @Produce json // @Param id path string true "Source id" // @Success 200 {object} library.Info +// @Failure 503 {object} schema.ErrorResponse "on a frontend: the add-on owner is not reachable, does not serve library calls, or runs on another platform; the message says which" // @Router /api/library/sources/{id} [put] func (e LibrarySourcesEndpoints) Update() echo.HandlerFunc { return func(c echo.Context) error { @@ -137,30 +106,14 @@ func (e LibrarySourcesEndpoints) Update() echo.HandlerFunc { if err := c.Bind(&req); err != nil { return librarySourceError(c, http.StatusBadRequest, "the request is not valid JSON") } - store, err := e.open(c) - if store == nil { - return err - } - kind := "" - for _, in := range library.List(store) { - if in.ID == id { - kind = string(in.Kind) - } - } - mirrors, err := cleanMirrors(kind, req.Mirrors) + in, err := e.Admin.SetMirrors(c.Request().Context(), id, req.Mirrors) if err != nil { - return librarySourceError(c, http.StatusBadRequest, err.Error()) - } - if err := store.SetMirrors(id, mirrors); err != nil { return storeError(c, err, id) } - e.applied() - for _, in := range library.List(store) { - if in.ID == id { - return c.JSON(http.StatusOK, in) - } + if in == nil { + return c.NoContent(http.StatusNoContent) } - return c.NoContent(http.StatusNoContent) + return c.JSON(http.StatusOK, in) } } @@ -171,6 +124,7 @@ func (e LibrarySourcesEndpoints) Update() echo.HandlerFunc { // @Accept json // @Produce json // @Success 201 {object} library.Info +// @Failure 503 {object} schema.ErrorResponse "on a frontend: the add-on owner is not reachable, does not serve library calls, or runs on another platform; the message says which" // @Router /api/library/sources [post] func (e LibrarySourcesEndpoints) Add() echo.HandlerFunc { return func(c echo.Context) error { @@ -178,42 +132,11 @@ func (e LibrarySourcesEndpoints) Add() echo.HandlerFunc { if err := c.Bind(&req); err != nil { return librarySourceError(c, http.StatusBadRequest, "the request is not valid JSON") } - req.URL, req.ID, req.Name = strings.TrimSpace(req.URL), strings.TrimSpace(req.ID), strings.TrimSpace(req.Name) - switch req.Kind { - case string(catalog.KindAddon), string(catalog.KindBackend), string(catalog.KindModel): - default: - return librarySourceError(c, http.StatusBadRequest, "kind must be addon, backend or model") - } - format := "" - switch req.Format { - case "", "library": - case "gallery": - format = catalog.FormatGallery - default: - return librarySourceError(c, http.StatusBadRequest, "format must be library or gallery") - } - if req.URL == "" { - return librarySourceError(c, http.StatusBadRequest, "url is required") - } - mirrors, err := cleanMirrors(req.Kind, req.Mirrors) - if err != nil { - return librarySourceError(c, http.StatusBadRequest, err.Error()) - } - store, err := e.open(c) - if store == nil { - return err - } - src, err := library.AddWithMirrors(store, catalog.Kind(req.Kind), req.URL, req.ID, req.Name, format, mirrors) + in, err := e.Admin.Add(c.Request().Context(), library.AddRequest{Kind: req.Kind, URL: req.URL, ID: req.ID, Name: req.Name, Format: req.Format, Mirrors: req.Mirrors}) if err != nil { return storeError(c, err, req.ID) } - e.applied() - for _, in := range library.List(store) { - if in.ID == src.ID { - return c.JSON(http.StatusCreated, in) - } - } - return c.JSON(http.StatusCreated, library.Info{Kind: src.Kind, ID: src.ID, Name: src.Name, URL: src.URL, Enabled: true}) + return c.JSON(http.StatusCreated, in) } } @@ -224,25 +147,20 @@ func (e LibrarySourcesEndpoints) Add() echo.HandlerFunc { // @Produce json // @Param id path string true "Source id" // @Success 200 {object} library.Info +// @Failure 503 {object} schema.ErrorResponse "on a frontend: the add-on owner is not reachable, does not serve library calls, or runs on another platform; the message says which" // @Router /api/library/sources/{id}/enable [post] // @Router /api/library/sources/{id}/disable [post] func (e LibrarySourcesEndpoints) SetEnabled(enabled bool) echo.HandlerFunc { return func(c echo.Context) error { id := c.Param("id") - store, err := e.open(c) - if store == nil { - return err - } - if err := store.SetEnabled(id, enabled); err != nil { + in, err := e.Admin.SetEnabled(c.Request().Context(), id, enabled) + if err != nil { return storeError(c, err, id) } - e.applied() - for _, in := range library.List(store) { - if in.ID == id { - return c.JSON(http.StatusOK, in) - } + if in == nil { + return c.NoContent(http.StatusNoContent) } - return c.NoContent(http.StatusNoContent) + return c.JSON(http.StatusOK, in) } } @@ -253,25 +171,15 @@ func (e LibrarySourcesEndpoints) SetEnabled(enabled bool) echo.HandlerFunc { // @Produce json // @Param id path string true "Source id" // @Success 200 {object} map[string]any +// @Failure 503 {object} schema.ErrorResponse "on a frontend: the add-on owner is not reachable, does not serve library calls, or runs on another platform; the message says which" // @Router /api/library/sources/{id} [delete] func (e LibrarySourcesEndpoints) Remove() echo.HandlerFunc { return func(c echo.Context) error { id := c.Param("id") - store, err := e.open(c) - if store == nil { - return err - } - url := library.URLOf(store, id) - wasDefault, err := library.Remove(store, id) + res, err := e.Admin.Remove(c.Request().Context(), id) if err != nil { return storeError(c, err, id) } - // A tombstone keeps its record, so only a deleted source is dropped by address. - if wasDefault { - e.applied() - } else { - e.applied(url) - } - return c.JSON(http.StatusOK, map[string]any{"id": id, "default": wasDefault, "tombstone": wasDefault}) + return c.JSON(http.StatusOK, map[string]any{"id": res.ID, "default": res.Default, "tombstone": res.Tombstone}) } } diff --git a/core/http/endpoints/localai/library_sources_test.go b/core/http/endpoints/localai/library_sources_test.go index d6630b72cc8a..2716cc74f584 100644 --- a/core/http/endpoints/localai/library_sources_test.go +++ b/core/http/endpoints/localai/library_sources_test.go @@ -70,11 +70,11 @@ var _ = Describe("Library source routes", func() { return next(c) } }) - e := LibrarySourcesEndpoints{ + e := LibrarySourcesEndpoints{Admin: library.LocalAdmin{ Store: func() (*catalog.Sources, error) { return store, nil }, Official: func() bool { return official }, Applied: func(d ...string) error { applied++; dropped = append(dropped, d...); return nil }, - } + }} admin := auth.RequireAdmin() app.GET("/api/library/sources", e.List(), admin) app.POST("/api/library/sources", e.Add(), admin) diff --git a/core/http/endpoints/localai/settings.go b/core/http/endpoints/localai/settings.go index d95800317951..d79bf239b94c 100644 --- a/core/http/endpoints/localai/settings.go +++ b/core/http/endpoints/localai/settings.go @@ -110,6 +110,16 @@ func UpdateSettingsEndpoint(app *application.Application) echo.HandlerFunc { } } + // The owner's value is the deployment's value. A frontend reads it from + // the owner at every refresh, so a change made here would be undone and + // would not reach the other frontends. + if app.AddonFrontend() && settings.OfficialLibraries != nil && *settings.OfficialLibraries != appConfig.OfficialLibraries { + return c.JSON(http.StatusConflict, schema.SettingsResponse{ + Success: false, + Error: "official_libraries is set on the add-on owner, not on a frontend", + }) + } + // Generate P2P token before saving so the real token is persisted (not "0") if settings.P2PToken != nil && *settings.P2PToken == "0" { token := p2p.GenerateToken(60, 60) diff --git a/core/http/endpoints/mcp/localai_assistant_test.go b/core/http/endpoints/mcp/localai_assistant_test.go index 36407565b198..45835bf535d2 100644 --- a/core/http/endpoints/mcp/localai_assistant_test.go +++ b/core/http/endpoints/mcp/localai_assistant_test.go @@ -67,6 +67,10 @@ func (stubClient) ListBackends(_ context.Context) ([]localaitools.Backend, error return []localaitools.Backend{{Name: "stub-backend", Installed: true}}, nil } +func (stubClient) ListBackendLibraryUpdates(_ context.Context) ([]localaitools.BackendLibraryUpdate, error) { + return nil, nil +} + func (stubClient) ListKnownBackends(_ context.Context) ([]schema.KnownBackend, error) { return []schema.KnownBackend{}, nil } diff --git a/core/http/middleware/addons_feature.go b/core/http/middleware/addons_feature.go new file mode 100644 index 000000000000..c80afedaa127 --- /dev/null +++ b/core/http/middleware/addons_feature.go @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: MIT +package middleware + +import ( + "context" + + "github.com/labstack/echo/v4" + "github.com/mudler/LocalAI/core/services/addons/gateway" +) + +// AddonsFeatureGate is the one question the add-on routes ask: is the add-on +// side on for this instance. Application.AddonsFeature answers it. +type AddonsFeatureGate interface { + AddonsFeature(ctx context.Context) bool +} + +// addonsSwitch is implemented by a gate that knows whether the instance +// switch is on, so the refusal can say which of the two causes applies. +type addonsSwitch interface { + AddonsEnabled() bool +} + +func unavailableReason(app AddonsFeatureGate) string { + if s, ok := app.(addonsSwitch); ok && !s.AddonsEnabled() { + return gateway.ReasonAddonsOff + } + if app == nil { + return gateway.ReasonAddonsOff + } + return gateway.ReasonOwnerUnreached +} + +// RequireAddonsFeature answers every request with the typed unavailable +// response while the gate is false. It is the instance gate and not the +// per-user permission: auth.RequireFeature runs after it. +func RequireAddonsFeature(app AddonsFeatureGate) echo.MiddlewareFunc { + return func(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + if app == nil || !app.AddonsFeature(c.Request().Context()) { + gateway.WriteUnavailable(c.Response(), unavailableReason(app)) + return nil + } + return next(c) + } + } +} + +// AddonsUnavailable is the handler registered in place of the add-on, +// fine-tuning and quantization routes when add-ons are off. A client sees the +// same response for every path under the prefix, whether or not it exists. +func AddonsUnavailable() echo.HandlerFunc { + return func(c echo.Context) error { + gateway.WriteUnavailable(c.Response(), gateway.ReasonAddonsOff) + return nil + } +} + +// LibrarySourcesOnOwner is the handler of the library-source routes on a +// frontend that cannot reach an owner. +func LibrarySourcesOnOwner() echo.HandlerFunc { + return func(c echo.Context) error { + gateway.WriteUnavailable(c.Response(), gateway.ReasonLibraryOnOwner) + return nil + } +} + +// ModelExportsGate answers whether this instance may write an exported model. +// Application.ModelExportsAllowed answers it. +type ModelExportsGate interface { + ModelExportsAllowed(ctx context.Context) bool +} + +// RequireModelExports blocks the export and import routes while the shared +// models path check fails. An unknown check does not block (the owner may be +// older or briefly down). A nil gate allows, for hosts without the add-on side. +func RequireModelExports(app ModelExportsGate) echo.MiddlewareFunc { + return func(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + if app != nil && !app.ModelExportsAllowed(c.Request().Context()) { + gateway.WriteUnavailable(c.Response(), gateway.ReasonModelsNotShared) + return nil + } + return next(c) + } + } +} diff --git a/core/http/react-ui/e2e/addon-navigation.spec.js b/core/http/react-ui/e2e/addon-navigation.spec.js index b9d14688c256..65df355c6f04 100644 --- a/core/http/react-ui/e2e/addon-navigation.spec.js +++ b/core/http/react-ui/e2e/addon-navigation.spec.js @@ -9,10 +9,10 @@ import { FACE_MODEL, clipFile, faces, match, mockIdentity, people, photoFile } f // Studio and Talk are packaged add-ons, not built-in tiles: the specs install // the real packages. No page is built in today, so `builtin` is empty unless a // spec makes one up to check the built-in rules. -async function setup(page, { apps = [], pins = ['studio', 'talk'], addons = true, builtin = [], studio = true, user = MEMBER, installed, releases } = {}) { +async function setup(page, { apps = [], pins = ['studio', 'talk'], addons = true, builtin = [], studio = true, user = MEMBER, installed, releases, features = { addon_navigation: true } } = {}) { let saved = pins await page.route('**/api/auth/status', route => route.fulfill({ json: sessionFor(user, { addons }) })) - await page.route('**/api/features', route => route.fulfill({ json: { addon_navigation: true } })) + await page.route('**/api/features', route => route.fulfill({ json: features })) await page.route('**/api/addons/builtin', route => route.fulfill({ json: { builtin } })) // mockAddons installs the Apps routes with the real response shape ({ apps }). // The packages are served from the build output, so a page that opens one runs it. @@ -968,3 +968,63 @@ test.describe('Add-on navigation: the Face recognition page', () => { expect(calls.forget[0]).toMatchObject({ face: true, id: 'f-anna' }) }) }) + +test.describe('Add-on navigation: the addons feature gate', () => { + const OFF = { addon_navigation: true, addons: false, fine_tuning: false, quantization: false } + + // With add-ons off the server answers 503 on every add-on route, so a request + // to the pins or built-in routes is a failure of the page, not of the server. + async function watch(page) { + const calls = [] + page.on('request', request => { + const { pathname } = new URL(request.url()) + if (pathname === '/api/addons/pins' || pathname === '/api/addons/builtin') calls.push(pathname) + }) + return calls + } + + test('with add-ons off the sidebar has no Add-ons entry, no pinned app and no fine-tuning or quantization entry', async ({ page }) => { + const calls = await watch(page) + await setup(page, { features: OFF, user: ADMIN }) + await page.goto('/app') + const nav = page.getByRole('navigation') + await expect(nav.getByRole('link', { name: 'Chat' })).toBeVisible() + await expect(nav.getByRole('link', { name: 'Add-ons', exact: true })).toHaveCount(0) + await expect(nav.getByTestId('apps-pinned-link')).toHaveCount(0) + await expect(nav.getByRole('link', { name: /Fine-?tun/i })).toHaveCount(0) + await expect(nav.getByRole('link', { name: /Quantiz/i })).toHaveCount(0) + expect(calls).toEqual([]) + }) + + test('with add-ons off the Build hub has no Add-ons, fine-tuning or quantization tab, in either layout', async ({ page }) => { + for (const addon_navigation of [true, false]) { + await setup(page, { features: { ...OFF, addon_navigation }, user: ADMIN }) + await page.goto('/app/build') + await expect(page.getByRole('tab', { name: 'Agents' }).or(page.getByRole('link', { name: 'Agents' })).first()).toBeVisible() + for (const name of [/^Add-ons$/, /Fine-?tun/i, /Quantiz/i]) { + await expect(page.getByRole('tab', { name })).toHaveCount(0) + await expect(page.locator('.hub-tabs, [role="tablist"]').getByRole('link', { name })).toHaveCount(0) + } + await page.unroute('**/api/features') + } + }) + + for (const address of ['/app/addons', '/app/addons/studio/home', '/app/fine-tune', '/app/quantize']) { + test(`with add-ons off ${address} ends at /app and calls no add-on route`, async ({ page }) => { + const calls = await watch(page) + await setup(page, { features: OFF, user: ADMIN }) + await page.goto(address) + await expect(page).toHaveURL(/\/app\/?$/) + await expect(page.getByRole('navigation').getByRole('link', { name: 'Chat' })).toBeVisible() + expect(calls).toEqual([]) + }) + } + + test('an older server without the addons key keeps today\'s behaviour', async ({ page }) => { + await setup(page) + await page.goto('/app') + const nav = page.getByRole('navigation') + await expect(nav.getByRole('link', { name: 'Add-ons', exact: true })).toHaveCount(1) + await expect(nav.getByTestId('apps-pinned-link')).toHaveCount(2) + }) +}) diff --git a/core/http/react-ui/src/components/Sidebar.jsx b/core/http/react-ui/src/components/Sidebar.jsx index a31e9840bfcc..654fe5f696db 100644 --- a/core/http/react-ui/src/components/Sidebar.jsx +++ b/core/http/react-ui/src/components/Sidebar.jsx @@ -166,7 +166,7 @@ export default function Sidebar({ isOpen, onClose }) { // The previous sidebar returns when the operator sets the legacy switch. const legacy = features.addon_navigation === false const hasBuiltin = builtin.length > 0 - const navAuth = { isAdmin, hasFeature, hasBuiltin } + const navAuth = { isAdmin, hasFeature, hasBuiltin, features } const primary = visiblePrimary(primaryDestinations, navAuth) // One badge, on the always-visible sidebar entry. The Operate tab bar only @@ -221,7 +221,7 @@ export default function Sidebar({ isOpen, onClose }) { {/* The Apps group: the pinned apps in pin order, under the Apps entry. */} - {!legacy && item.addonsEntry && !collapsed && } + {!legacy && item.addonsEntry && features.addons !== false && !collapsed && } ))} diff --git a/core/http/react-ui/src/components/hub/useHubAuth.js b/core/http/react-ui/src/components/hub/useHubAuth.js index c73a715ff750..8c40b059b091 100644 --- a/core/http/react-ui/src/components/hub/useHubAuth.js +++ b/core/http/react-ui/src/components/hub/useHubAuth.js @@ -11,13 +11,16 @@ let featuresCache = {} export function useHubAuth() { const { isAdmin, authEnabled, hasFeature } = useAuth() const [features, setFeatures] = useState(featuresCache) + // True once /api/features answered or failed, so a route that depends on a + // switch does not act on the empty map of a first render. + const [featuresLoaded, setFeaturesLoaded] = useState(Object.keys(featuresCache).length > 0) useEffect(() => { fetch(apiUrl('/api/features')) .then(r => r.json()) - .then(f => { featuresCache = f; setFeatures(f) }) - .catch(() => {}) + .then(f => { featuresCache = f; setFeatures(f); setFeaturesLoaded(true) }) + .catch(() => { setFeaturesLoaded(true) }) }, []) - return { isAdmin, authEnabled, hasFeature, features } + return { isAdmin, authEnabled, hasFeature, features, featuresLoaded } } diff --git a/core/http/react-ui/src/context/NavigationContext.jsx b/core/http/react-ui/src/context/NavigationContext.jsx index e3e8f1e84cac..114376487f6e 100644 --- a/core/http/react-ui/src/context/NavigationContext.jsx +++ b/core/http/react-ui/src/context/NavigationContext.jsx @@ -1,5 +1,6 @@ import { createContext, useCallback, useContext, useEffect, useMemo, useRef, useState } from 'react' import { useAuth } from './AuthContext' +import { apiUrl } from '../utils/basePath' import { pinsApi } from '../utils/pinsClient' import { PIN_CAP, togglePin, movePin } from '../utils/addonTiles' @@ -27,7 +28,16 @@ export function NavigationProvider({ children }) { if (!signedIn) return undefined const ctl = new AbortController() setState(s => ({ ...s, status: 'loading' })) - Promise.all([pinsApi.listBuiltin(ctl.signal), pinsApi.listPins(ctl.signal)]) + // With add-ons off the server answers 503 on both routes, so the shell does + // not call them. An unreadable /api/features counts as on. + const addonsOn = fetch(apiUrl('/api/features'), { signal: ctl.signal }) + .then(r => r.json()) + .then(f => f?.addons !== false) + .catch(() => true) + addonsOn + .then(on => (on + ? Promise.all([pinsApi.listBuiltin(ctl.signal), pinsApi.listPins(ctl.signal)]) + : [[], []])) .then(([builtin, pinIds]) => setState({ status: 'ready', builtin, pinIds })) .catch(() => { if (!ctl.signal.aborted) setState(s => ({ ...s, status: 'error' })) }) return () => ctl.abort() diff --git a/core/http/react-ui/src/router.jsx b/core/http/react-ui/src/router.jsx index 3bf785d4d54d..e265e237467d 100644 --- a/core/http/react-ui/src/router.jsx +++ b/core/http/react-ui/src/router.jsx @@ -210,9 +210,21 @@ function Admin({ children }) { // The layout of the add-on routes: the Build hub bar under the legacy switch, // the bare page otherwise. +// With add-ons off (features.addons === false) the add-on addresses and the +// old fine-tuning and quantization addresses lead back to /app. An absent key +// counts as on. +function AddonsOffRedirect({ children }) { + const { features, featuresLoaded } = useHubAuth() + if (!featuresLoaded) return null + if (features.addons === false) return + return children +} + function AddonsShell() { - const { features } = useHubAuth() + const { features, featuresLoaded } = useHubAuth() const outletContext = useOutletContext() + if (!featuresLoaded) return null + if (features.addons === false) return if (features.addon_navigation === false) return return } @@ -284,8 +296,8 @@ const appChildren = [ { path: 'collections/:name', element: }, { path: 'agent-jobs', element: }, { path: 'agent-jobs/tasks/:id', element: }, - { path: 'fine-tune', element: }, - { path: 'quantize', element: }, + { path: 'fine-tune', element: }, + { path: 'quantize', element: }, { path: 'face', element: }, { path: 'face/:model', element: }, { path: 'voice', element: }, diff --git a/core/http/react-ui/src/utils/navigationModel.js b/core/http/react-ui/src/utils/navigationModel.js index b3844f7e9ca2..a6bf08f92a90 100644 --- a/core/http/react-ui/src/utils/navigationModel.js +++ b/core/http/react-ui/src/utils/navigationModel.js @@ -11,9 +11,12 @@ export const primaryDestinations = Object.freeze([ { path: '/app/addons', icon: 'puzzle', labelKey: 'items.addons', addonsEntry: true }, ]) -export function visiblePrimary(items, { isAdmin, hasFeature, hasBuiltin }) { +// `features` is the /api/features map. An absent `addons` key (an older server) +// counts as on; only an explicit false hides the Add-ons entry. +export function visiblePrimary(items, { isAdmin, hasFeature, hasBuiltin, features }) { return items.filter(item => { if (item.adminOnly && !isAdmin) return false + if (item.addonsEntry && features?.addons === false) return false if (item.addonsEntry && !isAdmin && !hasFeature('addons') && !hasBuiltin) return false return true }) diff --git a/core/http/routes/addon_navigation.go b/core/http/routes/addon_navigation.go index 0db33dbdf832..f4796785b7d1 100644 --- a/core/http/routes/addon_navigation.go +++ b/core/http/routes/addon_navigation.go @@ -20,20 +20,20 @@ import ( // It returns an error when the pin store cannot be opened, for example a // corrupt pins file. Startup must stop then: a memory-only fallback would lose // every unpin at the next restart. -func RegisterAddonNavigationRoutes(e *echo.Echo, a *application.Application) error { +func RegisterAddonNavigationRoutes(e *echo.Echo, a *application.Application, gate ...echo.MiddlewareFunc) error { store, err := a.AddonPinStore() if err != nil { return err } - e.GET("/api/addons/builtin", endpoints.ListBuiltin(a.AuthDB())) - RegisterAddonPinRoutes(e, a.AddonGateway(), a.AuthDB(), a.AddonOwnershipMode(), &pins.Service{Store: store}) + e.GET("/api/addons/builtin", endpoints.ListBuiltin(a.AuthDB()), gate...) + RegisterAddonPinRoutes(e, a.AddonGateway(), a.AuthDB(), a.AddonOwnershipMode(), &pins.Service{Store: store}, gate...) return nil } // RegisterAddonPinRoutes registers the two user-scoped pin routes. The caller // is filtered like the Apps list. -func RegisterAddonPinRoutes(e *echo.Echo, gw *gateway.Gateway, db *gorm.DB, ownership string, svc *pins.Service) { +func RegisterAddonPinRoutes(e *echo.Echo, gw *gateway.Gateway, db *gorm.DB, ownership string, svc *pins.Service, gate ...echo.MiddlewareFunc) { mode := endpoints.WithOwnership(ownership) - e.GET("/api/addons/pins", endpoints.GetPins(gw, db, svc, mode)) - e.PUT("/api/addons/pins", endpoints.PutPins(gw, db, svc, mode), auth.CSRFMiddleware(), endpoints.BrowserProtection()) + e.GET("/api/addons/pins", endpoints.GetPins(gw, db, svc, mode), gate...) + e.PUT("/api/addons/pins", endpoints.PutPins(gw, db, svc, mode), append(gate, auth.CSRFMiddleware(), endpoints.BrowserProtection())...) } diff --git a/core/http/routes/addons.go b/core/http/routes/addons.go index a64e15745741..bde2a0b89707 100644 --- a/core/http/routes/addons.go +++ b/core/http/routes/addons.go @@ -6,19 +6,24 @@ import ( "github.com/mudler/LocalAI/core/application" "github.com/mudler/LocalAI/core/http/auth" endpoints "github.com/mudler/LocalAI/core/http/endpoints/addons" + httpMiddleware "github.com/mudler/LocalAI/core/http/middleware" "github.com/mudler/LocalAI/core/services/addons/gateway" "gorm.io/gorm" ) -func RegisterAddonRoutes(e *echo.Echo, a *application.Application) { - RegisterAddonRoutesFor(e, a.AddonGateway(), a.AuthDB(), a.AddonOwnershipMode()) +// RegisterAddonRoutes registers the routes over the gateway of a. The caller +// passes the instance gate (middleware.RequireAddonsFeature(a)); app.go does. +func RegisterAddonRoutes(e *echo.Echo, a *application.Application, gate ...echo.MiddlewareFunc) { + RegisterAddonRoutesFor(e, a.AddonGateway(), a.AuthDB(), a.AddonOwnershipMode(), gate...) } // RegisterAddonRoutesFor registers the addon routes over one gateway. A // frontend of a distributed deployment serves exactly the routes of the owner -// over its forwarding gateway, so the routes are one list. -func RegisterAddonRoutesFor(e *echo.Echo, gw *gateway.Gateway, db *gorm.DB, ownership string) { - group := e.Group("/api/addons", auth.RequireFeature(db, auth.FeatureAddons), auth.CSRFMiddleware(), endpoints.BrowserProtection()) +// over its forwarding gateway, so the routes are one list. The instance gate +// (middleware.RequireAddonsFeature) runs first and the per-user permission +// after it. +func RegisterAddonRoutesFor(e *echo.Echo, gw *gateway.Gateway, db *gorm.DB, ownership string, gate ...echo.MiddlewareFunc) { + group := e.Group("/api/addons", append(gate, auth.RequireFeature(db, auth.FeatureAddons), auth.CSRFMiddleware(), endpoints.BrowserProtection())...) mode := endpoints.WithOwnership(ownership) group.POST("/dispatch", endpoints.Dispatch(gw, db, mode)) group.POST("/relay", endpoints.Relay(gw, db, mode)) @@ -95,3 +100,13 @@ func RegisterAddonRoutesFor(e *echo.Echo, gw *gateway.Gateway, db *gorm.DB, owne // files of the current package. No MCP tool maps to it. group.GET("/:id/assets/:digest/*", endpoints.GetAsset(gw, db, mode)) } + +// RegisterAddonsUnavailable answers every method on a prefix and below it with +// the typed unavailable response of an absent owner. With add-ons off, it is +// registered in place of the add-on, fine-tuning and quantization routes, so a +// client sees the same response for every path under the prefix. +func RegisterAddonsUnavailable(e *echo.Echo, prefix string) { + h := httpMiddleware.AddonsUnavailable() + e.Any(prefix, h) + e.Any(prefix+"/*", h) +} diff --git a/core/http/routes/addons_gate_test.go b/core/http/routes/addons_gate_test.go new file mode 100644 index 000000000000..5b4b3037e82a --- /dev/null +++ b/core/http/routes/addons_gate_test.go @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: MIT + +package routes_test + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + goruntime "runtime" + + "github.com/labstack/echo/v4" + "github.com/mudler/LocalAI/core/application" + "github.com/mudler/LocalAI/core/config" + httpMiddleware "github.com/mudler/LocalAI/core/http/middleware" + "github.com/mudler/LocalAI/core/http/routes" + "github.com/mudler/LocalAI/core/services/addons/gateway" + "github.com/mudler/LocalAI/pkg/addons/contract" + "github.com/mudler/LocalAI/pkg/system" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Routes behind the addons gate", func() { + newApp := func(opts ...config.AppOption) *application.Application { + dir, err := filepath.EvalSymlinks(GinkgoT().TempDir()) + Expect(err).NotTo(HaveOccurred()) + models, backends := filepath.Join(dir, "models"), filepath.Join(dir, "backends") + Expect(os.Mkdir(models, 0750)).To(Succeed()) + Expect(os.Mkdir(backends, 0750)).To(Succeed()) + state, err := system.GetSystemState(system.WithBackendPath(backends), system.WithModelPath(models)) + Expect(err).NotTo(HaveOccurred()) + ctx, cancel := context.WithCancel(context.Background()) + DeferCleanup(cancel) + opts = append([]config.AppOption{config.WithContext(ctx), config.WithSystemState(state)}, opts...) + app, err := application.New(opts...) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { _ = app.Shutdown() }) + return app + } + get := func(e *echo.Echo, path string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + e.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) + return w + } + paths := []string{"/api/addons", "/api/addons/does-not-exist", "/api/fine-tuning/jobs", "/api/quantization/jobs"} + + It("answers every add-on path with the same 503 body when the handlers are the catch-alls", func() { + e := echo.New() + for _, prefix := range []string{"/api/addons", "/api/fine-tuning", "/api/quantization"} { + routes.RegisterAddonsUnavailable(e, prefix) + } + var body string + for _, p := range paths { + w := get(e, p) + Expect(w.Code).To(Equal(http.StatusServiceUnavailable), p) + if body == "" { + body = w.Body.String() + } + Expect(w.Body.String()).To(Equal(body), p) + } + // The body is the one the gateway writes for a disabled feature. + rec := httptest.NewRecorder() + gateway.WriteUnavailable(rec, gateway.ReasonAddonsOff) + Expect(body).To(Equal(rec.Body.String())) + }) + + It("answers the same 503 from the real routes while the gate is false", func() { + app := newApp() + e := echo.New() + gate := httpMiddleware.RequireAddonsFeature(app) + routes.RegisterAddonRoutes(e, app, gate) + Expect(routes.RegisterAddonNavigationRoutes(e, app, gate)).To(Succeed()) + for _, p := range []string{"/api/addons/apps", "/api/addons/builtin", "/api/addons/pins", "/api/addons/does-not-exist"} { + w := get(e, p) + Expect(w.Code).To(Equal(http.StatusServiceUnavailable), p) + Expect(w.Body.String()).To(ContainSubstring(gateway.ReasonAddonsOff), p) + Expect(w.Body.String()).To(ContainSubstring(`"code":"unavailable"`), p) + } + }) + + It("lets a request through to the real routes when the owner is present", func() { + if goruntime.GOOS != "linux" { + Skip("the add-on runtime exists on Linux only") + } + root, err := filepath.EvalSymlinks(GinkgoT().TempDir()) + Expect(err).NotTo(HaveOccurred()) + root = filepath.Join(root, "addons") + transport, address, err := gateway.NewLoopbackCarrierConfig("local", root) + Expect(err).NotTo(HaveOccurred()) + app := newApp(config.WithAddons(config.AddonConfig{ + Enabled: true, + InstanceID: "local", OwnerID: "local", OwnerAddress: transport.OwnerAddress, + StorageID: "local", Root: root, ListenAddress: address, Transport: transport, + OwnershipMode: "local-owner", + Host: contract.HostCompatibility{Host: 1, API: 1, Protocol: 1, SDK: 1, DataFormat: 1, OS: "linux", Arch: "amd64"}, + })) + e := echo.New() + gate := httpMiddleware.RequireAddonsFeature(app) + routes.RegisterAddonRoutes(e, app, gate) + Expect(routes.RegisterAddonNavigationRoutes(e, app, gate)).To(Succeed()) + Expect(get(e, "/api/addons/builtin").Code).To(Equal(http.StatusOK)) + }) +}) diff --git a/core/http/routes/features.go b/core/http/routes/features.go new file mode 100644 index 000000000000..23b6dc7c9b38 --- /dev/null +++ b/core/http/routes/features.go @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: MIT +package routes + +import ( + "github.com/labstack/echo/v4" + "github.com/mudler/LocalAI/core/application" + "github.com/mudler/LocalAI/core/config" +) + +// RegisterFeatureRoutes serves GET /api/features, the switches the UI reads. +// Fine-tuning and quantization are add-ons, so they follow the add-on gate and +// have no switch of their own. addon_navigation is the layout (new or legacy) +// and is not a substitute for addons. +func RegisterFeatureRoutes(router *echo.Echo, appConfig *config.ApplicationConfig, app *application.Application) { + router.GET("/api/features", func(c echo.Context) error { + addons := app.AddonsFeature(c.Request().Context()) + // model_exports is false only when this instance does not see the + // owner's models path; an unknown check leaves it true. + return c.JSON(200, map[string]bool{ + "agents": appConfig.AgentPool.Enabled, + "mcp": !appConfig.DisableMCP, + "addons": addons, + "fine_tuning": addons, + "quantization": addons, + "model_exports": addons && app.ModelExportsAllowed(c.Request().Context()), + "distributed": appConfig.Distributed.Enabled, + "backend_install": !appConfig.DisableBackendInstall, + "localai_assistant": !appConfig.DisableLocalAIAssistant && app.LocalAIAssistant() != nil, + "addon_navigation": !appConfig.LegacyNavigation, + }) + }) +} diff --git a/core/http/routes/features_test.go b/core/http/routes/features_test.go new file mode 100644 index 000000000000..3f57ba2a447e --- /dev/null +++ b/core/http/routes/features_test.go @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: MIT + +package routes_test + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + goruntime "runtime" + + "github.com/labstack/echo/v4" + "github.com/mudler/LocalAI/core/application" + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/http/routes" + "github.com/mudler/LocalAI/core/services/addons/gateway" + "github.com/mudler/LocalAI/pkg/addons/contract" + "github.com/mudler/LocalAI/pkg/system" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("GET /api/features and the addons gate", func() { + features := func(opts ...config.AppOption) map[string]bool { + dir, err := filepath.EvalSymlinks(GinkgoT().TempDir()) + Expect(err).NotTo(HaveOccurred()) + models, backends := filepath.Join(dir, "models"), filepath.Join(dir, "backends") + Expect(os.Mkdir(models, 0750)).To(Succeed()) + Expect(os.Mkdir(backends, 0750)).To(Succeed()) + state, err := system.GetSystemState(system.WithBackendPath(backends), system.WithModelPath(models)) + Expect(err).NotTo(HaveOccurred()) + ctx, cancel := context.WithCancel(context.Background()) + DeferCleanup(cancel) + opts = append([]config.AppOption{config.WithContext(ctx), config.WithSystemState(state)}, opts...) + app, err := application.New(opts...) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { _ = app.Shutdown() }) + e := echo.New() + routes.RegisterFeatureRoutes(e, app.ApplicationConfig(), app) + w := httptest.NewRecorder() + e.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/api/features", nil)) + Expect(w.Code).To(Equal(http.StatusOK)) + var got map[string]bool + Expect(json.Unmarshal(w.Body.Bytes(), &got)).To(Succeed()) + return got + } + + It("reports addons, fine_tuning and quantization as false with add-ons off", func() { + got := features() + Expect(got).To(HaveKeyWithValue("addons", false)) + Expect(got).To(HaveKeyWithValue("fine_tuning", false)) + Expect(got).To(HaveKeyWithValue("quantization", false)) + }) + + It("reports them as true with the owner present", func() { + if goruntime.GOOS != "linux" { + Skip("the add-on runtime exists on Linux only") + } + root, err := filepath.EvalSymlinks(GinkgoT().TempDir()) + Expect(err).NotTo(HaveOccurred()) + root = filepath.Join(root, "addons") + transport, address, err := gateway.NewLoopbackCarrierConfig("local", root) + Expect(err).NotTo(HaveOccurred()) + got := features(config.WithAddons(config.AddonConfig{ + Enabled: true, + InstanceID: "local", OwnerID: "local", OwnerAddress: transport.OwnerAddress, + StorageID: "local", Root: root, ListenAddress: address, Transport: transport, + OwnershipMode: "local-owner", + Host: contract.HostCompatibility{Host: 1, API: 1, Protocol: 1, SDK: 1, DataFormat: 1, OS: "linux", Arch: "amd64"}, + })) + Expect(got).To(HaveKeyWithValue("addons", true)) + Expect(got).To(HaveKeyWithValue("fine_tuning", true)) + Expect(got).To(HaveKeyWithValue("quantization", true)) + }) +}) diff --git a/core/http/routes/finetuning.go b/core/http/routes/finetuning.go index a122798787ac..3c3cb0f334fa 100644 --- a/core/http/routes/finetuning.go +++ b/core/http/routes/finetuning.go @@ -7,6 +7,7 @@ import ( "github.com/mudler/LocalAI/core/application" "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/http/endpoints/localai" + httpMiddleware "github.com/mudler/LocalAI/core/http/middleware" "github.com/mudler/LocalAI/core/services/finetune" ) @@ -28,7 +29,8 @@ func RegisterFineTuningRoutes(e *echo.Echo, ftService *finetune.FineTuneService, } } - ft := e.Group("/api/fine-tuning", readyMw, fineTuningMw) + ft := e.Group("/api/fine-tuning", httpMiddleware.RequireAddonsFeature(app), readyMw, fineTuningMw) + ft.GET("/status", localai.ModelExportsStatusEndpoint(app)) ft.GET("/backends", localai.ListFineTuneBackendsEndpoint(appConfig, ClusterCapabilityProviderFor(app), ClusterInstalledProviderFor(app))) ft.POST("/jobs", localai.StartFineTuneJobEndpoint(ftService)) ft.GET("/jobs", localai.ListFineTuneJobsEndpoint(ftService)) @@ -37,7 +39,7 @@ func RegisterFineTuningRoutes(e *echo.Echo, ftService *finetune.FineTuneService, ft.DELETE("/jobs/:id", localai.DeleteFineTuneJobEndpoint(ftService)) ft.GET("/jobs/:id/progress", localai.FineTuneProgressEndpoint(ftService)) ft.GET("/jobs/:id/checkpoints", localai.ListCheckpointsEndpoint(ftService)) - ft.POST("/jobs/:id/export", localai.ExportModelEndpoint(ftService)) + ft.POST("/jobs/:id/export", localai.ExportModelEndpoint(ftService), httpMiddleware.RequireModelExports(app)) ft.GET("/jobs/:id/download", localai.DownloadExportedModelEndpoint(ftService)) ft.POST("/datasets", localai.UploadDatasetEndpoint(ftService)) } diff --git a/core/http/routes/library_frontend_test.go b/core/http/routes/library_frontend_test.go new file mode 100644 index 000000000000..bfb597c8b7b9 --- /dev/null +++ b/core/http/routes/library_frontend_test.go @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: MIT + +package routes_test + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + + "github.com/labstack/echo/v4" + "github.com/mudler/LocalAI/core/application" + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/http/routes" + "github.com/mudler/LocalAI/core/services/addons/gateway" + "github.com/mudler/LocalAI/pkg/system" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("library sources on a frontend", func() { + It("answers every library route with the add-on 503 body and writes no file", func() { + dataPath, err := filepath.EvalSymlinks(GinkgoT().TempDir()) + Expect(err).NotTo(HaveOccurred()) + models, backends := filepath.Join(dataPath, "models"), filepath.Join(dataPath, "backends") + Expect(os.Mkdir(models, 0750)).To(Succeed()) + Expect(os.Mkdir(backends, 0750)).To(Succeed()) + state, err := system.GetSystemState(system.WithBackendPath(backends), system.WithModelPath(models)) + Expect(err).NotTo(HaveOccurred()) + ctx, cancel := context.WithCancel(context.Background()) + DeferCleanup(cancel) + // Add-ons off, frontend role: the case the CLI builds for + // --addons-enabled=false --addons-role=frontend. + app, err := application.New(config.WithContext(ctx), config.WithSystemState(state), config.WithDataPath(dataPath), + config.WithAddons(config.AddonConfig{Frontend: true})) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { _ = app.Shutdown() }) + e := echo.New() + pass := func(next echo.HandlerFunc) echo.HandlerFunc { return next } + routes.RegisterLibrarySourceRoutes(e, app, app.ApplicationConfig(), pass) + + want := httptest.NewRecorder() + gateway.WriteUnavailable(want, gateway.ReasonLibraryOnOwner) + for _, c := range []struct{ method, path string }{ + {http.MethodGet, "/api/library/sources"}, + {http.MethodPost, "/api/library/sources"}, + {http.MethodPost, "/api/library/sources/x/enable"}, + {http.MethodPost, "/api/library/sources/x/disable"}, + {http.MethodPut, "/api/library/sources/x"}, + {http.MethodDelete, "/api/library/sources/x"}, + } { + w := httptest.NewRecorder() + e.ServeHTTP(w, httptest.NewRequest(c.method, c.path, strings.NewReader(`{"kind":"model","url":"https://example.org/a.yaml"}`))) + Expect(w.Code).To(Equal(http.StatusServiceUnavailable), c.path) + Expect(w.Body.String()).To(Equal(want.Body.String()), c.path) + Expect(w.Header().Get("X-LocalAI-Reason")).To(BeEmpty()) + var env struct { + Error struct{ Message, Type, Code string } + } + Expect(json.Unmarshal(w.Body.Bytes(), &env)).To(Succeed()) + Expect(env.Error.Message).To(Equal("library sources are managed on the owner")) + Expect(env.Error.Code).To(Equal("unavailable")) + Expect(env.Error.Type).NotTo(BeEmpty()) + } + Expect(filepath.Join(dataPath, "addons")).NotTo(BeADirectory()) + Expect(filepath.Join(dataPath, "sources.json")).NotTo(BeAnExistingFile()) + }) +}) diff --git a/core/http/routes/library_split_test.go b/core/http/routes/library_split_test.go new file mode 100644 index 000000000000..eee11b3008a1 --- /dev/null +++ b/core/http/routes/library_split_test.go @@ -0,0 +1,259 @@ +// SPDX-License-Identifier: MIT + +package routes_test + +import ( + "context" + "encoding/json" + "fmt" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + goruntime "runtime" + "strconv" + "strings" + + "github.com/labstack/echo/v4" + "github.com/mudler/LocalAI/core/application" + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/http/auth" + "github.com/mudler/LocalAI/core/http/endpoints/localai" + "github.com/mudler/LocalAI/core/http/routes" + "github.com/mudler/LocalAI/core/services/addons/gateway" + "github.com/mudler/LocalAI/pkg/addons/contract" + "github.com/mudler/LocalAI/pkg/system" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// An owner and frontends run as separate applications with separate data +// paths. They share the carrier key set and the owner address only. +var _ = Describe("library routes through a split owner", func() { + var ( + keys string + address string + port string + stops map[*application.Application]context.CancelFunc + ) + + newApp := func(role, dataPath string, official bool) *application.Application { + models, backends := filepath.Join(dataPath, "models"), filepath.Join(dataPath, "backends") + Expect(os.MkdirAll(models, 0750)).To(Succeed()) + Expect(os.MkdirAll(backends, 0750)).To(Succeed()) + state, err := system.GetSystemState(system.WithBackendPath(backends), system.WithModelPath(models)) + Expect(err).NotTo(HaveOccurred()) + transport, err := gateway.LoadCarrierConfig(keys, role, address) + Expect(err).NotTo(HaveOccurred()) + cfg := config.AddonConfig{ + Enabled: true, + InstanceID: transport.InstanceID, OwnerID: gateway.OwnerIdentity, OwnerAddress: address, + OfficialLibrary: true, OfficialBackendLibrary: true, + StorageID: "local", Transport: transport, OwnershipMode: auth.AddonModeLocalOwner, + Host: contract.HostCompatibility{Host: 1, API: 1, Protocol: 1, SDK: 1, DataFormat: 1, OS: goruntime.GOOS, Arch: goruntime.GOARCH}, + } + if role == "frontend" { + cfg.Frontend = true + } else { + cfg.Root = filepath.Join(dataPath, "addons") + cfg.ListenAddress = "127.0.0.1:" + port + Expect(os.MkdirAll(cfg.Root, 0700)).To(Succeed()) + } + ctx, cancel := context.WithCancel(context.Background()) + app, err := application.New(config.WithContext(ctx), config.WithSystemState(state), config.WithDataPath(dataPath), + config.WithOfficialLibraries(official), config.WithAddons(cfg), + config.WithBackendGalleries(config.DefaultBackendGalleriesJSON), config.WithStringGalleries(config.DefaultGalleriesJSON)) + Expect(err).NotTo(HaveOccurred()) + stops[app] = cancel + DeferCleanup(func() { cancel(); _ = app.Shutdown() }) + return app + } + stop := func(app *application.Application) { + stops[app]() + _ = app.Shutdown() + } + serve := func(app *application.Application, user *auth.User) *echo.Echo { + e := echo.New() + pass := func(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + if user != nil { + c.Set("auth_user", user) + } + return next(c) + } + } + routes.RegisterLibrarySourceRoutes(e, app, app.ApplicationConfig(), pass) + e.POST("/api/settings", localai.UpdateSettingsEndpoint(app)) + return e + } + send := func(e *echo.Echo, method, path, body string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + r := httptest.NewRequest(method, path, strings.NewReader(body)) + r.Header.Set("Content-Type", "application/json") + e.ServeHTTP(w, r) + return w + } + urls := func(g []config.Gallery) []string { + var out []string + for _, x := range g { + out = append(out, x.URL) + } + return out + } + envelope := func(w *httptest.ResponseRecorder) (message, code string) { + var b struct { + Error struct{ Message, Code string } + } + ExpectWithOffset(1, json.Unmarshal(w.Body.Bytes(), &b)).To(Succeed(), w.Body.String()) + return b.Error.Message, b.Error.Code + } + + BeforeEach(func() { + if goruntime.GOOS != "linux" { + Skip("the add-on runtime exists on Linux only") + } + stops = map[*application.Application]context.CancelFunc{} + keysDir, err := filepath.EvalSymlinks(GinkgoT().TempDir()) + Expect(err).NotTo(HaveOccurred()) + keys = filepath.Join(keysDir, "keys") + Expect(gateway.GenerateCarrierKeys(keys, []string{"127.0.0.1"})).To(Succeed()) + l, err := net.Listen("tcp", "127.0.0.1:0") + Expect(err).NotTo(HaveOccurred()) + port = strconv.Itoa(l.Addr().(*net.TCPAddr).Port) + Expect(l.Close()).To(Succeed()) + address = "https://127.0.0.1:" + port + }) + + dir := func() string { + d, err := filepath.EvalSymlinks(GinkgoT().TempDir()) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + return d + } + + It("lists the sources through the owner, adds one on the owner, and shows it on every frontend after a refresh", func() { + owner := newApp("owner", dir(), true) + frontendData := dir() + first, second := newApp("frontend", frontendData, true), newApp("frontend", dir(), true) + e := serve(first, nil) + + w := send(e, http.MethodGet, "/api/library/sources", "") + Expect(w.Code).To(Equal(http.StatusOK), w.Body.String()) + var list struct { + Sources []struct{ ID, URL string } + OfficialLibraries bool + } + Expect(json.Unmarshal(w.Body.Bytes(), &list)).To(Succeed()) + Expect(list.Sources).NotTo(BeEmpty()) + Expect(list.OfficialLibraries).To(BeTrue()) + + const added = "https://example.org/extra-models.yaml" + w = send(e, http.MethodPost, "/api/library/sources", `{"kind":"model","url":"`+added+`","format":"gallery"}`) + Expect(w.Code).To(Equal(http.StatusCreated), w.Body.String()) + // The owner holds the source; the frontend wrote nothing. + var onOwner []string + for _, s := range owner.AddonSources().Entries() { + onOwner = append(onOwner, s.URL) + } + Expect(onOwner).To(ContainElement(added)) + Expect(filepath.Join(frontendData, "addons")).NotTo(BeADirectory()) + // The frontend that forwarded the change follows at once; the other one + // at its next refresh. + Expect(urls(first.ApplicationConfig().Galleries)).To(ContainElement(added)) + Expect(urls(second.ApplicationConfig().Galleries)).NotTo(ContainElement(added)) + Expect(second.RefreshLibraries(context.Background())).To(Succeed()) + Expect(urls(second.ApplicationConfig().Galleries)).To(ContainElement(added)) + Expect(second.LibraryListsStale()).To(BeFalse()) + }) + + It("answers the statuses of the owner's own routes", func() { + newApp("owner", dir(), true) + e := serve(newApp("frontend", dir(), true), nil) + body := `{"kind":"model","url":"https://example.org/dup.yaml","id":"dup"}` + Expect(send(e, http.MethodPost, "/api/library/sources", body).Code).To(Equal(http.StatusCreated)) + Expect(send(e, http.MethodPost, "/api/library/sources", body).Code).To(Equal(http.StatusConflict)) + Expect(send(e, http.MethodPost, "/api/library/sources/nope/enable", "").Code).To(Equal(http.StatusNotFound)) + Expect(send(e, http.MethodPost, "/api/library/sources", `{"kind":"x","url":"https://example.org/a"}`).Code).To(Equal(http.StatusBadRequest)) + Expect(send(e, http.MethodPost, "/api/library/sources/dup/disable", "").Code).To(Equal(http.StatusOK)) + Expect(send(e, http.MethodDelete, "/api/library/sources/dup", "").Code).To(Equal(http.StatusOK)) + }) + + It("refuses a write with the 503 envelope while the owner is down", func() { + owner := newApp("owner", dir(), true) + frontend := newApp("frontend", dir(), true) + e := serve(frontend, nil) + Expect(send(e, http.MethodGet, "/api/library/sources", "").Code).To(Equal(http.StatusOK)) + stop(owner) + for _, c := range []struct{ method, path, body string }{ + {http.MethodPost, "/api/library/sources", `{"kind":"model","url":"https://example.org/a.yaml"}`}, + {http.MethodGet, "/api/library/sources", ""}, + } { + w := send(e, c.method, c.path, c.body) + Expect(w.Code).To(Equal(http.StatusServiceUnavailable), c.method) + message, code := envelope(w) + Expect(message).To(Equal("the add-on owner is not reachable")) + Expect(code).To(Equal("unavailable")) + } + }) + + It("refuses to change official_libraries on a frontend with 409", func() { + newApp("owner", dir(), true) + frontend := newApp("frontend", dir(), true) + e := serve(frontend, nil) + w := send(e, http.MethodPost, "/api/settings", `{"official_libraries": false}`) + Expect(w.Code).To(Equal(http.StatusConflict), w.Body.String()) + Expect(frontend.ApplicationConfig().OfficialLibraries).To(BeTrue()) + }) + + It("rebuilds the gallery lists of a frontend from the owner after official_libraries changes there", func() { + owner := newApp("owner", dir(), true) + frontend := newApp("frontend", dir(), true) + same := func() { + ExpectWithOffset(1, frontend.RefreshLibraries(context.Background())).To(Succeed()) + ExpectWithOffset(1, urls(frontend.ApplicationConfig().Galleries)).To(Equal(urls(owner.ApplicationConfig().Galleries))) + ExpectWithOffset(1, urls(frontend.ApplicationConfig().BackendGalleries)).To(Equal(urls(owner.ApplicationConfig().BackendGalleries))) + ExpectWithOffset(1, frontend.ApplicationConfig().OfficialLibraries).To(Equal(owner.ApplicationConfig().OfficialLibraries)) + } + // The owner applies the shipped backend library when it is on. + Expect(owner.ApplyLibraries()).To(Succeed()) + same() + on := len(owner.ApplicationConfig().BackendGalleries) + + // What the settings handler does on the owner. + owner.ApplicationConfig().OfficialLibraries = false + Expect(owner.ApplyLibraries()).To(Succeed()) + Expect(owner.ApplicationConfig().BackendGalleries).To(HaveLen(on - 1)) + same() + Expect(frontend.ApplicationConfig().OfficialLibraries).To(BeFalse()) + + // And back on. + owner.ApplicationConfig().OfficialLibraries = true + Expect(owner.ApplyLibraries()).To(Succeed()) + same() + Expect(frontend.ApplicationConfig().BackendGalleries).To(HaveLen(on)) + }) + + It("keeps the last gallery lists and reports them stale while the owner is down", func() { + owner := newApp("owner", dir(), true) + frontend := newApp("frontend", dir(), true) + Expect(frontend.RefreshLibraries(context.Background())).To(Succeed()) + before := urls(frontend.ApplicationConfig().Galleries) + Expect(before).NotTo(BeEmpty()) + stop(owner) + Expect(frontend.RefreshLibraries(context.Background())).NotTo(Succeed()) + Expect(frontend.LibraryListsStale()).To(BeTrue()) + Expect(urls(frontend.ApplicationConfig().Galleries)).To(Equal(before)) + }) + + It("refuses a legacy API key on a frontend library route", func() { + newApp("owner", dir(), true) + e := serve(newApp("frontend", dir(), true), &auth.User{ID: "legacy-api-key", Role: auth.RoleAdmin}) + for _, c := range []struct{ method, path string }{ + {http.MethodGet, "/api/library/sources"}, + {http.MethodPost, "/api/library/sources"}, + {http.MethodDelete, "/api/library/sources/x"}, + } { + Expect(send(e, c.method, c.path, `{"kind":"model","url":"https://example.org/a.yaml"}`).Code).To(Equal(http.StatusUnauthorized), fmt.Sprint(c)) + } + }) +}) diff --git a/core/http/routes/localai.go b/core/http/routes/localai.go index 03b35ee9a149..3cd6ad8b71d5 100644 --- a/core/http/routes/localai.go +++ b/core/http/routes/localai.go @@ -23,6 +23,35 @@ import ( echoswagger "github.com/swaggo/echo-swagger" ) +// RegisterLibrarySourceRoutes registers the library-source routes. On a +// combined or owner process they work on the local store. On a frontend with +// add-ons on they forward to the owner, which is the only writer, after the +// actor middleware. On a frontend with add-ons off there is no owner to reach, +// and they answer unavailable. +func RegisterLibrarySourceRoutes(router *echo.Echo, app *application.Application, appConfig *config.ApplicationConfig, adminMiddleware echo.MiddlewareFunc) { + librarySources := localai.LibrarySourcesEndpoints{Admin: app.LibraryAdmin()} + mws := []echo.MiddlewareFunc{adminMiddleware} + if app.AddonFrontend() { + if !app.AddonsEnabled() { + refuse := middleware.LibrarySourcesOnOwner() + router.GET("/api/library/sources", refuse, adminMiddleware) + router.POST("/api/library/sources", refuse, adminMiddleware) + router.POST("/api/library/sources/:id/enable", refuse, adminMiddleware) + router.POST("/api/library/sources/:id/disable", refuse, adminMiddleware) + router.PUT("/api/library/sources/:id", refuse, adminMiddleware) + router.DELETE("/api/library/sources/:id", refuse, adminMiddleware) + return + } + mws = append(mws, localai.LibraryActor(app)) + } + router.GET("/api/library/sources", librarySources.List(), mws...) + router.POST("/api/library/sources", librarySources.Add(), mws...) + router.POST("/api/library/sources/:id/enable", librarySources.SetEnabled(true), mws...) + router.POST("/api/library/sources/:id/disable", librarySources.SetEnabled(false), mws...) + router.PUT("/api/library/sources/:id", librarySources.Update(), mws...) + router.DELETE("/api/library/sources/:id", librarySources.Remove(), mws...) +} + func RegisterLocalAIRoutes(router *echo.Echo, requestExtractor *middleware.RequestExtractor, cl *config.ModelConfigLoader, @@ -45,17 +74,7 @@ func RegisterLocalAIRoutes(router *echo.Echo, // Library sources of the add-on, backend and model kinds: the same store // and functions as `local-ai library`. Administrators only. - librarySources := localai.LibrarySourcesEndpoints{ - Store: app.LibrarySources, - Official: func() bool { return appConfig.OfficialLibraries }, - Applied: app.ApplyLibraries, - } - router.GET("/api/library/sources", librarySources.List(), adminMiddleware) - router.POST("/api/library/sources", librarySources.Add(), adminMiddleware) - router.POST("/api/library/sources/:id/enable", librarySources.SetEnabled(true), adminMiddleware) - router.POST("/api/library/sources/:id/disable", librarySources.SetEnabled(false), adminMiddleware) - router.PUT("/api/library/sources/:id", librarySources.Update(), adminMiddleware) - router.DELETE("/api/library/sources/:id", librarySources.Remove(), adminMiddleware) + RegisterLibrarySourceRoutes(router, app, appConfig, adminMiddleware) motion := localai.NewMotionEndpoints(app) router.POST("/api/motion/sessions", motion.Create) @@ -94,8 +113,18 @@ func RegisterLocalAIRoutes(router *echo.Echo, app.UpgradeChecker()) router.POST("/backends/apply", backendGalleryEndpointService.ApplyBackendEndpoint(appConfig.SystemState), adminMiddleware, middleware.RequireBackendInstallEnabled(appConfig)) if lib := app.BackendLibrary(); lib != nil { - router.POST("/backends/library/install", localai.InstallBackendFromLibraryEndpoint(lib, galleryService), adminMiddleware, middleware.RequireBackendInstallEnabled(appConfig)) - router.GET("/backends/library/updates", localai.BackendLibraryUpdatesEndpoint(lib), adminMiddleware) + // On a frontend the owner resolves the entry and this node installs + // it, so the request carries the actor that the owner checks. + libMws := []echo.MiddlewareFunc{adminMiddleware} + if app.AddonFrontend() { + libMws = append(libMws, localai.LibraryActor(app)) + } + router.POST("/backends/library/install", localai.InstallBackendFromLibraryEndpoint(lib, galleryService), append(libMws, middleware.RequireBackendInstallEnabled(appConfig))...) + router.GET("/backends/library/updates", localai.BackendLibraryUpdatesEndpoint(lib), libMws...) + } else if app.AddonFrontend() { + refuse := middleware.LibrarySourcesOnOwner() + router.POST("/backends/library/install", refuse, adminMiddleware) + router.GET("/backends/library/updates", refuse, adminMiddleware) } router.POST("/backends/delete/:name", backendGalleryEndpointService.DeleteBackendEndpoint(), adminMiddleware) router.GET("/backends", backendGalleryEndpointService.ListBackendsEndpoint(), adminMiddleware) @@ -493,18 +522,7 @@ func RegisterLocalAIRoutes(router *echo.Echo, router.GET("/api/instructions", localai.ListAPIInstructionsEndpoint()) router.GET("/api/instructions/:name", localai.GetAPIInstructionEndpoint()) - router.GET("/api/features", func(c echo.Context) error { - return c.JSON(200, map[string]bool{ - "agents": appConfig.AgentPool.Enabled, - "mcp": !appConfig.DisableMCP, - "fine_tuning": true, - "quantization": true, - "distributed": appConfig.Distributed.Enabled, - "backend_install": !appConfig.DisableBackendInstall, - "localai_assistant": !appConfig.DisableLocalAIAssistant && app.LocalAIAssistant() != nil, - "addon_navigation": !appConfig.LegacyNavigation, - }) - }) + RegisterFeatureRoutes(router, appConfig, app) router.GET("/system", localai.SystemInformations(cl, ml, appConfig, monitoring.NewLocalProcessSampler()), adminMiddleware) diff --git a/core/http/routes/quantization.go b/core/http/routes/quantization.go index 92f5a15218f0..698650a92533 100644 --- a/core/http/routes/quantization.go +++ b/core/http/routes/quantization.go @@ -7,6 +7,7 @@ import ( "github.com/mudler/LocalAI/core/application" "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/http/endpoints/localai" + httpMiddleware "github.com/mudler/LocalAI/core/http/middleware" "github.com/mudler/LocalAI/core/services/quantization" ) @@ -28,7 +29,8 @@ func RegisterQuantizationRoutes(e *echo.Echo, qService *quantization.Quantizatio } } - q := e.Group("/api/quantization", readyMw, quantizationMw) + q := e.Group("/api/quantization", httpMiddleware.RequireAddonsFeature(app), readyMw, quantizationMw) + q.GET("/status", localai.ModelExportsStatusQuantizationEndpoint(app)) q.GET("/backends", localai.ListQuantizationBackendsEndpoint(appConfig, ClusterCapabilityProviderFor(app), ClusterInstalledProviderFor(app))) q.POST("/jobs", localai.StartQuantizationJobEndpoint(qService)) q.GET("/jobs", localai.ListQuantizationJobsEndpoint(qService)) @@ -36,6 +38,6 @@ func RegisterQuantizationRoutes(e *echo.Echo, qService *quantization.Quantizatio q.POST("/jobs/:id/stop", localai.StopQuantizationJobEndpoint(qService)) q.DELETE("/jobs/:id", localai.DeleteQuantizationJobEndpoint(qService)) q.GET("/jobs/:id/progress", localai.QuantizationProgressEndpoint(qService)) - q.POST("/jobs/:id/import", localai.ImportQuantizedModelEndpoint(qService)) + q.POST("/jobs/:id/import", localai.ImportQuantizedModelEndpoint(qService), httpMiddleware.RequireModelExports(app)) q.GET("/jobs/:id/download", localai.DownloadQuantizedModelEndpoint(qService)) } diff --git a/core/library/admin.go b/core/library/admin.go new file mode 100644 index 000000000000..6aa38e575d67 --- /dev/null +++ b/core/library/admin.go @@ -0,0 +1,275 @@ +// SPDX-License-Identifier: MIT +package library + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "strings" + + "github.com/mudler/LocalAI/core/services/addons/catalog" + "github.com/mudler/LocalAI/pkg/addons/contract" + "github.com/mudler/xlog" +) + +// ErrStoreUnavailable says the source store cannot be opened or reached. +var ErrStoreUnavailable = errors.New("the library sources are not available") + +// BadRequestError is a request the admin refuses as invalid. Message is shown +// to the caller as written. +type BadRequestError struct{ Message string } + +func (e *BadRequestError) Error() string { return e.Message } + +func badRequest(format string, args ...any) error { + return &BadRequestError{Message: fmt.Sprintf(format, args...)} +} + +// Listing is the answer of List: every source with its state and the +// official-libraries switch of the process that owns the store. +type Listing struct { + Sources []Info `json:"sources"` + OfficialLibraries bool `json:"officialLibraries"` + // Version identifies the settings and the source list as far as they + // decide the galleries a server reads. A caller that holds a version asks + // for the list again only when it changed. + Version string `json:"version"` + // Unchanged is true when the caller's version is the current one; Sources + // is then empty. + Unchanged bool `json:"unchanged,omitempty"` +} + +// AddRequest is a source to add. +type AddRequest struct { + Kind string `json:"kind"` + URL string `json:"url"` + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Format string `json:"format,omitempty"` + Mirrors []string `json:"mirrors,omitempty"` +} + +// RemoveResult says what a removal did. +type RemoveResult struct { + ID string `json:"id"` + Default bool `json:"default"` + Tombstone bool `json:"tombstone"` +} + +// Admin is the administration of the library sources. The process that owns +// the store implements it on the store (LocalAdmin); a frontend implements it +// by forwarding to the owner. The HTTP routes work on this interface only, so +// they answer the same on both. +type Admin interface { + // List returns the sources. A non-empty known that equals the current + // version gives an Unchanged listing without sources. + List(ctx context.Context, known string) (Listing, error) + Add(ctx context.Context, req AddRequest) (Info, error) + // SetMirrors replaces the mirrors of a source. A nil Info means the + // change was stored but the source is not listed any more. + SetMirrors(ctx context.Context, id string, mirrors []string) (*Info, error) + SetEnabled(ctx context.Context, id string, enabled bool) (*Info, error) + Remove(ctx context.Context, id string) (RemoveResult, error) +} + +type actorKey struct{} + +// WithActor returns a context that carries the actor of an HTTP request, for +// the Admin of a frontend, which forwards it to the owner. +func WithActor(ctx context.Context, a contract.Actor) context.Context { + return context.WithValue(ctx, actorKey{}, a) +} + +// ActorFrom returns the actor WithActor stored. +func ActorFrom(ctx context.Context) (contract.Actor, bool) { + a, ok := ctx.Value(actorKey{}).(contract.Actor) + return a, ok +} + +// Version is the identity of a source list and the official-libraries switch. +// It covers what decides the galleries a server reads and leaves out the fetch +// state, which changes on every refresh. +func Version(infos []Info, official bool) string { + type row struct { + Kind string + ID, URL, Format string + Enabled, Default, Removed, Suspended bool + Mirrors []string + } + rows := make([]row, 0, len(infos)) + for _, in := range infos { + rows = append(rows, row{string(in.Kind), in.ID, in.URL, in.Format, in.Enabled, in.Default, in.Removed, in.Suspended, in.Mirrors}) + } + b, _ := json.Marshal(struct { + Official bool + Rows []row + }{official, rows}) + sum := sha256.Sum256(b) + return hex.EncodeToString(sum[:16]) +} + +// CleanMirrors trims the mirrors and checks them as a gallery address is +// checked. An add-on source takes none. +func CleanMirrors(kind string, in []string) ([]string, error) { + var out []string + for _, m := range in { + if m = strings.TrimSpace(m); m != "" { + out = append(out, m) + } + } + if len(out) > 0 && kind == string(catalog.KindAddon) { + return nil, badRequest("an add-on source takes no mirrors") + } + if err := catalog.ValidateMirrors(out); err != nil { + return nil, &BadRequestError{Message: err.Error()} + } + return out, nil +} + +// LocalAdmin is the Admin of the process that holds the store. +type LocalAdmin struct { + // Store returns the one store of the running server. + Store func() (*catalog.Sources, error) + // Official says whether the official libraries are on. + Official func() bool + // Applied runs after a change, so the server reads the new state. + Applied func(dropped ...string) error +} + +func (a LocalAdmin) open() (*catalog.Sources, error) { + store, err := a.Store() + if err != nil { + xlog.Warn("library sources are not available", "error", err) + return nil, ErrStoreUnavailable + } + return store, nil +} + +func (a LocalAdmin) applied(dropped ...string) { + if a.Applied == nil { + return + } + if err := a.Applied(dropped...); err != nil { + xlog.Warn("could not apply the library sources", "error", err) + } +} + +func (a LocalAdmin) official() bool { + if a.Official == nil { + return true + } + return a.Official() +} + +func info(store *catalog.Sources, id string) *Info { + for _, in := range List(store) { + if in.ID == id { + return &in + } + } + return nil +} + +func (a LocalAdmin) List(_ context.Context, known string) (Listing, error) { + store, err := a.open() + if err != nil { + return Listing{}, err + } + infos, official := List(store), a.official() + v := Version(infos, official) + if known != "" && known == v { + return Listing{OfficialLibraries: official, Version: v, Unchanged: true}, nil + } + return Listing{Sources: infos, OfficialLibraries: official, Version: v}, nil +} + +func (a LocalAdmin) Add(_ context.Context, req AddRequest) (Info, error) { + req.URL, req.ID, req.Name = strings.TrimSpace(req.URL), strings.TrimSpace(req.ID), strings.TrimSpace(req.Name) + switch req.Kind { + case string(catalog.KindAddon), string(catalog.KindBackend), string(catalog.KindModel): + default: + return Info{}, badRequest("kind must be addon, backend or model") + } + format := "" + switch req.Format { + case "", "library": + case "gallery": + format = catalog.FormatGallery + default: + return Info{}, badRequest("format must be library or gallery") + } + if req.URL == "" { + return Info{}, badRequest("url is required") + } + mirrors, err := CleanMirrors(req.Kind, req.Mirrors) + if err != nil { + return Info{}, err + } + store, err := a.open() + if err != nil { + return Info{}, err + } + src, err := AddWithMirrors(store, catalog.Kind(req.Kind), req.URL, req.ID, req.Name, format, mirrors) + if err != nil { + return Info{}, err + } + a.applied() + if in := info(store, src.ID); in != nil { + return *in, nil + } + return Info{Kind: src.Kind, ID: src.ID, Name: src.Name, URL: src.URL, Enabled: true}, nil +} + +func (a LocalAdmin) SetMirrors(_ context.Context, id string, mirrors []string) (*Info, error) { + store, err := a.open() + if err != nil { + return nil, err + } + kind := "" + if in := info(store, id); in != nil { + kind = string(in.Kind) + } + clean, err := CleanMirrors(kind, mirrors) + if err != nil { + return nil, err + } + if err := store.SetMirrors(id, clean); err != nil { + return nil, err + } + a.applied() + return info(store, id), nil +} + +func (a LocalAdmin) SetEnabled(_ context.Context, id string, enabled bool) (*Info, error) { + store, err := a.open() + if err != nil { + return nil, err + } + if err := store.SetEnabled(id, enabled); err != nil { + return nil, err + } + a.applied() + return info(store, id), nil +} + +func (a LocalAdmin) Remove(_ context.Context, id string) (RemoveResult, error) { + store, err := a.open() + if err != nil { + return RemoveResult{}, err + } + url := URLOf(store, id) + wasDefault, err := Remove(store, id) + if err != nil { + return RemoveResult{}, err + } + // A tombstone keeps its record, so only a deleted source is dropped by address. + if wasDefault { + a.applied() + } else { + a.applied(url) + } + return RemoveResult{ID: id, Default: wasDefault, Tombstone: wasDefault}, nil +} diff --git a/core/library/startup.go b/core/library/startup.go index 4e67dc7d3a55..6b79bbbb70ce 100644 --- a/core/library/startup.go +++ b/core/library/startup.go @@ -143,16 +143,31 @@ func Option(dataPath string, official, backendLibrary, useOfficial bool) config. // dropped names the addresses of sources that were deleted: a deleted source is // no longer in the store, so only the caller can say its gallery must go. func Reconcile(store *catalog.Sources, models, backends []config.Gallery, dropped ...string) ([]config.Gallery, []config.Gallery) { - return reconcileKind(store, catalog.KindModel, models, dropped), reconcileKind(store, catalog.KindBackend, backends, dropped) + entries := store.Entries() + return reconcileKind(entries, catalog.KindModel, models, dropped), reconcileKind(entries, catalog.KindBackend, backends, dropped) } -func reconcileKind(store *catalog.Sources, kind catalog.Kind, current []config.Gallery, dropped []string) []config.Gallery { +// ReconcileInfos is Reconcile for a source list that was read from the owner +// of the store, as a frontend does. Each Info carries the state the host acts +// on, so the result is the one the owner's own Reconcile gives. +func ReconcileInfos(infos []Info, models, backends []config.Gallery) ([]config.Gallery, []config.Gallery) { + entries := make([]catalog.Source, 0, len(infos)) + for _, in := range infos { + entries = append(entries, catalog.Source{ + ID: in.ID, Name: in.Name, URL: in.URL, Kind: in.Kind, Format: in.Format, + Enabled: in.Enabled, Default: in.Default, Removed: in.Removed, Suspended: in.Suspended, Mirrors: in.Mirrors, + }) + } + return reconcileKind(entries, catalog.KindModel, models, nil), reconcileKind(entries, catalog.KindBackend, backends, nil) +} + +func reconcileKind(entries []catalog.Source, kind catalog.Kind, current []config.Gallery, dropped []string) []config.Gallery { live := map[string]catalog.Source{} known := map[string]bool{} for _, u := range dropped { known[u] = true } - for _, e := range store.Entries() { + for _, e := range entries { if e.Kind == kind { known[e.URL] = true } @@ -174,7 +189,7 @@ func reconcileKind(store *catalog.Sources, kind catalog.Kind, current []config.G have[g.URL] = true } } - for _, e := range store.Entries() { + for _, e := range entries { if _, ok := live[e.URL]; ok && e.Kind == kind && readsAsGallery(kind, e) && !have[e.URL] { out = append(out, config.Gallery{Name: e.Name, URL: e.URL, Mirrors: e.Mirrors, Library: e.Format != catalog.FormatGallery}) have[e.URL] = true diff --git a/core/services/addons/backendlib/install.go b/core/services/addons/backendlib/install.go index edb46ba61116..6be3bc655fa5 100644 --- a/core/services/addons/backendlib/install.go +++ b/core/services/addons/backendlib/install.go @@ -42,9 +42,55 @@ var ( ErrNoCapability = errors.New("the meta backend has no backend for this host") ) +// Plan is what the owner of the libraries decides for one install: the entry +// resolved, a meta backend replaced by the backend that serves the capability +// of the asking host, and the image pinned by digest with the library's +// signature policy. Whoever installs it runs the gallery install on its own +// node and applies its own size limit and strict mode. A Plan crosses the +// carrier between a frontend and the owner. +type Plan struct { + // Name and Library are the backend ID and the library it came from. + Name string `json:"name"` + Library string `json:"library"` + Alias string `json:"alias,omitempty"` + // URI and Mirrors name the image by digest. + URI string `json:"uri"` + Mirrors []string `json:"mirrors,omitempty"` + Version string `json:"version"` + // SourceURL is the address of the library index. + SourceURL string `json:"sourceUrl,omitempty"` + // Verification is set when the library requires a signature. + Verification *config.GalleryVerification `json:"verification,omitempty"` +} + +// Ref names one library backend whose releases a caller asks for. +type Ref struct { + SourceID string `json:"sourceId"` + Name string `json:"name"` +} + +// Remote is the owner of the libraries, reached over the carrier. A frontend +// has no library index: it asks the owner to resolve an entry and to list the +// releases of the backends it has installed. +type Remote interface { + // Resolve returns the plan of an entry. capability is the capability + // the asking host reports, used to pick the backend of a meta backend. + Resolve(ctx context.Context, sourceID, backend, version, capability string) (Plan, error) + // Releases returns the releases of each ref, by "/". + Releases(ctx context.Context, refs []Ref) (map[string][]catalog.Release, error) + // Target is the OS and architecture the owner resolves for. + Target(ctx context.Context) (string, string, error) +} + +// RefKey is the key of a ref in the map that Remote.Releases returns. +func RefKey(sourceID, name string) string { return sourceID + "/" + name } + // Installer installs library backends on the local node. type Installer struct { - Catalog catalog.Catalog + Catalog catalog.Catalog + // Remote, when set, replaces Catalog: the owner resolves the entry and + // this node installs it. + Remote Remote SystemState *system.SystemState ModelLoader *model.ModelLoader // MaxBytes bounds the compressed size of the image. Zero means the default @@ -107,42 +153,86 @@ func (p Progress) step(s string) { // or the channel of the entry, as for an add-on. func (i *Installer) Install(ctx context.Context, sourceID, backend, version string, progress Progress) error { progress.step(StepResolving) - r, err := i.Catalog.ResolveKind(ctx, catalog.KindBackend, sourceID, backend, version) + var plan Plan + var err error + if i.Remote != nil { + capability := "" + if i.SystemState != nil { + capability = i.SystemState.ReportedCapability() + } + plan, err = i.Remote.Resolve(ctx, sourceID, backend, version, capability) + } else { + plan, err = i.Resolve(ctx, sourceID, backend, version, "") + } if err != nil { return err } + return i.installPlan(ctx, plan, progress) +} + +// Resolve is the owner's half of an install: it resolves the entry, replaces a +// meta backend by the backend that serves capability, and pins the image. An +// empty capability means the capability of this host. It installs nothing and +// checks no collision, because the node that installs may be another one. +func (i *Installer) Resolve(ctx context.Context, sourceID, backend, version, capability string) (Plan, error) { + r, err := i.Catalog.ResolveKind(ctx, catalog.KindBackend, sourceID, backend, version) + if err != nil { + return Plan{}, err + } if r.IsMeta() { // A meta backend installs nothing itself: the host picks the backend // that serves its capability and that one is installed, pinned and // verified like any other. - if r, err = i.concrete(ctx, sourceID, r); err != nil { - return err + if r, err = i.concrete(ctx, sourceID, r, capability); err != nil { + return Plan{}, err } } gb, err := i.plan(ctx, r) if err != nil { - return err + return Plan{}, err } - if err := i.checkCollision(r); err != nil { + p := Plan{Name: gb.Name, Library: gb.Library, Alias: gb.Alias, URI: gb.URI, Mirrors: gb.Mirrors, Version: gb.Version, SourceURL: gb.Gallery.URL, Verification: gb.Gallery.Verification} + return p, nil +} + +// installPlan is the node's half: collision check and the gallery install, with +// this node's strict mode and size limit. +func (i *Installer) installPlan(ctx context.Context, p Plan, progress Progress) error { + if p.Name == "" || !strings.HasPrefix(p.URI, "oci://") { + return ErrInvalid + } + if i.RequireIntegrity && p.Verification == nil { + return fmt.Errorf("%w (library %q)", ErrIntegrity, p.Library) + } + if err := i.checkCollision(p.Name, p.Library); err != nil { return err } install := i.install if install == nil { install = gallery.InstallBackend } + gb := &gallery.GalleryBackend{URI: p.URI, Version: p.Version, Alias: p.Alias, Library: p.Library, MaxBytes: i.limit(), Mirrors: p.Mirrors} + gb.Name = p.Name + gb.Gallery = config.Gallery{Name: "library:" + p.Library, URL: p.SourceURL, Verification: p.Verification} // With a verification policy the gallery install verifies the signature of // the pinned image before it fetches any layer, then applies the size limit. gb.Stage = progress.Step return install(ctx, i.SystemState, i.ModelLoader, gb, progress.Bytes, i.RequireIntegrity) } -// concrete returns the release a meta backend names for this host, in the same -// library and the same channel. -func (i *Installer) concrete(ctx context.Context, sourceID string, meta catalog.Release) (catalog.Release, error) { - if i.SystemState == nil || meta.Channel == "" { +// concrete returns the release a meta backend names for a host that reports +// capability (this host when it is empty), in the same library and channel. +func (i *Installer) concrete(ctx context.Context, sourceID string, meta catalog.Release, capability string) (catalog.Release, error) { + if meta.Channel == "" || (capability == "" && i.SystemState == nil) { return catalog.Release{}, ErrInvalid } - id := meta.Capabilities[i.SystemState.Capability(meta.Capabilities)] + var pick string + if capability == "" { + pick = i.SystemState.Capability(meta.Capabilities) + } else { + pick = system.PickCapability(capability, meta.Capabilities) + } + id := meta.Capabilities[pick] if id == "" || id == meta.ID { return catalog.Release{}, fmt.Errorf("%w: %q", ErrNoCapability, meta.ID) } @@ -220,7 +310,7 @@ func (i *Installer) plan(ctx context.Context, r catalog.Release) (*gallery.Galle // checkCollision refuses to replace a backend that did not come from this // library: one installed from a gallery, from another library or by hand. // Installing again from the same library is a reinstall. -func (i *Installer) checkCollision(r catalog.Release) error { +func (i *Installer) checkCollision(name, library string) error { backends, err := gallery.ListSystemBackends(i.SystemState) if errors.Is(err, os.ErrNotExist) { // No backends directory yet: nothing can collide. @@ -229,12 +319,12 @@ func (i *Installer) checkCollision(r catalog.Release) error { if err != nil { return err } - have, ok := backends.Get(r.ID) + have, ok := backends.Get(name) if !ok || have.IsSystem { return nil } - if have.Metadata == nil || have.Metadata.Library != r.SourceID { - return fmt.Errorf("%w: %q", ErrCollision, r.ID) + if have.Metadata == nil || have.Metadata.Library != library { + return fmt.Errorf("%w: %q", ErrCollision, name) } return nil } diff --git a/core/services/addons/backendlib/remote_test.go b/core/services/addons/backendlib/remote_test.go new file mode 100644 index 000000000000..a7fad5bce473 --- /dev/null +++ b/core/services/addons/backendlib/remote_test.go @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: MIT +package backendlib + +import ( + "context" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + + "github.com/google/go-containerregistry/pkg/registry" + "github.com/mudler/LocalAI/core/services/addons/catalog" + "github.com/mudler/LocalAI/pkg/addons/contract" + "github.com/mudler/LocalAI/pkg/model" + "github.com/mudler/LocalAI/pkg/system" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// ownerRemote answers a frontend's installer from the owner's installer, as the +// carrier does. +type ownerRemote struct{ owner *Installer } + +func (r ownerRemote) Resolve(ctx context.Context, sourceID, backend, version, capability string) (Plan, error) { + return r.owner.Resolve(ctx, sourceID, backend, version, capability) +} + +func (r ownerRemote) Releases(ctx context.Context, refs []Ref) (map[string][]catalog.Release, error) { + out := map[string][]catalog.Release{} + for _, ref := range refs { + rel, err := r.owner.Catalog.KindReleases(ctx, catalog.KindBackend, ref.SourceID, ref.Name) + if err == nil { + out[RefKey(ref.SourceID, ref.Name)] = rel + } + } + return out, nil +} + +func (r ownerRemote) Target(context.Context) (string, string, error) { + os, arch := r.owner.Catalog.Target() + return os, arch, nil +} + +var _ = Describe("a frontend installing through the owner's plan", func() { + It("resolves on the owner, installs on the frontend's own backends path and reports updates", func() { + reg := httptest.NewServer(registry.New()) + DeferCleanup(reg.Close) + host := strings.TrimPrefix(reg.URL, "http://") + digest := push(host, "acme/backends", "latest-cpu-demo", "#!/bin/sh\necho demo\n") + release := catalog.Release{ + Kind: catalog.KindBackend, ID: "cpu-demo", Name: "Demo (CPU)", Version: "0.0.0-latest.aaaaaaaaaaaa", + Artifact: "oci://" + host + "/acme/backends:latest-cpu-demo", Digest: digest, OS: "linux", Arch: "amd64", + } + lib := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _ = catalog.WriteIndex(w, []catalog.Release{release}) + })) + DeferCleanup(lib.Close) + s, err := catalog.NewSources(filepath.Join(GinkgoT().TempDir(), "sources.json"), allow, catalog.CredentialResolver{}) + Expect(err).NotTo(HaveOccurred()) + Expect(s.Save(ctx, contract.Actor{}, catalog.Source{ID: "pilot", Name: "Pilot", URL: lib.URL, Enabled: true})).To(Succeed()) + _, err = s.Refresh(ctx, "pilot") + Expect(err).NotTo(HaveOccurred()) + + ownerDir, frontendDir := GinkgoT().TempDir(), GinkgoT().TempDir() + ownerState, err := system.GetSystemState(system.WithBackendPath(ownerDir)) + Expect(err).NotTo(HaveOccurred()) + frontState, err := system.GetSystemState(system.WithBackendPath(frontendDir)) + Expect(err).NotTo(HaveOccurred()) + owner := &Installer{Catalog: catalog.Catalog{Sources: s, TargetOS: "linux", TargetArch: "amd64"}, SystemState: ownerState, ModelLoader: model.NewModelLoader(ownerState)} + frontend := &Installer{Remote: ownerRemote{owner}, SystemState: frontState, ModelLoader: model.NewModelLoader(frontState)} + + plan, err := owner.Resolve(ctx, "pilot", "cpu-demo", release.Version, "") + Expect(err).NotTo(HaveOccurred()) + Expect(plan.URI).To(ContainSubstring("@" + digest)) + Expect(frontend.Install(ctx, "pilot", "cpu-demo", release.Version, Progress{})).To(Succeed()) + Expect(filepath.Join(frontendDir, "cpu-demo", "run.sh")).To(BeARegularFile()) + Expect(ownerDir).To(BeADirectory()) + Expect(filepath.Join(ownerDir, "cpu-demo")).NotTo(BeADirectory()) + + // The frontend's strict mode applies to a plan with no signature. + frontend.RequireIntegrity = true + Expect(frontend.Install(ctx, "pilot", "cpu-demo", release.Version, Progress{})).To(MatchError(ErrIntegrity)) + + // The owner's cache lists the same digest, so there is no update. + updates, err := frontend.Updates(ctx) + Expect(err).NotTo(HaveOccurred()) + Expect(updates).To(BeEmpty()) + }) +}) diff --git a/core/services/addons/backendlib/updates.go b/core/services/addons/backendlib/updates.go index a9a075aa938b..76d90bddb5a2 100644 --- a/core/services/addons/backendlib/updates.go +++ b/core/services/addons/backendlib/updates.go @@ -48,14 +48,34 @@ func (i *Installer) Updates(ctx context.Context) ([]Update, error) { if err != nil { return nil, err } - target, arch := i.Catalog.Target() - out := []Update{} + var target, arch string + if i.Remote != nil { + if target, arch, err = i.Remote.Target(ctx); err != nil { + return nil, err + } + } else { + target, arch = i.Catalog.Target() + } + var refs []Ref for name, b := range backends { if b.IsSystem || b.Metadata == nil || b.Metadata.Library == "" || b.Metadata.Digest == "" { continue } - releases, err := i.Catalog.KindReleases(ctx, catalog.KindBackend, b.Metadata.Library, name) - if err != nil { + refs = append(refs, Ref{SourceID: b.Metadata.Library, Name: name}) + } + var remote map[string][]catalog.Release + if i.Remote != nil && len(refs) > 0 { + if remote, err = i.Remote.Releases(ctx, refs); err != nil { + return nil, err + } + } + out := []Update{} + for _, ref := range refs { + b, name := backends[ref.Name], ref.Name + var releases []catalog.Release + if i.Remote != nil { + releases = remote[RefKey(ref.SourceID, ref.Name)] + } else if releases, err = i.Catalog.KindReleases(ctx, catalog.KindBackend, b.Metadata.Library, name); err != nil { if ctx.Err() != nil { return nil, ctx.Err() } diff --git a/core/services/addons/gateway/carrier.go b/core/services/addons/gateway/carrier.go index 47aa015c243a..bdb325c9dfcf 100644 --- a/core/services/addons/gateway/carrier.go +++ b/core/services/addons/gateway/carrier.go @@ -13,6 +13,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/mudler/LocalAI/core/services/addons/modelsprobe" "io" "net/http" "net/url" @@ -41,6 +42,7 @@ type CarrierConfig struct { type Carrier struct { config CarrierConfig client *http.Client + caps capsCache } type OutcomeUnknown struct { OperationID string `json:"operationId"` @@ -48,6 +50,17 @@ type OutcomeUnknown struct { func (e *OutcomeUnknown) Error() string { return "addon mutation outcome unknown" } func (e *OutcomeUnknown) Unwrap() error { return ErrUnavailable } + +// validSigningKey reports whether the signing key fits the role. A frontend +// signs every request it forwards and needs a full key. The owner never signs +// outbound requests, so it needs none; a key it has must still be well formed. +func validSigningKey(c CarrierConfig) bool { + if len(c.SigningKey) == 0 { + return c.InstanceID == c.OwnerID + } + return len(c.SigningKey) == ed25519.PrivateKeySize +} + func NewCarrier(c CarrierConfig) (*Carrier, error) { limits, e := c.Limits.resolved() if e != nil { @@ -55,7 +68,7 @@ func NewCarrier(c CarrierConfig) (*Carrier, error) { } c.Limits = limits u, e := url.Parse(c.OwnerAddress) - if e != nil || u.Scheme != "https" || u.Host == "" || u.User != nil || u.Path != "" || u.RawQuery != "" || u.Fragment != "" || c.InstanceID == "" || c.OwnerID == "" || c.Roots == nil || len(c.Certificate.Certificate) == 0 || len(c.SigningKey) != ed25519.PrivateKeySize { + if e != nil || u.Scheme != "https" || u.Host == "" || u.User != nil || u.Path != "" || u.RawQuery != "" || u.Fragment != "" || c.InstanceID == "" || c.OwnerID == "" || c.Roots == nil || len(c.Certificate.Certificate) == 0 || !validSigningKey(c) { return nil, ErrForbidden } if _, ok := c.Pins[c.OwnerID]; !ok { @@ -122,6 +135,9 @@ type wireRequest struct { // grants, rename, deletion and conversation selections. Such a call is // never a typed dispatch. Resource *ResourceCall `json:"resource,omitempty"` + // Library marks a library call: a change of the library sources, or the + // owner's half of a backend install. Such a call is never a typed dispatch. + Library *LibraryCall `json:"library,omitempty"` } func decode(r io.Reader, dst any) error { @@ -161,6 +177,49 @@ func (c *Carrier) OwnerHandler(g *Gateway) (http.Handler, error) { }{v.Audience()}) return } + if r.Method == "GET" && r.URL.Path == "/addons/capabilities" { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(g.capabilities()) + return + } + if r.Method == "GET" && r.URL.Path == "/addons/models-probe" { + var p modelsprobe.Probe + ok := false + if g.ModelsProbe != nil { + p, ok = g.ModelsProbe() + } + if !ok { + typedError(w, ErrUnavailable) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(p) + return + } + if r.Method == "GET" && r.URL.Path == "/addons/library/snapshot" { + if g.Libraries == nil { + typedError(w, ErrUnavailable) + return + } + raw, e := g.Libraries.Snapshot(r.Context(), r.URL.Query().Get("known")) + if e != nil { + typedError(w, e) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(raw) + return + } + if r.Method == "POST" && r.URL.Path == "/addons/library/system" { + r.Body = http.MaxBytesReader(w, r.Body, 1<<20) + var call LibraryCall + if decode(r.Body, &call) != nil { + http.Error(w, "invalid request", 400) + return + } + g.librarySystemLocal(r.Context(), w, call) + return + } if r.Method != "POST" || r.URL.Path != "/addons/forward" { http.Error(w, "not found", 404) return @@ -181,9 +240,19 @@ func (c *Carrier) OwnerHandler(g *Gateway) (http.Handler, error) { http.Error(w, "invalid binding", 400) return } + if wire.Library != nil { + call := *wire.Library + // The owner classifies by the call, not by the action the peer wrote. + if wire.Relay != nil || wire.Proposal != nil || wire.Artifact != nil || wire.Apps != nil || wire.Resource != nil || !call.valid() || call.System || wire.Request.Action != call.wireAction() { + http.Error(w, "invalid binding", 400) + return + } + g.libraryLocal(r.Context(), w, actor, call) + return + } if wire.Resource != nil { call := *wire.Resource - if wire.Relay != nil || wire.Proposal != nil || wire.Artifact != nil || wire.Apps != nil || wire.Request.Action != call.wireAction() { + if wire.Relay != nil || wire.Proposal != nil || wire.Artifact != nil || wire.Apps != nil || wire.Library != nil || wire.Request.Action != call.wireAction() { http.Error(w, "invalid binding", 400) return } @@ -322,6 +391,40 @@ func StatusFor(e error) int { return http.StatusServiceUnavailable } +// Reasons that WriteUnavailable puts in the body, so a client can tell a +// disabled feature from an absent owner. +const ( + ReasonAddonsOff = "add-ons are turned off on this instance" + ReasonOwnerUnreached = "the add-on owner is not reachable" + ReasonLibraryOnOwner = "library sources are managed on the owner" + ReasonModelsNotShared = "the models path of this instance is not the shared models storage of the add-on owner; model exports are blocked" + unavailableErrorType = "server_error" + unavailableErrorCode = "unavailable" +) + +// unavailableBody has the shape of the LocalAI error envelope +// (schema.ErrorResponse), which the rest of the API writes. +type unavailableBody struct { + Error struct { + Message string `json:"message"` + Type string `json:"type"` + Code string `json:"code"` + } `json:"error"` +} + +// WriteUnavailable writes the response of an add-on request that no owner can +// serve: status 503, the status StatusFor gives ErrUnavailable, and the LocalAI +// error envelope with code "unavailable". The message says why. The add-on +// feature gate uses it so a disabled add-on side and an unreachable owner share +// one status and one shape, and differ only in the message. +func WriteUnavailable(w http.ResponseWriter, reason string) { + var b unavailableBody + b.Error.Message, b.Error.Type, b.Error.Code = reason, unavailableErrorType, unavailableErrorCode + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(StatusFor(ErrUnavailable)) + _ = json.NewEncoder(w).Encode(b) +} + // typedError writes the owner's typed failure response. func typedError(w http.ResponseWriter, e error) { if e = Refused(e); errors.Is(e, ErrInvalidRequest) { @@ -392,14 +495,16 @@ func (c *Carrier) relayBudget(kind string) (int64, error) { } return 0, ErrForbidden } -func (c *Carrier) send(ctx context.Context, a contract.Actor, wire wireRequest) (*http.Response, error) { + +// audience asks the owner for the audience its envelopes must name. +func (c *Carrier) audience(ctx context.Context) (string, error) { req, e := http.NewRequestWithContext(ctx, "GET", c.config.OwnerAddress+"/addons/audience", nil) if e != nil { - return nil, ErrUnavailable + return "", ErrUnavailable } resp, e := c.client.Do(req) if e != nil { - return nil, ErrUnavailable + return "", ErrUnavailable } var bootstrap struct { Audience string `json:"audience"` @@ -407,7 +512,48 @@ func (c *Carrier) send(ctx context.Context, a contract.Actor, wire wireRequest) e = decode(io.LimitReader(resp.Body, 4096), &bootstrap) _ = resp.Body.Close() if e != nil || resp.StatusCode != 200 || len(bootstrap.Audience) != 64 { - return nil, ErrUnavailable + return "", ErrUnavailable + } + return bootstrap.Audience, nil +} + +// ModelsProbe asks the owner for the probe it wrote into its models path. An +// owner that has none, or that does not know the call, gives ErrUnavailable. +func (c *Carrier) ModelsProbe(ctx context.Context) (modelsprobe.Probe, error) { + ctx, cancel := context.WithTimeout(ctx, ReachableTimeout) + defer cancel() + req, e := http.NewRequestWithContext(ctx, "GET", c.config.OwnerAddress+"/addons/models-probe", nil) + if e != nil { + return modelsprobe.Probe{}, ErrUnavailable + } + resp, e := c.client.Do(req) + if e != nil { + return modelsprobe.Probe{}, ErrUnavailable + } + defer func() { _ = resp.Body.Close() }() + var p modelsprobe.Probe + if resp.StatusCode != http.StatusOK || decode(io.LimitReader(resp.Body, 4096), &p) != nil || !p.Valid() { + return modelsprobe.Probe{}, ErrUnavailable + } + return p, nil +} + +// ReachableTimeout bounds one reachability probe of the owner. +const ReachableTimeout = 2 * time.Second + +// Reachable reports whether the owner answered the audience call. It signs +// nothing and changes nothing, and it has its own timeout. +func (c *Carrier) Reachable(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, ReachableTimeout) + defer cancel() + _, e := c.audience(ctx) + return e +} + +func (c *Carrier) send(ctx context.Context, a contract.Actor, wire wireRequest) (*http.Response, error) { + audience, e := c.audience(ctx) + if e != nil { + return nil, e } body, e := json.Marshal(wire) if e != nil || len(body) > 2<<20 { @@ -417,15 +563,19 @@ func (c *Carrier) send(ctx context.Context, a contract.Actor, wire wireRequest) if _, e = rand.Read(nonce[:]); e != nil { return nil, ErrUnavailable } - m := ForwardedRequest{PeerID: c.config.InstanceID, Audience: bootstrap.Audience, Actor: a, OperationID: wire.Request.OperationID, Nonce: hex.EncodeToString(nonce[:]), ExpiresAt: time.Now().Add(30 * time.Second), Body: body} + if len(c.config.SigningKey) != ed25519.PrivateKeySize { + // Only an owner can lack a key, and an owner never forwards. + return nil, ErrForbidden + } + m := ForwardedRequest{PeerID: c.config.InstanceID, Audience: audience, Actor: a, OperationID: wire.Request.OperationID, Nonce: hex.EncodeToString(nonce[:]), ExpiresAt: time.Now().Add(30 * time.Second), Body: body} m.Sign(c.config.SigningKey) b, _ := json.Marshal(m) - req, e = http.NewRequestWithContext(ctx, "POST", c.config.OwnerAddress+"/addons/forward", bytes.NewReader(b)) + req, e := http.NewRequestWithContext(ctx, "POST", c.config.OwnerAddress+"/addons/forward", bytes.NewReader(b)) if e != nil { return nil, e } req.Header.Set("Content-Type", "application/json") - resp, e = c.client.Do(req) + resp, e := c.client.Do(req) if e != nil { if wire.Proposal != nil { return nil, uncertainProposal(*wire.Proposal, wire.Request.OperationID) @@ -504,7 +654,7 @@ var resourceRead = map[string]bool{"resource-list": true, "resource-grants": tru // mutation fails closed: only the known read actions are exempt, so an action // added later is reported as an unknown outcome until it is classified. func mutation(action string) bool { - return action != "read" && action != "operation-status" && action != "proposal-get" && action != "proposal-reconcile" && action != "proposal-list" && action != "artifact-status" && action != appsListAction && !readManagement[action] && !resourceRead[action] + return action != "read" && action != "operation-status" && action != "proposal-get" && action != "proposal-reconcile" && action != "proposal-list" && action != "artifact-status" && action != appsListAction && !readManagement[action] && !resourceRead[action] && !libraryRead[action] } func (c *Carrier) Dispatch(ctx context.Context, a contract.Actor, r contract.Request) (contract.Result, error) { resp, e := c.send(ctx, a, wireRequest{Request: r}) diff --git a/core/services/addons/gateway/carrier_test.go b/core/services/addons/gateway/carrier_test.go index 77978bd26a49..1352f9bd7d52 100644 --- a/core/services/addons/gateway/carrier_test.go +++ b/core/services/addons/gateway/carrier_test.go @@ -64,17 +64,27 @@ var _ = Describe("mTLS carrier", func() { delete(cfg.Pins, "owner") Expect(c.config.Pins).To(HaveKey("owner")) Expect(c.config.Limits).To(Equal(DefaultRelayLimits())) + // The owner never signs, so it needs no key; a key it has must be whole. + owner := valid() + owner.InstanceID, owner.Certificate = "owner", certs["owner"] + owner.SigningKey = nil + _, err = NewCarrier(owner) + Expect(err).NotTo(HaveOccurred()) + owner.SigningKey = key[:8] + _, err = NewCarrier(owner) + Expect(err).To(MatchError(ErrForbidden)) for name, mutate := range map[string]func(*CarrierConfig){ - "owner pin missing": func(c *CarrierConfig) { delete(c.Pins, "owner") }, - "own pin missing": func(c *CarrierConfig) { delete(c.Pins, "front-a") }, - "plain http owner": func(c *CarrierConfig) { c.OwnerAddress = "http://127.0.0.1:9443" }, - "owner path": func(c *CarrierConfig) { c.OwnerAddress = "https://127.0.0.1:9443/x" }, - "owner credentials": func(c *CarrierConfig) { c.OwnerAddress = "https://u:p@127.0.0.1:9443" }, - "no trust roots": func(c *CarrierConfig) { c.Roots = nil }, - "no client certificate": func(c *CarrierConfig) { c.Certificate = tls.Certificate{} }, - "short signing key": func(c *CarrierConfig) { c.SigningKey = key[:8] }, - "empty instance": func(c *CarrierConfig) { c.InstanceID = "" }, - "partial limit override": func(c *CarrierConfig) { c.Limits = RelayLimits{Bytes: 1 << 20} }, + "frontend without a signing key": func(c *CarrierConfig) { c.SigningKey = nil }, + "owner pin missing": func(c *CarrierConfig) { delete(c.Pins, "owner") }, + "own pin missing": func(c *CarrierConfig) { delete(c.Pins, "front-a") }, + "plain http owner": func(c *CarrierConfig) { c.OwnerAddress = "http://127.0.0.1:9443" }, + "owner path": func(c *CarrierConfig) { c.OwnerAddress = "https://127.0.0.1:9443/x" }, + "owner credentials": func(c *CarrierConfig) { c.OwnerAddress = "https://u:p@127.0.0.1:9443" }, + "no trust roots": func(c *CarrierConfig) { c.Roots = nil }, + "no client certificate": func(c *CarrierConfig) { c.Certificate = tls.Certificate{} }, + "short signing key": func(c *CarrierConfig) { c.SigningKey = key[:8] }, + "empty instance": func(c *CarrierConfig) { c.InstanceID = "" }, + "partial limit override": func(c *CarrierConfig) { c.Limits = RelayLimits{Bytes: 1 << 20} }, } { cfg := valid() mutate(&cfg) diff --git a/core/services/addons/gateway/gateway.go b/core/services/addons/gateway/gateway.go index c71f2c344a73..4d76a2a8e411 100644 --- a/core/services/addons/gateway/gateway.go +++ b/core/services/addons/gateway/gateway.go @@ -6,6 +6,7 @@ import ( "context" "encoding/json" "errors" + "github.com/mudler/LocalAI/core/services/addons/modelsprobe" "sync" "github.com/mudler/LocalAI/core/services/addons" @@ -35,7 +36,16 @@ type Gateway struct { // Resources is the owner's UI-only resource management: listing, // creation, grants, rename, deletion and conversation selections. // Frontends leave it nil and forward through Forward. - Resources ResourceManager + Resources ResourceManager + // Libraries is the owner's library administration. Frontends leave it nil + // and forward through Forward. + Libraries LibraryAdmin + // ModelsProbe returns the probe the owner wrote into its models path, or + // false when it has none. Frontends leave it nil. + ModelsProbe func() (modelsprobe.Probe, bool) + // OS and Arch are the platform the owner resolves libraries for. Empty + // means the platform of this process. + OS, Arch string streamsMu sync.Mutex streams int Revalidate func(context.Context, contract.Actor) error diff --git a/core/services/addons/gateway/keys.go b/core/services/addons/gateway/keys.go new file mode 100644 index 000000000000..3c272f9fb373 --- /dev/null +++ b/core/services/addons/gateway/keys.go @@ -0,0 +1,290 @@ +// SPDX-License-Identifier: MIT +package gateway + +import ( + "crypto/ed25519" + "crypto/rand" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "errors" + "fmt" + "io" + "math/big" + "net" + "os" + "path/filepath" + "time" +) + +// Carrier identities of a split deployment. The owner identity is the same +// value a single-node root already stores, so such a root keeps passing the +// owner identity check. Every frontend replica shares one identity: the owner +// keys replay protection by peer and nonce, so it needs no list of replicas. +const ( + OwnerIdentity = "local" + FrontendIdentity = "frontend" +) + +// Names of the files in a carrier key directory. +const ( + carrierCAFile = "ca.pem" + carrierOwnerCert = "owner.pem" + carrierOwnerKey = "owner.key" + carrierFrontendCert = "frontend.pem" + carrierFrontendKey = "frontend.key" + carrierFrontendSign = "frontend-sign.key" + carrierFrontendVerify = "frontend-sign.pub" +) + +var carrierKeyFiles = []string{carrierCAFile, carrierOwnerCert, carrierOwnerKey, carrierFrontendCert, carrierFrontendKey, carrierFrontendSign, carrierFrontendVerify} + +// carrierRoleFiles lists what each role reads. A role never reads another +// role's private keys. +var carrierRoleFiles = map[string][]string{ + "owner": {carrierCAFile, carrierOwnerCert, carrierOwnerKey, carrierFrontendCert, carrierFrontendVerify}, + "frontend": {carrierCAFile, carrierOwnerCert, carrierFrontendCert, carrierFrontendKey, carrierFrontendSign}, +} + +// GenerateCarrierKeys writes a static carrier key set into dir: a CA +// certificate, an owner leaf (server and client auth, with a SAN for each of +// ownerHosts), a frontend leaf (client auth) and the frontend signing key pair. +// The CA key stays in memory and is dropped, so nobody can issue a third leaf +// from this set. It refuses to overwrite any existing file. +func GenerateCarrierKeys(dir string, ownerHosts []string) error { + if len(ownerHosts) == 0 { + return errors.New("at least one owner host is required") + } + for _, h := range ownerHosts { + if h == "" { + return errors.New("owner host must not be empty") + } + } + if err := os.MkdirAll(dir, 0o700); err != nil { + return err + } + for _, name := range carrierKeyFiles { + if _, err := os.Lstat(filepath.Join(dir, name)); err == nil { + return fmt.Errorf("%s already exists in %s; refusing to overwrite a carrier key set", name, dir) + } else if !os.IsNotExist(err) { + return err + } + } + caPub, caKey, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + return err + } + notBefore := time.Now().Add(-time.Hour) + notAfter := time.Now().AddDate(10, 0, 0) + caTemplate := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "addon carrier CA"}, NotBefore: notBefore, NotAfter: notAfter, IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign} + caDER, err := x509.CreateCertificate(rand.Reader, caTemplate, caTemplate, caPub, caKey) + if err != nil { + return err + } + ca, err := x509.ParseCertificate(caDER) + if err != nil { + return err + } + ownerTemplate := &x509.Certificate{SerialNumber: big.NewInt(2), Subject: pkix.Name{CommonName: OwnerIdentity}, NotBefore: notBefore, NotAfter: notAfter, KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}} + for _, h := range ownerHosts { + if ip := net.ParseIP(h); ip != nil { + ownerTemplate.IPAddresses = append(ownerTemplate.IPAddresses, ip) + } else { + ownerTemplate.DNSNames = append(ownerTemplate.DNSNames, h) + } + } + frontendTemplate := &x509.Certificate{SerialNumber: big.NewInt(3), Subject: pkix.Name{CommonName: FrontendIdentity}, NotBefore: notBefore, NotAfter: notAfter, KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}} + ownerCert, ownerKey, err := signedLeaf(ownerTemplate, ca, caKey) + if err != nil { + return err + } + frontendCert, frontendKey, err := signedLeaf(frontendTemplate, ca, caKey) + if err != nil { + return err + } + signPub, signKey, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + return err + } + signKeyDER, err := x509.MarshalPKCS8PrivateKey(signKey) + if err != nil { + return err + } + signPubDER, err := x509.MarshalPKIXPublicKey(signPub) + if err != nil { + return err + } + files := []struct { + name string + data []byte + }{ + {carrierCAFile, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caDER})}, + {carrierOwnerCert, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: ownerCert})}, + {carrierOwnerKey, ownerKey}, + {carrierFrontendCert, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: frontendCert})}, + {carrierFrontendKey, frontendKey}, + {carrierFrontendSign, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: signKeyDER})}, + {carrierFrontendVerify, pem.EncodeToMemory(&pem.Block{Type: "PUBLIC KEY", Bytes: signPubDER})}, + } + for i, f := range files { + if err := writeNewKeyFile(filepath.Join(dir, f.name), f.data); err != nil { + // Do not leave a partial set that a later run would refuse. + for _, done := range files[:i] { + _ = os.Remove(filepath.Join(dir, done.name)) + } + return err + } + } + return nil +} + +func signedLeaf(template *x509.Certificate, ca *x509.Certificate, caKey ed25519.PrivateKey) (certDER, keyPEM []byte, err error) { + pub, key, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + return nil, nil, err + } + certDER, err = x509.CreateCertificate(rand.Reader, template, ca, pub, caKey) + if err != nil { + return nil, nil, err + } + keyDER, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + return nil, nil, err + } + return certDER, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}), nil +} + +func writeNewKeyFile(path string, data []byte) error { + f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return err + } + if _, err := f.Write(data); err != nil { + _ = f.Close() + return err + } + return f.Close() +} + +// LoadCarrierConfig reads the files of one role from a key directory made by +// GenerateCarrierKeys and returns a carrier config. The role is "owner" or +// "frontend". Only that role's files are read, and a file that group or other +// users can access is refused. +func LoadCarrierConfig(dir, role, ownerAddress string) (*CarrierConfig, error) { + names, ok := carrierRoleFiles[role] + if !ok { + return nil, fmt.Errorf("unknown carrier role %q", role) + } + raw := map[string][]byte{} + for _, name := range names { + b, err := readKeyFile(filepath.Join(dir, name)) + if err != nil { + return nil, err + } + raw[name] = b + } + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM(raw[carrierCAFile]) { + return nil, fmt.Errorf("%s holds no certificate", filepath.Join(dir, carrierCAFile)) + } + ownerLeaf, err := leafDER(raw[carrierOwnerCert], carrierOwnerCert) + if err != nil { + return nil, err + } + frontendLeaf, err := leafDER(raw[carrierFrontendCert], carrierFrontendCert) + if err != nil { + return nil, err + } + cfg := &CarrierConfig{ + OwnerID: OwnerIdentity, OwnerAddress: ownerAddress, Roots: roots, + Pins: map[string][32]byte{OwnerIdentity: sha256.Sum256(ownerLeaf), FrontendIdentity: sha256.Sum256(frontendLeaf)}, + } + if role == "owner" { + cfg.InstanceID = OwnerIdentity + cfg.Certificate, err = tls.X509KeyPair(raw[carrierOwnerCert], raw[carrierOwnerKey]) + if err != nil { + return nil, fmt.Errorf("owner certificate and key: %w", err) + } + pub, err := parsePublicKey(raw[carrierFrontendVerify]) + if err != nil { + return nil, fmt.Errorf("%s: %w", carrierFrontendVerify, err) + } + cfg.VerificationKeys = map[string]ed25519.PublicKey{FrontendIdentity: pub} + // The owner never signs an outbound request, so it holds no signing key. + return cfg, nil + } + cfg.InstanceID = FrontendIdentity + cfg.Certificate, err = tls.X509KeyPair(raw[carrierFrontendCert], raw[carrierFrontendKey]) + if err != nil { + return nil, fmt.Errorf("frontend certificate and key: %w", err) + } + key, err := parseSigningKey(raw[carrierFrontendSign]) + if err != nil { + return nil, fmt.Errorf("%s: %w", carrierFrontendSign, err) + } + cfg.SigningKey = key + return cfg, nil +} + +// readKeyFile reads a regular file. Key files must not be accessible to group +// or other users, so a file that someone else could have changed fails closed. +func readKeyFile(path string) ([]byte, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer func() { _ = f.Close() }() + info, err := f.Stat() + if err != nil { + return nil, err + } + if !info.Mode().IsRegular() { + return nil, fmt.Errorf("%s must be a regular file", path) + } + if info.Mode().Perm()&0o077 != 0 { + return nil, fmt.Errorf("%s must have mode 0600 or less", path) + } + return io.ReadAll(io.LimitReader(f, 1<<20)) +} + +func leafDER(certPEM []byte, name string) ([]byte, error) { + block, _ := pem.Decode(certPEM) + if block == nil || block.Type != "CERTIFICATE" { + return nil, fmt.Errorf("%s holds no certificate", name) + } + return block.Bytes, nil +} + +func parsePublicKey(b []byte) (ed25519.PublicKey, error) { + block, _ := pem.Decode(b) + if block == nil { + return nil, errors.New("not a PEM file") + } + k, err := x509.ParsePKIXPublicKey(block.Bytes) + if err != nil { + return nil, err + } + pub, ok := k.(ed25519.PublicKey) + if !ok { + return nil, errors.New("not an ed25519 public key") + } + return pub, nil +} + +func parseSigningKey(b []byte) (ed25519.PrivateKey, error) { + block, _ := pem.Decode(b) + if block == nil { + return nil, errors.New("not a PEM file") + } + k, err := x509.ParsePKCS8PrivateKey(block.Bytes) + if err != nil { + return nil, err + } + key, ok := k.(ed25519.PrivateKey) + if !ok { + return nil, errors.New("not an ed25519 private key") + } + return key, nil +} diff --git a/core/services/addons/gateway/keys_test.go b/core/services/addons/gateway/keys_test.go new file mode 100644 index 000000000000..8713bd097296 --- /dev/null +++ b/core/services/addons/gateway/keys_test.go @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: MIT +package gateway + +import ( + "context" + "crypto/ed25519" + "errors" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + + "github.com/mudler/LocalAI/pkg/addons/contract" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("carrier key set", func() { + var keyActor = contract.Actor{UserID: "user", CredentialID: "key", Mode: "named-user"} + allow := func(context.Context, contract.Actor) error { return nil } + + generate := func() string { + dir := filepath.Join(GinkgoT().TempDir(), "keys") + Expect(GenerateCarrierKeys(dir, []string{"127.0.0.1", "owner.example"})).To(Succeed()) + return dir + } + // startOwner serves an owner carrier loaded from dir on a loopback port and + // returns its address. + startOwner := func(dir string) string { + server := httptest.NewUnstartedServer(nil) + address := "https://" + server.Listener.Addr().String() + cfg, e := LoadCarrierConfig(dir, "owner", address) + Expect(e).NotTo(HaveOccurred()) + owner, e := NewCarrier(*cfg) + Expect(e).NotTo(HaveOccurred()) + handler, e := owner.OwnerHandler(&Gateway{Manager: ownerManager(keyActor, true), Revalidate: allow}) + Expect(e).NotTo(HaveOccurred()) + server.Config.Handler = http.Handler(handler) + server.TLS = owner.ServerTLS() + server.StartTLS() + DeferCleanup(server.Close) + return address + } + frontend := func(dir, address string) *Gateway { + cfg, e := LoadCarrierConfig(dir, "frontend", address) + Expect(e).NotTo(HaveOccurred()) + carrier, e := NewCarrier(*cfg) + Expect(e).NotTo(HaveOccurred()) + return &Gateway{Forward: carrier, Revalidate: allow} + } + + It("writes the set with private modes and no CA key", func() { + dir := generate() + info, e := os.Stat(dir) + Expect(e).NotTo(HaveOccurred()) + Expect(info.Mode().Perm()).To(Equal(os.FileMode(0o700))) + entries, e := os.ReadDir(dir) + Expect(e).NotTo(HaveOccurred()) + var names []string + for _, entry := range entries { + names = append(names, entry.Name()) + fi, e := entry.Info() + Expect(e).NotTo(HaveOccurred()) + Expect(fi.Mode().Perm()).To(Equal(os.FileMode(0o600)), entry.Name()) + } + Expect(names).To(ConsistOf("ca.pem", "owner.pem", "owner.key", "frontend.pem", "frontend.key", "frontend-sign.key", "frontend-sign.pub")) + }) + + It("loads both roles into carriers and serves a frontend call from the owner", func() { + dir := generate() + address := startOwner(dir) + list, e := frontend(dir, address).Apps(context.Background(), keyActor) + Expect(e).NotTo(HaveOccurred()) + Expect(list.Apps).NotTo(BeEmpty()) + }) + + It("gives the owner no signing key and the frontend its own", func() { + dir := generate() + owner, e := LoadCarrierConfig(dir, "owner", "https://owner.example:8443") + Expect(e).NotTo(HaveOccurred()) + Expect(owner.SigningKey).To(BeEmpty()) + Expect(owner.VerificationKeys).To(HaveKey(FrontendIdentity)) + front, e := LoadCarrierConfig(dir, "frontend", "https://owner.example:8443") + Expect(e).NotTo(HaveOccurred()) + Expect(front.SigningKey).To(HaveLen(ed25519.PrivateKeySize)) + }) + + It("accepts two frontends made from the same set", func() { + dir := generate() + address := startOwner(dir) + for range 2 { + _, e := frontend(dir, address).Apps(context.Background(), keyActor) + Expect(e).NotTo(HaveOccurred()) + } + }) + + It("refuses a frontend from another generation", func() { + address := startOwner(generate()) + _, e := frontend(generate(), address).Apps(context.Background(), keyActor) + Expect(e).To(HaveOccurred()) + // The carrier reports a refused TLS handshake as an unavailable owner, + // like every other untrusted peer (see "certificate pins"). + Expect(errors.Is(e, ErrUnavailable)).To(BeTrue(), "got %v", e) + }) + + It("reads only the files of the role", func() { + dir := generate() + Expect(os.Remove(filepath.Join(dir, "frontend-sign.key"))).To(Succeed()) + Expect(os.Remove(filepath.Join(dir, "frontend.key"))).To(Succeed()) + _, e := LoadCarrierConfig(dir, "owner", "https://owner.example:8443") + Expect(e).NotTo(HaveOccurred()) + Expect(os.Remove(filepath.Join(dir, "owner.key"))).To(Succeed()) + Expect(os.Remove(filepath.Join(dir, "frontend-sign.pub"))).To(Succeed()) + _, e = LoadCarrierConfig(dir, "frontend", "https://owner.example:8443") + Expect(e).To(HaveOccurred()) + }) + + It("refuses a key file with group or other access", func() { + dir := generate() + Expect(os.Chmod(filepath.Join(dir, "owner.key"), 0o644)).To(Succeed()) + _, e := LoadCarrierConfig(dir, "owner", "https://owner.example:8443") + Expect(e).To(MatchError(ContainSubstring("owner.key"))) + }) + + It("refuses an unknown role", func() { + _, e := LoadCarrierConfig(generate(), "worker", "https://owner.example:8443") + Expect(e).To(MatchError(ContainSubstring("worker"))) + }) + + It("refuses to overwrite a file and leaves it unchanged", func() { + dir := generate() + before, e := os.ReadFile(filepath.Join(dir, "ca.pem")) + Expect(e).NotTo(HaveOccurred()) + Expect(GenerateCarrierKeys(dir, []string{"owner.example"})).To(MatchError(ContainSubstring("already exists"))) + after, e := os.ReadFile(filepath.Join(dir, "ca.pem")) + Expect(e).NotTo(HaveOccurred()) + Expect(after).To(Equal(before)) + + partial := filepath.Join(GinkgoT().TempDir(), "partial") + Expect(os.Mkdir(partial, 0o700)).To(Succeed()) + Expect(os.WriteFile(filepath.Join(partial, "owner.key"), []byte("x"), 0o600)).To(Succeed()) + Expect(GenerateCarrierKeys(partial, []string{"owner.example"})).To(HaveOccurred()) + entries, e := os.ReadDir(partial) + Expect(e).NotTo(HaveOccurred()) + Expect(entries).To(HaveLen(1)) + }) + + It("needs an owner host", func() { + Expect(GenerateCarrierKeys(filepath.Join(GinkgoT().TempDir(), "keys"), nil)).To(HaveOccurred()) + }) +}) diff --git a/core/services/addons/gateway/library.go b/core/services/addons/gateway/library.go new file mode 100644 index 000000000000..5c0cea02d298 --- /dev/null +++ b/core/services/addons/gateway/library.go @@ -0,0 +1,375 @@ +// SPDX-License-Identifier: MIT +package gateway + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/url" + "runtime" + "sync" + + "github.com/mudler/LocalAI/pkg/addons/contract" +) + +// The operations of a library call. Each one is a carrier verb with the action +// "library-". The first four sources verbs and the two backend verbs are +// reads; every other verb is a mutation, so a reply lost on the way is an +// unknown outcome with its operation ID. +const ( + LibraryList = "list" + LibraryAdd = "add" + LibraryUpdate = "update" + LibraryEnable = "enable" + LibraryDisable = "disable" + LibraryRemove = "remove" + LibraryBackendResolve = "backend-resolve" + LibraryBackendReleases = "backend-releases" +) + +// CallLibrarySystem is the capability of an owner that serves the system call +// of the library: the background check for newer backend builds. +const CallLibrarySystem = "library-system" + +// maxLibraryRefs bounds the backends of one backend-releases call. +const maxLibraryRefs = 256 + +// LibraryRef names one library backend in a backend-releases call. +type LibraryRef struct { + SourceID string `json:"sourceId"` + Name string `json:"name"` +} + +// LibraryCall asks the owner for a library operation: a change of the library +// sources, or the owner's half of a backend install. Only the fields of the +// operation are set. +type LibraryCall struct { + Op string `json:"op"` + // ID names the source of update, enable, disable and remove. + ID string `json:"id,omitempty"` + // Kind, URL, Name, Format and Mirrors describe the source of add; update + // takes Mirrors. + Kind string `json:"kind,omitempty"` + URL string `json:"url,omitempty"` + Name string `json:"name,omitempty"` + Format string `json:"format,omitempty"` + Mirrors []string `json:"mirrors,omitempty"` + // Known is the list version the caller holds (list only). + Known string `json:"known,omitempty"` + // SourceID, Backend and Version name the entry of backend-resolve. + // Capability is the capability the installing host reports, which picks the + // backend of a meta backend. + SourceID string `json:"sourceId,omitempty"` + Backend string `json:"backend,omitempty"` + Version string `json:"version,omitempty"` + Capability string `json:"capability,omitempty"` + // Refs are the backends of backend-releases, at most 256. + Refs []LibraryRef `json:"refs,omitempty"` + // System marks a call of the frontend itself, with no user behind it: the + // background check for newer backend builds. The owner accepts it only on + // the peer-authenticated system route and only for backend-releases, and it + // runs no administrator check because it reads and changes nothing a user + // owns. A call from /addons/forward never carries it. + System bool `json:"system,omitempty"` +} + +func (c LibraryCall) wireAction() string { return "library-" + c.Op } + +func (c LibraryCall) valid() bool { + if c.System && c.Op != LibraryBackendReleases { + return false + } + switch c.Op { + case LibraryList, LibraryAdd, LibraryUpdate, LibraryEnable, LibraryDisable, LibraryRemove, LibraryBackendResolve: + return len(c.Refs) == 0 + case LibraryBackendReleases: + return len(c.Refs) <= maxLibraryRefs + } + return false +} + +// libraryRead are the library actions that change nothing. +var libraryRead = map[string]bool{"library-" + LibraryList: true, "library-" + LibraryBackendResolve: true, "library-" + LibraryBackendReleases: true} + +// The error types of a LibraryError. The frontend maps each to the sentinel the +// owner's own routes would have seen, so both answer with the same status. +const ( + LibraryNotFound = "not_found" + LibraryExists = "exists" + LibraryDuplicate = "duplicate" + LibraryInvalid = "invalid" + LibraryBadRequest = "bad_request" + LibraryInternal = "internal" +) + +// LibraryError is a refusal the owner decided. It travels in the reply envelope +// with status 200, because typedError would map several of these to other +// statuses than the routes answer. +type LibraryError struct { + Type string `json:"type"` + Message string `json:"message"` +} + +func (e *LibraryError) Error() string { return e.Message } + +// libraryReply is the reply of a library call: data or error. +type libraryReply struct { + Data json.RawMessage `json:"data,omitempty"` + Error *LibraryError `json:"error,omitempty"` +} + +// LibraryAdmin is the owner's side of a library call. core/library imports this +// package through core/config, so the application implements it. Library runs +// after the verifier checked the signature, the audience and the actor, and it +// must check the administrator right of the actor again. Snapshot is the +// peer-authenticated read of the gallery-relevant source list and has no actor. +type LibraryAdmin interface { + Library(ctx context.Context, actor contract.Actor, call LibraryCall) (json.RawMessage, error) + Snapshot(ctx context.Context, known string) (json.RawMessage, error) +} + +// UnavailableError is an unavailable answer that says why. It matches +// ErrUnavailable. +type UnavailableError struct{ Reason string } + +func (e *UnavailableError) Error() string { return e.Reason } +func (e *UnavailableError) Unwrap() error { return ErrUnavailable } + +// UnavailableReason returns the message for the 503 of an unavailable error: +// its own reason when it has one, else the unreachable owner. +func UnavailableReason(e error) string { + var u *UnavailableError + if errors.As(e, &u) { + return u.Reason + } + var unknown *OutcomeUnknown + if errors.As(e, &unknown) { + return "the add-on owner did not confirm the change (operation " + unknown.OperationID + "); read the list again" + } + return ReasonOwnerUnreached +} + +// Reasons of the library calls that the owner cannot serve. +const ( + ReasonLibraryUnsupported = "the add-on owner does not support library operations; upgrade the owner" + ReasonPlatformMismatch = "this instance and the add-on owner run on different operating systems or architectures" +) + +// Capabilities is the answer of GET /addons/capabilities, a separate call from +// the audience, whose reply an older frontend decodes strictly. Protocol 1 is an +// owner that answers 404. +type Capabilities struct { + Protocol int `json:"protocol"` + Calls []string `json:"calls"` + // OS and Arch are the platform the owner resolves libraries for. + OS string `json:"os,omitempty"` + Arch string `json:"arch,omitempty"` +} + +func (c Capabilities) Supports(call string) bool { + for _, s := range c.Calls { + if s == call { + return true + } + } + return false +} + +func (g *Gateway) capabilities() Capabilities { + c := Capabilities{Protocol: 2, OS: g.OS, Arch: g.Arch} + if c.OS == "" { + c.OS = runtime.GOOS + } + if c.Arch == "" { + c.Arch = runtime.GOARCH + } + if g.Libraries != nil { + c.Calls = append(c.Calls, "library") + } + if g.Libraries != nil { + c.Calls = append(c.Calls, CallLibrarySystem) + } + if g.ModelsProbe != nil { + c.Calls = append(c.Calls, "models-probe") + } + return c +} + +// librarySystemLocal runs the system call on the owner. It has no actor. The +// call is read-only by construction: valid() lets System through for +// backend-releases only. +func (g *Gateway) librarySystemLocal(ctx context.Context, w http.ResponseWriter, call LibraryCall) { + if !call.System || call.Op != LibraryBackendReleases || !call.valid() { + http.Error(w, "invalid binding", 400) + return + } + g.libraryLocal(ctx, w, contract.Actor{}, call) +} + +// libraryLocal runs a library call on the owner and writes the reply envelope. +func (g *Gateway) libraryLocal(ctx context.Context, w http.ResponseWriter, actor contract.Actor, call LibraryCall) { + if g.Libraries == nil { + typedError(w, ErrUnavailable) + return + } + raw, e := g.Libraries.Library(ctx, actor, call) + var le *LibraryError + switch { + case e == nil: + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(libraryReply{Data: raw}) + case errors.As(e, &le): + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(libraryReply{Error: le}) + default: + typedError(w, e) + } +} + +// capsCache keeps the owner's capabilities for one owner audience. +type capsCache struct { + mu sync.Mutex + audience string + caps Capabilities +} + +// Capabilities asks the owner what it serves. The answer is kept for the +// owner's current audience, so it is asked again after the owner restarts. An +// owner that answers 404 is protocol 1. +func (c *Carrier) Capabilities(ctx context.Context) (Capabilities, error) { + audience, e := c.audience(ctx) + if e != nil { + return Capabilities{}, e + } + c.caps.mu.Lock() + defer c.caps.mu.Unlock() + if c.caps.audience == audience { + return c.caps.caps, nil + } + req, e := http.NewRequestWithContext(ctx, "GET", c.config.OwnerAddress+"/addons/capabilities", nil) + if e != nil { + return Capabilities{}, ErrUnavailable + } + resp, e := c.client.Do(req) + if e != nil { + return Capabilities{}, ErrUnavailable + } + defer func() { _ = resp.Body.Close() }() + var caps Capabilities + switch { + case resp.StatusCode == http.StatusNotFound: + caps = Capabilities{Protocol: 1} + case resp.StatusCode != http.StatusOK: + return Capabilities{}, ErrUnavailable + default: + if decode(io.LimitReader(resp.Body, 4096), &caps) != nil || caps.Protocol < 1 { + return Capabilities{}, ErrUnavailable + } + } + c.caps.audience, c.caps.caps = audience, caps + return caps, nil +} + +// Library forwards one library call to the owner and returns the data of its +// reply. A refusal the owner decided is a *LibraryError. The caller checks +// Capabilities first. +func (c *Carrier) Library(ctx context.Context, a contract.Actor, call LibraryCall) (json.RawMessage, error) { + if !call.valid() { + return nil, ErrForbidden + } + id, e := newResourceOperationID() + if e != nil { + return nil, ErrUnavailable + } + resp, e := c.send(ctx, a, wireRequest{Request: contract.Request{OperationID: id, Action: call.wireAction()}, Library: &call}) + if e != nil { + return nil, e + } + defer func() { _ = resp.Body.Close() }() + raw, e := io.ReadAll(io.LimitReader(resp.Body, 2<<20+1)) + var reply libraryReply + if e != nil || len(raw) > 2<<20 || decode(bytes.NewReader(raw), &reply) != nil { + if mutation(call.wireAction()) { + return nil, &OutcomeUnknown{OperationID: id} + } + return nil, ErrUnavailable + } + if reply.Error != nil { + return nil, reply.Error + } + return reply.Data, nil +} + +// LibrarySnapshot asks the owner for the source list that decides the +// galleries, as the owner's peer-authenticated call: it carries no actor, so a +// frontend can poll it without a signed-in user. known is the version the +// caller holds; an unchanged list is answered without sources. +func (c *Carrier) LibrarySnapshot(ctx context.Context, known string) (json.RawMessage, error) { + if _, e := c.audience(ctx); e != nil { + return nil, e + } + u := c.config.OwnerAddress + "/addons/library/snapshot" + if known != "" { + u += "?known=" + url.QueryEscape(known) + } + req, e := http.NewRequestWithContext(ctx, "GET", u, nil) + if e != nil { + return nil, ErrUnavailable + } + resp, e := c.client.Do(req) + if e != nil { + return nil, ErrUnavailable + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return nil, ErrUnavailable + } + raw, e := io.ReadAll(io.LimitReader(resp.Body, 2<<20+1)) + if e != nil || len(raw) > 2<<20 { + return nil, ErrUnavailable + } + return raw, nil +} + +// LibrarySystem sends a system call to the owner on its peer-authenticated +// route. It carries no actor: the owner trusts the pinned carrier identity of +// this frontend, and accepts only a read-only verb. The caller checks +// Capabilities first. +func (c *Carrier) LibrarySystem(ctx context.Context, call LibraryCall) (json.RawMessage, error) { + call.System = true + if !call.valid() { + return nil, ErrForbidden + } + if _, e := c.audience(ctx); e != nil { + return nil, e + } + body, e := json.Marshal(call) + if e != nil { + return nil, ErrUnavailable + } + req, e := http.NewRequestWithContext(ctx, "POST", c.config.OwnerAddress+"/addons/library/system", bytes.NewReader(body)) + if e != nil { + return nil, ErrUnavailable + } + req.Header.Set("Content-Type", "application/json") + resp, e := c.client.Do(req) + if e != nil { + return nil, ErrUnavailable + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return nil, ErrUnavailable + } + raw, e := io.ReadAll(io.LimitReader(resp.Body, 2<<20+1)) + var reply libraryReply + if e != nil || len(raw) > 2<<20 || decode(bytes.NewReader(raw), &reply) != nil { + return nil, ErrUnavailable + } + if reply.Error != nil { + return nil, reply.Error + } + return reply.Data, nil +} diff --git a/core/services/addons/gateway/library_system_test.go b/core/services/addons/gateway/library_system_test.go new file mode 100644 index 000000000000..ede78a56433e --- /dev/null +++ b/core/services/addons/gateway/library_system_test.go @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: MIT +package gateway + +import ( + "bytes" + "context" + "crypto/tls" + "encoding/json" + "net/http" + + "github.com/mudler/LocalAI/pkg/addons/contract" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// recordingLibraries stands in for the owner's library. It records the actor +// and the call of each library verb. +type recordingLibraries struct { + actors []contract.Actor + calls []LibraryCall +} + +func (r *recordingLibraries) Library(_ context.Context, a contract.Actor, c LibraryCall) (json.RawMessage, error) { + r.actors = append(r.actors, a) + r.calls = append(r.calls, c) + return json.RawMessage(`{"official/whisper":[]}`), nil +} +func (r *recordingLibraries) Snapshot(context.Context, string) (json.RawMessage, error) { + return json.RawMessage(`{}`), nil +} + +var _ = Describe("the library system call", func() { + releases := LibraryCall{Op: LibraryBackendReleases, Refs: []LibraryRef{{SourceID: "official", Name: "whisper"}}} + + It("runs the backend-releases check for a pinned frontend without an actor", func() { + libs := &recordingLibraries{} + _, cs := carrierFixture(func(c *Carrier) http.Handler { + h, e := c.OwnerHandler(&Gateway{Libraries: libs, Revalidate: func(context.Context, contract.Actor) error { return nil }}) + Expect(e).NotTo(HaveOccurred()) + return h + }) + raw, e := cs["front-a"].LibrarySystem(context.Background(), releases) + Expect(e).NotTo(HaveOccurred()) + Expect(string(raw)).To(ContainSubstring("official/whisper")) + Expect(libs.calls).To(HaveLen(1)) + Expect(libs.calls[0].System).To(BeTrue()) + Expect(libs.actors[0]).To(Equal(contract.Actor{})) + }) + + It("refuses a system call from a peer that is not pinned", func() { + libs := &recordingLibraries{} + env := carrierFixtureWith(RelayLimits{}, func(c *Carrier) http.Handler { + h, e := c.OwnerHandler(&Gateway{Libraries: libs, Revalidate: func(context.Context, contract.Actor) error { return nil }}) + Expect(e).NotTo(HaveOccurred()) + return h + }) + stranger := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{ + Certificates: []tls.Certificate{env.certs["stranger"]}, RootCAs: env.roots, MinVersion: tls.VersionTLS13, + }}} + DeferCleanup(stranger.CloseIdleConnections) + body, _ := json.Marshal(LibraryCall{Op: LibraryBackendReleases, System: true}) + resp, e := stranger.Post(env.server.URL+"/addons/library/system", "application/json", bytes.NewReader(body)) + Expect(e).NotTo(HaveOccurred()) + defer func() { _ = resp.Body.Close() }() + Expect(resp.StatusCode).To(Equal(http.StatusForbidden)) + Expect(libs.calls).To(BeEmpty()) + }) + + It("never lets a system call change anything or ride the user-facing route", func() { + libs := &recordingLibraries{} + _, cs := carrierFixture(func(c *Carrier) http.Handler { + h, e := c.OwnerHandler(&Gateway{Libraries: libs, Revalidate: func(context.Context, contract.Actor) error { return nil }}) + Expect(e).NotTo(HaveOccurred()) + return h + }) + _, e := cs["front-a"].LibrarySystem(context.Background(), LibraryCall{Op: LibraryRemove, ID: "x"}) + Expect(e).To(MatchError(ErrForbidden)) + Expect(libs.calls).To(BeEmpty()) + Expect(LibraryCall{Op: LibraryBackendReleases, System: true}.valid()).To(BeTrue()) + Expect(LibraryCall{Op: LibraryAdd, System: true}.valid()).To(BeFalse()) + }) +}) diff --git a/core/services/addons/hostops/broker.go b/core/services/addons/hostops/broker.go index 39d61469fdfc..07d15f663035 100644 --- a/core/services/addons/hostops/broker.go +++ b/core/services/addons/hostops/broker.go @@ -7,6 +7,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/mudler/LocalAI/core/services/addons/modelsprobe" "io" "net/http" "strings" @@ -203,6 +204,8 @@ func (b *Broker) Handler(addonID, credential string, allowed []string) http.Hand status, code, msg = http.StatusNotFound, contract.HostNotFound, "not found" case errors.Is(err, ErrConflict): status, code, msg = http.StatusConflict, contract.HostConflict, "conflict" + case errors.Is(err, modelsprobe.ErrNotShared): + status, code, msg = http.StatusConflict, contract.HostConflict, modelsprobe.ErrNotShared.Error() case errors.Is(err, ErrBadRequest): // The message names the refused field. Handlers build it from // fixed text and field names, never from a server path. diff --git a/core/services/addons/manager_test.go b/core/services/addons/manager_test.go index f6f186ae68e7..98342c58a732 100644 --- a/core/services/addons/manager_test.go +++ b/core/services/addons/manager_test.go @@ -245,6 +245,20 @@ var _ = Describe("designated owner lifecycle", func() { _, e = NewManager(ctx, cfg, c, i, admin) Expect(e).To(HaveOccurred()) }) + It("needs a state root for the owner only", func() { + cfg, c, i := fixture() + cfg.InstanceID = "frontend" + cfg.Root = "" + m := open(cfg, c, i) + _, e := m.Status(ctx, "sample") + Expect(e).To(MatchError(ErrNotOwner)) + cfg.InstanceID = cfg.OwnerID + _, e = NewManager(ctx, cfg, c, i, admin) + Expect(e).To(MatchError(ErrConfiguration)) + cfg.Root = "relative/root" + _, e = NewManager(ctx, cfg, c, i, admin) + Expect(e).To(MatchError(ErrConfiguration)) + }) It("refuses commands and asset reads for an add-on ID the host serves as a route", func() { cfg, c, i := fixture() m := open(cfg, c, i) @@ -399,3 +413,48 @@ var _ = Describe("cancellable operation observation", func() { Eventually(done).Should(Receive(Equal(context.Canceled))) }) }) + +var _ = Describe("owner identity file", func() { + acquire := func(cfg OwnerConfig) error { + release, e := (OwnerGuard{Config: cfg}).Acquire(ctx) + if e == nil { + Expect(release()).To(Succeed()) + } + return e + } + identityPath := func(cfg OwnerConfig) string { return filepath.Join(cfg.Root, "identity.json") } + + It("writes the owner and storage only, with no address", func() { + cfg, _, _ := fixture() + Expect(acquire(cfg)).To(Succeed()) + b, e := os.ReadFile(identityPath(cfg)) + Expect(e).NotTo(HaveOccurred()) + var fields map[string]any + Expect(json.Unmarshal(b, &fields)).To(Succeed()) + Expect(fields).To(Equal(map[string]any{"OwnerID": "owner", "StorageID": "disk-one"})) + }) + It("loads a file written in the old format with an address", func() { + cfg, _, _ := fixture() + Expect(os.MkdirAll(cfg.Root, 0700)).To(Succeed()) + Expect(os.WriteFile(identityPath(cfg), []byte(`{"OwnerID":"owner","Address":"https://old.example","StorageID":"disk-one"}`), 0600)).To(Succeed()) + Expect(acquire(cfg)).To(Succeed()) + }) + It("loads when only the address changed", func() { + cfg, _, _ := fixture() + Expect(acquire(cfg)).To(Succeed()) + cfg.OwnerAddress = "https://moved.example:8443" + Expect(acquire(cfg)).To(Succeed()) + }) + It("refuses a changed owner ID", func() { + cfg, _, _ := fixture() + Expect(acquire(cfg)).To(Succeed()) + cfg.OwnerID, cfg.InstanceID = "owner-two", "owner-two" + Expect(acquire(cfg)).To(MatchError(ErrConfiguration)) + }) + It("refuses a changed storage ID", func() { + cfg, _, _ := fixture() + Expect(acquire(cfg)).To(Succeed()) + cfg.StorageID = "disk-two" + Expect(acquire(cfg)).To(MatchError(ErrConfiguration)) + }) +}) diff --git a/core/services/addons/modelsprobe/modelsprobe.go b/core/services/addons/modelsprobe/modelsprobe.go new file mode 100644 index 000000000000..c9b20a643ffb --- /dev/null +++ b/core/services/addons/modelsprobe/modelsprobe.go @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: MIT + +// Package modelsprobe checks that the add-on owner and a frontend see the same +// models storage. The owner writes a small file with a random nonce into its +// models path; a frontend reads the file from its own models path and compares +// the nonce. It depends on nothing else in the add-on host, so the gateway and +// the application share it. +package modelsprobe + +import ( + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "time" +) + +// FileName starts with a dot and ends in .json so the model loader and the +// config loader skip it. +const FileName = ".localai-addons-probe.json" + +// maxSize bounds a read: a probe is a few hundred bytes. +const maxSize = 4096 + +// ErrNotShared is the typed refusal of an export or import when the probe of +// the owner's own models path is missing or changed. +var ErrNotShared = errors.New("the models path is not the shared models storage; see the add-ons docs") + +// Probe is the content of the probe file. +type Probe struct { + Nonce string `json:"nonce"` + ModelsPath string `json:"modelsPath"` + WrittenAt string `json:"writtenAt"` +} + +// Valid reports whether the probe carries a nonce. +func (p Probe) Valid() bool { return len(p.Nonce) == 64 && p.ModelsPath != "" } + +// Same reports whether two probes name the same nonce and the same absolute +// path. A symlink that points at the same storage differs on the path rule. +func Same(a, b Probe) bool { + return a.Valid() && b.Valid() && a.Nonce == b.Nonce && a.ModelsPath == b.ModelsPath +} + +// Write puts a new probe in dir. The file is written to a temporary name, +// synced and renamed, so a reader never sees a partial file. +func Write(dir string) (Probe, error) { + abs, err := filepath.Abs(dir) + if err != nil { + return Probe{}, err + } + raw := make([]byte, 32) + if _, err = rand.Read(raw); err != nil { + return Probe{}, err + } + p := Probe{Nonce: hex.EncodeToString(raw), ModelsPath: abs, WrittenAt: time.Now().UTC().Format(time.RFC3339)} + data, err := json.Marshal(p) + if err != nil { + return Probe{}, err + } + tmp, err := os.CreateTemp(abs, ".localai-addons-probe-*.tmp") + if err != nil { + return Probe{}, err + } + name := tmp.Name() + defer func() { _ = os.Remove(name) }() + if _, err = tmp.Write(data); err == nil { + err = tmp.Sync() + } + if cerr := tmp.Close(); err == nil { + err = cerr + } + if err != nil { + return Probe{}, err + } + if err = os.Chmod(name, 0o644); err != nil { + return Probe{}, err + } + if err = os.Rename(name, filepath.Join(abs, FileName)); err != nil { + return Probe{}, err + } + if d, derr := os.Open(abs); derr == nil { + _ = d.Sync() + _ = d.Close() + } + return p, nil +} + +// Read returns the probe in dir. A missing, oversized or malformed file is an +// error. +func Read(dir string) (Probe, error) { + f, err := os.Open(filepath.Join(dir, FileName)) + if err != nil { + return Probe{}, err + } + defer func() { _ = f.Close() }() + data, err := io.ReadAll(io.LimitReader(f, maxSize+1)) + if err != nil { + return Probe{}, err + } + if len(data) > maxSize { + return Probe{}, fmt.Errorf("probe file is too large") + } + var p Probe + if err = json.Unmarshal(data, &p); err != nil { + return Probe{}, err + } + if !p.Valid() { + return Probe{}, fmt.Errorf("probe file is not valid") + } + return p, nil +} diff --git a/core/services/addons/modelsprobe/modelsprobe_suite_test.go b/core/services/addons/modelsprobe/modelsprobe_suite_test.go new file mode 100644 index 000000000000..f21ade91ba12 --- /dev/null +++ b/core/services/addons/modelsprobe/modelsprobe_suite_test.go @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: MIT +package modelsprobe_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/mudler/LocalAI/core/services/addons/modelsprobe" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestModelsProbe(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Models probe") +} + +var _ = Describe("models probe", func() { + It("reads back what it wrote with the same nonce", func() { + dir := GinkgoT().TempDir() + w, err := modelsprobe.Write(dir) + Expect(err).NotTo(HaveOccurred()) + r, err := modelsprobe.Read(dir) + Expect(err).NotTo(HaveOccurred()) + Expect(modelsprobe.Same(w, r)).To(BeTrue()) + Expect(r.Nonce).To(HaveLen(64)) + }) + + It("uses a new nonce at every write", func() { + dir := GinkgoT().TempDir() + a, _ := modelsprobe.Write(dir) + b, _ := modelsprobe.Write(dir) + Expect(modelsprobe.Same(a, b)).To(BeFalse()) + }) + + It("reports a missing file in another directory", func() { + _, err := modelsprobe.Write(GinkgoT().TempDir()) + Expect(err).NotTo(HaveOccurred()) + _, err = modelsprobe.Read(GinkgoT().TempDir()) + Expect(err).To(HaveOccurred()) + }) + + It("differs when only the path differs", func() { + dir := GinkgoT().TempDir() + w, _ := modelsprobe.Write(dir) + other := w + other.ModelsPath = filepath.Join(dir, "elsewhere") + Expect(modelsprobe.Same(w, other)).To(BeFalse()) + }) + + It("refuses a partial or malformed file", func() { + dir := GinkgoT().TempDir() + Expect(os.WriteFile(filepath.Join(dir, modelsprobe.FileName), []byte(`{"nonce":"ab`), 0o644)).To(Succeed()) + _, err := modelsprobe.Read(dir) + Expect(err).To(HaveOccurred()) + }) + + It("leaves no temporary file behind", func() { + dir := GinkgoT().TempDir() + _, err := modelsprobe.Write(dir) + Expect(err).NotTo(HaveOccurred()) + entries, _ := os.ReadDir(dir) + Expect(entries).To(HaveLen(1)) + }) +}) diff --git a/core/services/addons/ownership.go b/core/services/addons/ownership.go index f2e491e26de8..3b1aad174e22 100644 --- a/core/services/addons/ownership.go +++ b/core/services/addons/ownership.go @@ -21,12 +21,20 @@ type OwnerConfig struct { func (c OwnerConfig) validate() error { u, e := url.Parse(c.OwnerAddress) - if e != nil || u.Scheme != "https" || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || c.InstanceID == "" || c.OwnerID == "" || c.StorageID == "" || !filepath.IsAbs(c.Root) || c.SharedStorage { + if e != nil || u.Scheme != "https" || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || c.InstanceID == "" || c.OwnerID == "" || c.StorageID == "" || c.SharedStorage { + return ErrConfiguration + } + // Only the owner has a state root. A frontend keeps no add-on state, so it + // has no root to name and nothing may read or create one. + if c.InstanceID == c.OwnerID && !filepath.IsAbs(c.Root) { return ErrConfiguration } return nil } +// ownerIdentity is what the identity file stores. +type ownerIdentity struct{ OwnerID, StorageID string } + type OwnerGuard struct{ Config OwnerConfig } func (g OwnerGuard) Acquire(ctx context.Context) (func() error, error) { @@ -64,14 +72,20 @@ func (g OwnerGuard) Acquire(ctx context.Context) (func() error, error) { once.Do(func() { releaseErr = errors.Join(syscall.Flock(int(f.Fd()), syscall.LOCK_UN), f.Close()) }) return releaseErr } - identity := struct{ OwnerID, Address, StorageID string }{c.OwnerID, c.OwnerAddress, c.StorageID} + // The address is runtime configuration and a frontend may reach the owner at + // a new one, so identity is the owner and the storage only. An older file + // that still carries an address is read and the address is ignored. + identity := ownerIdentity{OwnerID: c.OwnerID, StorageID: c.StorageID} expected, _ := json.Marshal(identity) p := filepath.Join(c.Root, "identity.json") b, e := os.ReadFile(p) // #nosec G304 -- path is under the add-on state root and is built from validated identifiers, never from a request if os.IsNotExist(e) { e = durableWrite(p, expected) - } else if e == nil && string(b) != string(expected) { - e = ErrConfiguration + } else if e == nil { + var stored ownerIdentity + if json.Unmarshal(b, &stored) != nil || stored != identity { + e = ErrConfiguration + } } if e != nil { _ = release() diff --git a/docs/content/features/addons.md b/docs/content/features/addons.md index 4c4983d54597..31d654e86bb5 100644 --- a/docs/content/features/addons.md +++ b/docs/content/features/addons.md @@ -64,6 +64,17 @@ store and forward the typed operation to the owner. The consequences are by desi A deployment with one instance is the owner and has no frontend; everything above still holds. +The `--addons-role` flag chooses how a process takes part. `combined` is the +default and is both the owner and the only frontend. `owner` and `frontend` split +them, as described in [Running the owner as its own workload](#running-the-owner-as-its-own-workload). + +**Distributed state.** In [distributed mode]({{% relref "features/distributed-mode" %}}), +the owner is a single runner by design: the operator runs one owner replica, and the +lock file in its root is the guard against a mistake, not a way to scale. The +frontends are stateless for add-ons. The one exception is the add-on pins of a +frontend that has no account database, which are per-instance state (see +[Frontend limits](#frontend-limits)). + ## Turning the owner on or off A normal `local-ai run` starts the addon owner, so Apps, Add-ons and Tables work @@ -73,12 +84,16 @@ loopback carrier with keys that are made at each start and never written to disk | Flag | Environment variable | Default | What it does | |---|---|---|---| -| `--addons-enabled` | `LOCALAI_ADDONS_ENABLED` | `true` | Runs the addon owner. `--addons-enabled=false` turns it off completely: no owner starts, no addon runs, and every addon request is refused as unavailable. | +| `--addons-enabled` | `LOCALAI_ADDONS_ENABLED` | `true` | Runs the addon owner. `--addons-enabled=false` turns it off completely: no owner starts, no addon runs, the UI hides the add-on entries, and every add-on, fine-tuning and quantization route answers `503`. See [The addons feature gate](#the-addons-feature-gate). | | `--addons-ownership-mode` | `LOCALAI_ADDONS_OWNERSHIP_MODE` | `local-owner` when authentication is off, `named-user` when it is on | `named-user` gives each account its own access. `local-owner` makes every caller share one owner. A value you set always wins over the default. Any other value stops startup with an error. | | `--addons-official-library` | `LOCALAI_ADDONS_OFFICIAL_LIBRARY` | `true` | Registers the [official library](#the-official-library) on the first start. `--addons-official-library=false` does not register it. | | `--addons-backend-library` | `LOCALAI_ADDONS_BACKEND_LIBRARY` | `true` | Registers the [backend library](#the-backend-library) on the first start, like the official library. A build from master reads it as the backend source; a tagged build reads the backend gallery until the release cutover. `--addons-backend-library=false` starts without it and reads the backend gallery on every build. A library you remove stays removed. | | `--official-libraries` | `LOCALAI_OFFICIAL_LIBRARIES` | `true` | Use the default library sources of add-ons, backends and models. `--official-libraries=false` leaves them listed but unread. It is also the runtime setting `official_libraries`. See [Library sources](#library-sources). | | `--addons-backend-max-bytes` | `LOCALAI_ADDONS_BACKEND_MAX_BYTES` | `8589934592` (8 GiB) | Largest compressed size of a backend image installed from a library. Add-ons keep their own limit of 512 MiB. | +| `--addons-role` | `LOCALAI_ADDONS_ROLE` | `combined` | `combined` is the single node described above. `owner` runs only the owner and serves the carrier on the network. `frontend` runs no owner and forwards every add-on request to it. Any other value stops startup with an error. | +| `--addons-owner-address` | `LOCALAI_ADDONS_OWNER_ADDRESS` | empty | `https://host:port` of the owner carrier, with no path. Use a stable service name, never a pod address. Required for `owner` and `frontend`; an error with `combined`. | +| `--addons-owner-listen` | `LOCALAI_ADDONS_OWNER_LISTEN` | empty | `owner` only. The address the carrier listens on. Empty listens on all interfaces, on the port of `--addons-owner-address`. | +| `--addons-carrier-dir` | `LOCALAI_ADDONS_CARRIER_DIR` | empty | Directory with the carrier key set made by `local-ai util addons-carrier-keys`. Required for `owner` and `frontend`; an error with `combined`. | `named-user` needs authentication. With authentication on and no mode set, LocalAI uses `named-user`. With authentication off there are no accounts, so LocalAI uses @@ -89,6 +104,276 @@ add-ons off on a server without authentication, set `--addons-enabled=false`, or set `--addons-ownership-mode=named-user`; with authentication off, `named-user` leaves the owner off and LocalAI logs a warning. +The `owner` and `frontend` roles split the owner from the frontends. The `frontend` +role does not create or open the `addons` folder of the data path. The key set for +both roles comes from `local-ai util addons-carrier-keys`; see +[Running the owner as its own workload](#running-the-owner-as-its-own-workload). + +### The addons feature gate + +Fine-tuning and quantization are add-ons, so `--addons-enabled=false` turns off all +three. `GET /api/features` reports one key, `addons`, and the keys `fine_tuning` +and `quantization` carry the same value. The key is `true` when add-ons are +available on this instance: + +- In the `combined` and `owner` roles it is `true` when add-ons are enabled. +- In the `frontend` role it is `true` when add-ons are enabled and the owner + answered a reachability call within 2 seconds. The answer is kept for 10 seconds, + so `/api/features` does not contact the owner on every request. + +While the key is `false`: + +- The UI shows no Add-ons entry, no pinned apps, and no fine-tuning or + quantization entry. `/app/addons`, `/app/fine-tune` and `/app/quantize` lead back + to `/app`. The UI does not call `/api/addons/pins` or `/api/addons/builtin`. A + server that sends no `addons` key counts as `true` in the UI. +- Every path under `/api/addons`, `/api/fine-tuning` and `/api/quantization` + answers `503` with the same body, also for a path that does not exist. The + answer does not show which routes exist. The body is the LocalAI error envelope + (`{"error": {"message", "type", "code"}}`, code `unavailable`). The message is + "add-ons are turned off on this instance" when the feature is off and "the + add-on owner is not reachable" when a frontend cannot reach the owner. +- With `--addons-enabled=false` the handlers of these routes are not registered at + all. A `frontend` with `--addons-enabled=false` needs no `--addons-owner-address` + and no `--addons-carrier-dir`. It loads no keys and builds no carrier. If you set + `--addons-role`, `--addons-owner-address`, `--addons-owner-listen` or + `--addons-carrier-dir`, LocalAI ignores them and logs one line. + +This gate is for the whole instance. It does not replace the permission of each +user: the `addons`, fine-tuning and quantization permissions of a user are checked +after the gate. + +The reachability answer of a frontend is a per-instance observation and is not +shared. Two frontends can disagree for up to 10 seconds after the owner goes up or +down. A request through a frontend that still reports `true` gets the `503` of the +carrier while the owner is down. + +## Running the owner as its own workload + +A normal `local-ai run` is both the owner and the only frontend (the `combined` +role). In a deployment with several frontend replicas, run the owner as one +separate workload and let the frontends forward to it. The owner keeps all its +state in files under one root, as before. Frontends keep none and scale freely. + +- **One owner, one volume.** Run exactly one owner replica (`--addons-role=owner`) + with its own volume for the data path. Use a recreate-style rollout, so two + owner processes never overlap: the old one must stop before the new one starts. + A second owner on the same root stops at once with "addon storage already owned" + and the process exits with an error. It does not wait. +- **Fixed mount path.** The journal stores absolute paths. Mount the volume at the + same path on every start. If the data path or the mount point changes, every + installed add-on points at a directory that does not exist. +- **A local volume.** The root needs a local file system with working `flock`, + atomic rename and directory `fsync`. A network block volume that one node mounts + at a time is fine. A shared network file system is not supported. +- **A stable owner address.** Frontends reach the owner at `--addons-owner-address`. + The owner does not store it, so you can change it between starts. Use a stable + service name, never a pod address. +- **Frontends have no add-on volume.** A frontend (`--addons-role=frontend`) does + not create or open the `addons` folder. Give it no volume for it. +- **Existing add-ons.** A root written by a single node loads under the owner role + when the owner uses the same owner and storage identity. The stored address does + not matter. The journal keeps absolute paths, so the owner must see the same data + path and mount point as the node did. Otherwise the installed add-ons point at + missing directories. For an empty volume, the owner starts fresh. + +{{% notice warning %}} +The owner is a node-bound singleton. If the owner is down, add-ons are down: the +frontends stay up and answer `503` for every add-on request. Failover is manual. +There is no election and no standby owner. +{{% /notice %}} + +### Make the carrier keys + +The frontends and the owner trust each other through a static key set. Make it +once, anywhere you can run `local-ai`: + +```bash +local-ai util addons-carrier-keys --out ./carrier-keys --owner-host owner.example +``` + +`--owner-host` is the name the frontends dial, and it must match the host in +`--addons-owner-address`. The command prints which files go where: + +| Role | Mounts | +|---|---| +| Owner | `ca.pem`, `owner.pem`, `owner.key`, `frontend.pem`, `frontend-sign.pub` | +| Frontend | `ca.pem`, `owner.pem`, `frontend.pem`, `frontend.key`, `frontend-sign.key` | + +Store the set as a secret and mount only the files of each role. Point +`--addons-carrier-dir` at the mount. LocalAI refuses key files that group or other +users can read, so give them mode `0600` or tighter. + +All frontend replicas share one identity and one signing key, so a new or +rescheduled frontend is accepted at once and the owner needs no list of replicas. +Anyone who holds the frontend files can act as a frontend. A forwarded request +still carries the user, and the owner checks that user against the account +database, so the key set does not give a user any authority on its own. The command +does not keep the CA key, so nobody can make a third certificate from the set. + +{{% notice note %}} +Rotation is a coordinated restart. The frontends pin the exact certificate of the +owner and the owner pins the one of the frontends. To change the key set, make a +new one, update the secret, and restart the owner and every frontend. There is no +overlap window and no automatic renewal. +{{% /notice %}} + +### Example + +This Docker Compose file shows the add-on flags of one owner and two frontends. +Add the other flags of your deployment (the database, NATS and the rest of +[distributed mode]({{% relref "features/distributed-mode" %}})) to each service as +usual. + +```yaml +services: + owner: + image: localai/localai:latest + command: run + hostname: owner.example + environment: + LOCALAI_ADDONS_ROLE: owner + LOCALAI_ADDONS_OWNER_ADDRESS: https://owner.example:8443 + LOCALAI_ADDONS_CARRIER_DIR: /run/addon-keys + LOCALAI_ADDONS_OWNERSHIP_MODE: named-user + LOCALAI_DATA_PATH: /data + LOCALAI_AUTH: "true" + LOCALAI_AUTH_DATABASE_URL: postgres://localai:secret@db.example:5432/localai + volumes: + - owner-data:/data + - ./carrier-keys-owner:/run/addon-keys:ro + deploy: + replicas: 1 + update_config: + order: stop-first + + frontend: + image: localai/localai:latest + command: run + environment: + LOCALAI_ADDONS_ROLE: frontend + LOCALAI_ADDONS_OWNER_ADDRESS: https://owner.example:8443 + LOCALAI_ADDONS_CARRIER_DIR: /run/addon-keys + LOCALAI_ADDONS_OWNERSHIP_MODE: named-user + LOCALAI_AUTH: "true" + LOCALAI_AUTH_DATABASE_URL: postgres://localai:secret@db.example:5432/localai + volumes: + - ./carrier-keys-frontend:/run/addon-keys:ro + deploy: + replicas: 2 + +volumes: + owner-data: +``` + +`./carrier-keys-owner` holds the owner files and `./carrier-keys-frontend` holds the +frontend files from the table above. + +### Settings every role shares + +- **The same ownership mode.** Every role of a deployment must use the same + `--addons-ownership-mode`. +- **The same account database.** In `named-user` mode all roles must use the same + account database. The owner checks the user of each forwarded request against it. +- `--addons-enabled=false` keeps its meaning in every role. + +### What a restart looks like + +When the owner stops, the kernel drops its lock and its add-on processes end with +it. The new owner takes the lock, checks its identity file, and reads the journal: +operations that had only started are marked failed, and operations that were +already prepared or applied are finished. Recovery journals and pending proposals +finish or wait for an administrator. Then the add-ons that were enabled start again. + +While this happens, the frontends answer `503`. A frontend that starts while the +owner is down starts normally and answers `503` until the owner is up. Some state +is in memory only and is lost: an upgrade that was being canceled becomes failed, +and the memory of failed automatic upgrades is cleared. A mutation whose reply was +lost is reported as `outcome_unknown` with its operation ID, and relays that were +in progress (progress, file and asset) are cut. The owner picks a new audience at +each start. A request signed for the old owner is refused, and the frontend gets +the new audience on its next call without any action. + +### The models path + +Fine-tuning and quantization export models into the models path of the owner. +The owner and every frontend must mount the same models storage at the same +absolute `--models-path`. LocalAI checks this with a probe file: + +- At start, the owner writes `.localai-addons-probe.json` into its models path. + It holds a random nonce that changes at every start. Do not delete it, and do + not exclude it from the shared volume. LocalAI does not list it as a model. +- Each frontend checks at start, and then every 60 seconds, that it sees the + same file with the same nonce at the same absolute path. If it does not, the + frontend logs an error, reports `model_exports: false` in `/api/features`, + and answers the export and import routes with `503`. +- The owner refuses an export or import while its own probe file is missing, + also for the add-on host operations. If the owner cannot write the probe at + start, it keeps running and refuses exports. +- If the owner is unreachable, or is an older version without the probe, the + state is `unknown`. LocalAI logs a warning and leaves exports on. The state + clears within 60 seconds after the owner is back. + +The `/api/fine-tuning` and `/api/quantization` routes of a frontend still write +to the models path of that frontend. The probe is what blocks them when that +path is not the shared storage. + +### Frontend limits + +A process with `--addons-role=frontend` holds no add-on state. This has these +consequences: + +- **Library sources live on the owner and are changed through it.** The frontend + never opens, creates or writes `sources.json` or anything else under the `addons` + folder of the data path. Every `/api/library/sources` route is forwarded to the + owner, which is the only writer. The caller must be a named account and an + administrator: the frontend checks the account, and the owner checks the + administrator right again, as it does for its own routes. A legacy API key has no + named user and gets `401` on a frontend, in `local-owner` mode too. The routes + answer the same statuses as on a single node (`404` for an unknown source, `409` for + a duplicate, `400` for an invalid request). With add-ons off a frontend has no owner + to reach, and these routes answer `503` with the message "library sources are + managed on the owner". When the owner is down they answer `503` with + "the add-on owner is not reachable". A change whose reply was lost answers `503` + with the operation ID in the message; read the list again before you repeat it. +- **The gallery lists follow the owner.** A frontend asks the owner for its source + list every 60 seconds (a read over the pinned carrier that needs no signed-in user) + and, only when the list or the `official_libraries` setting + changed, rebuilds its model and backend gallery lists from the galleries named by + `--galleries` and `--backend-galleries` plus the owner's sources. A change made on + the owner shows up on every frontend after at most 60 seconds, and a change made + through a frontend shows up on that frontend at once. Until the owner has answered + once, a frontend reads only its own flags. While the owner does not answer, the + frontend keeps its last lists and logs that they may be stale. This is a + per-instance observation: two frontends can show different lists for up to 60 + seconds. +- **`official_libraries` is set on the owner.** The owner's value is the value of + the deployment, and a frontend takes it at each refresh. `POST /api/settings` with a + different `official_libraries` value answers `409` on a frontend. Set it on the + owner. +- **Backend installs run on the frontend node.** `POST /backends/library/install` + and `GET /backends/library/updates` work on a frontend. The owner resolves the + entry (including the choice of a meta backend for the capability of the frontend) + and pins the image by digest. The frontend then installs it into its own backends + path, verifies the signature there, and applies its own size limit and + `--require-backend-integrity`. The owner's backends path stays unchanged. The + update check compares the local backends with the releases the owner lists. + The background update check has no user, so it works in named-user mode through + one read-only system call (`POST /addons/library/system`, capability + `library-system`). The owner accepts it only from a pinned frontend, only for the + release lookup, and without an actor. Every call a user starts still needs the + administrator check on the owner. +- **The owner and the frontends must run on the same OS and architecture.** The + owner resolves entries for its own platform, so a frontend checks the platform the + owner reports (`GET /addons/capabilities`) and refuses every library call to an owner + on another one with a `503`. A frontend also refuses library calls to an owner that + does not serve them, with "the add-on owner does not support library operations; + upgrade the owner". Upgrade the owner before the frontends. +- **Pins are per frontend without an account database.** With an account database, + the pins of the Apps entry are shared by all frontends. Without one, each + frontend keeps its own pins in a file in its own data path, so two frontends can + show different pins. A frontend logs a warning at start when it has no account database. + ## The official library On the first start, when the owner has no `sources.json` in the `addons` folder yet, @@ -169,13 +454,15 @@ model gallery index and backend gallery index are not used for search, install o update checks. They stay in the store, still marked `default`, and the Sources tab shows them as off (`suspended: true` in the routes). Each source keeps its own enabled state, so turning the setting back on restores exactly what was there. Sources you added are not affected. The -change applies at once, with no restart. +change applies at once, with no restart. On a frontend the switch is read-only: the +owner's value applies, and a change there reaches the frontends within 60 seconds. ### Source routes The routes are for administrators. A caller who is not an administrator gets `403`, and a caller who is not signed in gets `401` when authentication is on. They are not add-on -routes, so they do not need the add-ons owner to run. +routes, so they do not need the add-ons owner to run. On a frontend they go to the owner +(see [Frontend limits](#frontend-limits)). | Method and path | What it does | |---|---| @@ -310,8 +597,9 @@ lifecycle. The page installs an add-on and turns it on in one action (Install an on); the install itself leaves an add-on disabled. A backend has no on or off: it is ready to use when the install ends, and its button says Install. -The route is `POST /backends/library/install`. It is for administrators and exists only -when the add-on owner runs on this instance: +The route is `POST /backends/library/install`. It is for administrators and exists where +the libraries are: on the instance that runs the add-on owner, and on a frontend, which asks the +owner to resolve the entry and installs the backend on its own node: ```bash curl -X POST http://localhost:8080/backends/library/install \ @@ -358,6 +646,10 @@ more than one channel, the check says nothing instead of guessing. The Update bu starts the same install as above, with the same steps, for the version the check names. There is no automatic upgrade. +The LocalAI Assistant and `local-ai mcp-server` expose this check as the read-only +`list_backend_library_updates` tool. Taking an update stays a confirmed +`install_library_backend` call. + A library install touches only the node that serves the request. In a distributed deployment the backend is installed on that node only; the per-node routes of the backend gallery are unchanged. diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index 40a1ecc3b4f8..3439094c5b2c 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -81,6 +81,16 @@ The frontend is a standard LocalAI instance with distributed mode enabled. These | `--model-config-resync-interval` | `LOCALAI_MODEL_CONFIG_RESYNC_INTERVAL` | `30s` | How often each frontend compares its model configs with the shared models directory, to apply a change whose NATS message it missed. A frontend that missed a message serves the old config for at most this long. See [Model configs across frontends](#model-configs-across-frontends). | | `--expose-node-header` | `LOCALAI_EXPOSE_NODE_HEADER` | `false` | When enabled, inference responses carry an `X-LocalAI-Node` header with the ID of the worker node that served the request. Coverage spans the OpenAI-compatible endpoints (chat completions, completions, embeddings, audio transcriptions, audio speech / TTS, image generations, image inpainting), the Jina rerank endpoint (`/v1/rerank`), the VAD endpoints (`/v1/vad`, `/vad`), and the Anthropic Messages (`/v1/messages`) and Ollama (`/api/chat`, `/api/generate`, `/api/embed`) shims. Useful for debugging, observability and load-balancer attribution. Off by default: the node ID reveals internal cluster topology and should not be exposed on a public endpoint. Best-effort: under heavy concurrency for the same model across multiple replicas, the header may reflect a recent routing decision rather than this exact request's. Acceptable for observability and debugging. | +### Add-ons + +Every `local-ai run` process is its own add-on owner by default. With several +frontend replicas, run the owner as one separate workload with its own volume, and +start the frontends with `--addons-role=frontend`, so they forward add-on requests +to it and hold no add-on state. The library sources and the backend library +installs also go through the owner: a frontend follows the owner's sources and +installs backends on its own node. See +[Running the owner as its own workload]({{% relref "features/addons#running-the-owner-as-its-own-workload" %}}). + ### The model load deadline scales with the checkpoint The `LoadModel` deadline starts *after* the backend is installed and the model files are staged, so it covers only the worker backend's own checkpoint read and pipeline init. That work is proportional to the bytes on disk, which makes any fixed deadline a model-size cliff rather than a timeout: a 70 GB video checkpoint on a Jetson Thor worker failed reproducibly against the old fixed 5m default (`rpc error: code = DeadlineExceeded` after 953.5s of wall clock, roughly 11m of which was backend install and staging), and simply raising the constant would only move the cliff to the next larger model while making a genuinely wedged *small* model hang for the whole inflated duration. diff --git a/docs/content/features/fine-tuning.md b/docs/content/features/fine-tuning.md index cbcba5f2af6e..e9d2cce05a28 100644 --- a/docs/content/features/fine-tuning.md +++ b/docs/content/features/fine-tuning.md @@ -78,6 +78,7 @@ curl -X POST http://localhost:8080/api/fine-tuning/jobs/{job_id}/export \ | Method | Path | Description | |--------|------|-------------| +| `GET` | `/api/fine-tuning/status` | Whether model exports are allowed on this instance (`{"model_exports": bool}`) | | `POST` | `/api/fine-tuning/jobs` | Start a fine-tuning job | | `GET` | `/api/fine-tuning/jobs` | List all jobs | | `GET` | `/api/fine-tuning/jobs/:id` | Get job details | @@ -87,6 +88,8 @@ curl -X POST http://localhost:8080/api/fine-tuning/jobs/{job_id}/export \ | `POST` | `/api/fine-tuning/jobs/:id/export` | Export model | | `POST` | `/api/fine-tuning/datasets` | Upload dataset file | +`GET /api/fine-tuning/status` and `GET /api/quantization/status` return `{"model_exports": true|false}`. The value is `false` only when this instance does not see the models path of the add-on owner; it is the same value as the `model_exports` key of `GET /api/features`. The Web UI disables the export controls while it is `false`. Both routes need an authenticated user and answer `503` when add-ons are off. + ### Job Request Fields | Field | Type | Description | @@ -188,6 +191,10 @@ curl -X POST http://localhost:8080/api/fine-tuning/jobs \ }' ``` +{{% notice note %}} +In a split deployment (`--addons-role=owner` and `--addons-role=frontend`), the export writes into the models path. That path must be the shared models storage. See [The models path](../addons/#the-models-path). +{{% /notice %}} + ### Export Formats | Format | Description | Notes | @@ -222,6 +229,8 @@ The old link `/app/fine-tune` keeps working. It opens the add-on page while the The API does not change. The page calls `/api/fine-tuning/*` with the signed-in user's own session, and the server checks the fine-tuning permission on every call, so the add-on never widens what a user may do. +With `--addons-enabled=false` this feature is off, like every add-on. The UI shows no entry for it, `/app/fine-tune` leads back to `/app`, and every path under `/api/fine-tuning` answers `503`. See [The addons feature gate](../addons/#the-addons-feature-gate). + The add-on also offers tools to the chat: list jobs, propose a job, store a small dataset, read the status and the checkpoints of a job, stop a job and export a checkpoint as a model. A tool can only control a job that a tool started. A job started in the page is listed, but a tool cannot stop or export it. A tool never starts, stops or exports anything by itself: it proposes the action and a person confirms it in an approval card. A model or a dataset given as a local path is refused. The tools call the engine through the host operations (see [Host operations](../addons/#host-operations)), which the host setting `LOCALAI_ADDON_HOST_OPS` grants to this add-on by the `finetune.` prefix, and they run with the permissions of the person in the chat. ## Dataset Formats diff --git a/docs/content/features/quantization.md b/docs/content/features/quantization.md index 06ce7e7d2f08..c6f16f2f1c5e 100644 --- a/docs/content/features/quantization.md +++ b/docs/content/features/quantization.md @@ -69,6 +69,9 @@ curl -X POST http://localhost:8080/api/quantization/jobs/{job_id}/import \ | `POST` | `/api/quantization/jobs/:id/import` | Import quantized model into LocalAI | | `GET` | `/api/quantization/jobs/:id/download` | Download quantized GGUF file | | `GET` | `/api/quantization/backends` | List available quantization backends | +| `GET` | `/api/quantization/status` | Whether model imports are allowed on this instance | + +`GET /api/fine-tuning/status` and `GET /api/quantization/status` return `{"model_exports": true|false}`. The value is `false` only when this instance does not see the models path of the add-on owner; it is the same value as the `model_exports` key of `GET /api/features`. The Web UI disables the import controls while it is `false`. Both routes need an authenticated user and answer `503` when add-ons are off. ### Job Request Fields @@ -85,6 +88,10 @@ curl -X POST http://localhost:8080/api/quantization/jobs/{job_id}/import \ |-----|-------------| | `hf_token` | HuggingFace token for gated models | +{{% notice note %}} +In a split deployment (`--addons-role=owner` and `--addons-role=frontend`), the export writes into the models path. That path must be the shared models storage. See [The models path](../addons/#the-models-path). +{{% /notice %}} + ### Import Request Fields | Field | Type | Description | @@ -161,6 +168,8 @@ The old link `/app/quantize` keeps working. It opens the add-on page while the a The API does not change. The page calls `/api/quantization/*` with the signed-in user's own session, and the server checks the quantization permission on every call. +With `--addons-enabled=false` this feature is off, like every add-on. The UI shows no entry for it, `/app/quantize` leads back to `/app`, and every path under `/api/quantization` answers `503`. See [The addons feature gate](../addons/#the-addons-feature-gate). + The add-on also offers tools to the chat: list jobs, propose a job, read the status of a job, stop a job and import the result as a model. A tool can only control a job that a tool started. A job started in the page is listed, but a tool cannot stop or import it. A tool never starts, stops or imports anything by itself: it proposes the action and a person confirms it in an approval card. A model given as a local path is refused. The tools call the engine through the host operations (see [Host operations](../addons/#host-operations)), which the host setting `LOCALAI_ADDON_HOST_OPS` grants to this add-on by the `quantize.` prefix, and they run with the permissions of the person in the chat. ## Architecture diff --git a/docs/content/reference/cli-reference.md b/docs/content/reference/cli-reference.md index cac35cbe9b3b..4794e6a122ea 100644 --- a/docs/content/reference/cli-reference.md +++ b/docs/content/reference/cli-reference.md @@ -64,6 +64,10 @@ Application shutdown has no context parameter; work that ignores cancellation ca | `--models-config-file` | | YAML file containing a list of model backend configs (alias: `--config-file`) | `$LOCALAI_MODELS_CONFIG_FILE`, `$CONFIG_FILE` | | `--addon-host-ops` | `fine-tuning=finetune.,quantization=quantize.` | Add-ons that may call host operations, as a comma list of `addon-id=operation-prefix`. An add-on that is not listed gets no host operation channel; an empty value lists none. See [Host operations]({{% relref "features/addons" %}}) | `$LOCALAI_ADDON_HOST_OPS` | +| `--addons-role` | `combined` | How this process takes part in the add-on owner: `combined` (the node is its own owner), `owner` or `frontend`. See [Turning the owner on or off]({{% relref "features/addons#turning-the-owner-on-or-off" %}}) | `$LOCALAI_ADDONS_ROLE` | +| `--addons-owner-address` | | `https://host:port` of the add-on owner carrier, with no path. Required for the `owner` and `frontend` roles | `$LOCALAI_ADDONS_OWNER_ADDRESS` | +| `--addons-owner-listen` | | `owner` role only: the address the carrier listens on. Empty listens on all interfaces, on the port of `--addons-owner-address` | `$LOCALAI_ADDONS_OWNER_LISTEN` | +| `--addons-carrier-dir` | | Directory with the carrier key set from `local-ai util addons-carrier-keys`. Required for the `owner` and `frontend` roles | `$LOCALAI_ADDONS_CARRIER_DIR` | | `--artifact-download-concurrency` | `1` | How many files of a model artifact to download at once. `1` downloads sequentially. Raising it helps artifacts split into many files on a fast link, at the cost of more concurrent load on the models volume. Whole files only — a single file is never split, so resume and per-file checksum verification are unaffected | `$LOCALAI_ARTIFACT_DOWNLOAD_CONCURRENCY` | Backend processes receive a private scratch directory through `TMPDIR`, `TMP`, @@ -278,6 +282,21 @@ LocalAI supports several subcommands beyond `run`: Use `local-ai --help` for more information on each command. +### Add-on carrier keys + +`local-ai util addons-carrier-keys` makes the key set that a separate add-on owner and its frontends share. See [Running the owner as its own workload]({{% relref "features/addons#running-the-owner-as-its-own-workload" %}}) for how the set is used. + +| Flag | Description | +|------|-------------| +| `--out` | Directory to write the key set to. Required. The command fails if the directory already holds any of the files, and it never overwrites a file. | +| `--owner-host` | DNS name or IP address of the add-on owner, as the frontends dial it. Required. Repeat the flag for more than one. | + +```bash +local-ai util addons-carrier-keys --out ./carrier-keys --owner-host addons.owner.example +``` + +The command prints which files go to the owner and which go to the frontends. It writes the files with mode `0600` in a directory with mode `0700`, and it does not keep the CA key. + ## Examples ### Basic Usage diff --git a/pkg/mcp/localaitools/client.go b/pkg/mcp/localaitools/client.go index 962a10179dc2..22f7cb01fbd3 100644 --- a/pkg/mcp/localaitools/client.go +++ b/pkg/mcp/localaitools/client.go @@ -61,6 +61,10 @@ type LocalAIClient interface { ListBackends(ctx context.Context) ([]Backend, error) // ListKnownBackends returns the same shape as REST /backends/known. ListKnownBackends(ctx context.Context) ([]schema.KnownBackend, error) + // ListBackendLibraryUpdates returns the installed library backends that + // have a newer build in their library. It reads the cached index and + // installs nothing. + ListBackendLibraryUpdates(ctx context.Context) ([]BackendLibraryUpdate, error) InstallBackend(ctx context.Context, req InstallBackendRequest) (jobID string, err error) // InstallLibraryBackend installs a backend from a library entry. It // returns when the install is done. diff --git a/pkg/mcp/localaitools/coverage_test.go b/pkg/mcp/localaitools/coverage_test.go index da114455ff44..0eaa3e798f0a 100644 --- a/pkg/mcp/localaitools/coverage_test.go +++ b/pkg/mcp/localaitools/coverage_test.go @@ -26,27 +26,28 @@ import ( // the contributor explicitly acknowledges the asymmetry. var toolToHTTPRoute = map[string]string{ // Read-only tools. - ToolGallerySearch: "GET /models/available", - ToolListInstalledModels: "GET / (welcome JSON, ModelsConfig field)", - ToolListGalleries: "GET /models/galleries", - ToolGetJobStatus: "GET /models/jobs/:uuid", - ToolGetModelConfig: "(none) — no JSON-only REST yet; httpapi.Client returns a documented stub", - ToolListBackends: "GET /backends", - ToolListKnownBackends: "GET /backends/known", - ToolSystemInfo: "GET / (welcome JSON)", - ToolListNodes: "GET /api/nodes", - ToolListScheduling: "GET /api/nodes/scheduling", - ToolGetScheduling: "GET /api/nodes/scheduling/:model", - ToolVRAMEstimate: "POST /api/models/vram-estimate", - ToolGetBranding: "GET /api/branding", - ToolGetUsageStats: "GET /api/usage (or /api/usage/all when all=true)", - ToolGetPIIEvents: "GET /api/pii/events", - ToolGetMiddlewareStatus: "GET /api/middleware/status", - ToolGetRouterDecisions: "GET /api/router/decisions", - ToolGetRouterCorpusStats: "GET /api/router/:name/corpus/stats", - ToolListAliases: "GET /api/aliases", - ToolListVoiceProfiles: "GET /api/voice-profiles", - ToolListFailoverChains: "GET /api/failover", + ToolGallerySearch: "GET /models/available", + ToolListInstalledModels: "GET / (welcome JSON, ModelsConfig field)", + ToolListGalleries: "GET /models/galleries", + ToolGetJobStatus: "GET /models/jobs/:uuid", + ToolGetModelConfig: "(none) — no JSON-only REST yet; httpapi.Client returns a documented stub", + ToolListBackends: "GET /backends", + ToolListKnownBackends: "GET /backends/known", + ToolListBackendLibraryUpdates: "GET /backends/library/updates", + ToolSystemInfo: "GET / (welcome JSON)", + ToolListNodes: "GET /api/nodes", + ToolListScheduling: "GET /api/nodes/scheduling", + ToolGetScheduling: "GET /api/nodes/scheduling/:model", + ToolVRAMEstimate: "POST /api/models/vram-estimate", + ToolGetBranding: "GET /api/branding", + ToolGetUsageStats: "GET /api/usage (or /api/usage/all when all=true)", + ToolGetPIIEvents: "GET /api/pii/events", + ToolGetMiddlewareStatus: "GET /api/middleware/status", + ToolGetRouterDecisions: "GET /api/router/decisions", + ToolGetRouterCorpusStats: "GET /api/router/:name/corpus/stats", + ToolListAliases: "GET /api/aliases", + ToolListVoiceProfiles: "GET /api/voice-profiles", + ToolListFailoverChains: "GET /api/failover", // Mutating tools. ToolInstallModel: "POST /models/apply", diff --git a/pkg/mcp/localaitools/dto.go b/pkg/mcp/localaitools/dto.go index a2e3bf07a118..e2fb2411f343 100644 --- a/pkg/mcp/localaitools/dto.go +++ b/pkg/mcp/localaitools/dto.go @@ -91,6 +91,19 @@ type InstallLibraryBackendResult struct { SourceID string `json:"source_id"` } +// BackendLibraryUpdate is one installed library backend with a newer build in +// its library, as list_backend_library_updates returns it. Version is what an +// install_library_backend call names to take the update. +type BackendLibraryUpdate struct { + Name string `json:"name"` + SourceID string `json:"sourceId"` + InstalledVersion string `json:"installedVersion,omitempty"` + InstalledDigest string `json:"installedDigest,omitempty"` + Version string `json:"version"` + Digest string `json:"digest"` + Signed bool `json:"signed"` +} + // Backend is the LLM-facing summary returned by list_backends. We don't // expose gallery.SystemBackend directly because it carries filesystem // paths (RunFile, IsSystem, IsMeta, the full Metadata) the LLM doesn't diff --git a/pkg/mcp/localaitools/fakes_test.go b/pkg/mcp/localaitools/fakes_test.go index 37812bb1b7c2..979c5100aeaa 100644 --- a/pkg/mcp/localaitools/fakes_test.go +++ b/pkg/mcp/localaitools/fakes_test.go @@ -219,6 +219,11 @@ func (f *fakeClient) InstallBackend(_ context.Context, req InstallBackendRequest return "", errNotConfigured } +func (f *fakeClient) ListBackendLibraryUpdates(_ context.Context) ([]BackendLibraryUpdate, error) { + f.record("ListBackendLibraryUpdates", nil) + return []BackendLibraryUpdate{}, nil +} + func (f *fakeClient) InstallLibraryBackend(_ context.Context, req InstallLibraryBackendRequest) (*InstallLibraryBackendResult, error) { f.record("InstallLibraryBackend", req) if f.installLibraryBackend != nil { diff --git a/pkg/mcp/localaitools/httpapi/client.go b/pkg/mcp/localaitools/httpapi/client.go index db02796d1c15..d7dd5f890ece 100644 --- a/pkg/mcp/localaitools/httpapi/client.go +++ b/pkg/mcp/localaitools/httpapi/client.go @@ -413,6 +413,19 @@ func (c *Client) ListKnownBackends(ctx context.Context) ([]schema.KnownBackend, return out, nil } +func (c *Client) ListBackendLibraryUpdates(ctx context.Context) ([]localaitools.BackendLibraryUpdate, error) { + var resp struct { + Updates []localaitools.BackendLibraryUpdate `json:"updates"` + } + if err := c.do(ctx, http.MethodGet, routeBackendsLibraryUpdates, nil, &resp); err != nil { + return nil, err + } + if resp.Updates == nil { + resp.Updates = []localaitools.BackendLibraryUpdate{} + } + return resp.Updates, nil +} + func (c *Client) InstallBackend(ctx context.Context, req localaitools.InstallBackendRequest) (string, error) { body := map[string]any{"id": req.BackendName} if req.GalleryName != "" { diff --git a/pkg/mcp/localaitools/httpapi/client_test.go b/pkg/mcp/localaitools/httpapi/client_test.go index 06327f1971fa..5a1faf5cdb85 100644 --- a/pkg/mcp/localaitools/httpapi/client_test.go +++ b/pkg/mcp/localaitools/httpapi/client_test.go @@ -501,3 +501,23 @@ var _ = Describe("InstallLibraryBackend", func() { Expect(res.Status).To(Equal("installed")) }) }) + +var _ = Describe("ListBackendLibraryUpdates", func() { + It("reads the backend library updates route", func() { + var path string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + path = r.Method + " " + r.URL.Path + _ = json.NewEncoder(w).Encode(map[string]any{"updates": []map[string]any{ + {"name": "demo", "sourceId": "official", "installedVersion": "1.0.0", "version": "1.1.0", "digest": "sha256:aa", "signed": true}, + }}) + })) + DeferCleanup(srv.Close) + + res, err := New(srv.URL, "").ListBackendLibraryUpdates(context.Background()) + Expect(err).ToNot(HaveOccurred()) + Expect(path).To(Equal("GET /backends/library/updates")) + Expect(res).To(HaveLen(1)) + Expect(res[0].SourceID).To(Equal("official")) + Expect(res[0].Version).To(Equal("1.1.0")) + }) +}) diff --git a/pkg/mcp/localaitools/httpapi/routes.go b/pkg/mcp/localaitools/httpapi/routes.go index a890aad3f578..088f3ec97902 100644 --- a/pkg/mcp/localaitools/httpapi/routes.go +++ b/pkg/mcp/localaitools/httpapi/routes.go @@ -24,6 +24,7 @@ const ( routeBackendsKnown = "/backends/known" routeBackendsApply = "/backends/apply" routeBackendsLibraryInstall = "/backends/library/install" + routeBackendsLibraryUpdates = "/backends/library/updates" routeNodes = "/api/nodes" routeScheduling = "/api/nodes/scheduling" routeVRAMEstimate = "/api/models/vram-estimate" diff --git a/pkg/mcp/localaitools/inproc/client.go b/pkg/mcp/localaitools/inproc/client.go index 403000b20a83..6247205c4997 100644 --- a/pkg/mcp/localaitools/inproc/client.go +++ b/pkg/mcp/localaitools/inproc/client.go @@ -488,6 +488,14 @@ func (c *Client) InstallBackend(ctx context.Context, req localaitools.InstallBac // installer the REST route uses, so both paths share one code path. type LibraryBackendInstaller interface { Install(ctx context.Context, sourceID, backend, version string) error + Updates(ctx context.Context) ([]localaitools.BackendLibraryUpdate, error) +} + +func (c *Client) ListBackendLibraryUpdates(ctx context.Context) ([]localaitools.BackendLibraryUpdate, error) { + if c.BackendLibrary == nil { + return nil, errors.New("the backend library is not enabled on this instance") + } + return c.BackendLibrary.Updates(ctx) } func (c *Client) InstallLibraryBackend(ctx context.Context, req localaitools.InstallLibraryBackendRequest) (*localaitools.InstallLibraryBackendResult, error) { diff --git a/pkg/mcp/localaitools/prompts/20_tools.md b/pkg/mcp/localaitools/prompts/20_tools.md index 77f87b00f2a9..4a03bafeb7d8 100644 --- a/pkg/mcp/localaitools/prompts/20_tools.md +++ b/pkg/mcp/localaitools/prompts/20_tools.md @@ -8,6 +8,7 @@ The MCP `tools/list` endpoint also exposes the full input schema for each of the - `list_installed_models` — List models currently installed on this LocalAI. Optional `capability` filter (e.g. `chat`, `embed`, `image`). - `list_galleries` — List configured model galleries. - `list_backends` — List installed backends. +- `list_backend_library_updates` — List installed library backends that have a newer build in their library. Read-only. To take one, confirm, then call `install_library_backend` with the listed `sourceId`, `name` and `version`. - `list_known_backends` — List backends available to install from configured backend galleries. - `get_job_status` — Poll the status of an install/delete/upgrade job by id. - `get_model_config` — Read the YAML/JSON config of an installed model. diff --git a/pkg/mcp/localaitools/server_test.go b/pkg/mcp/localaitools/server_test.go index cbd29eba5546..5bbdb1ca8683 100644 --- a/pkg/mcp/localaitools/server_test.go +++ b/pkg/mcp/localaitools/server_test.go @@ -86,6 +86,7 @@ var expectedReadOnlyCatalog = sortedStrings( ToolListAliases, ToolListInstalledModels, ToolListKnownBackends, + ToolListBackendLibraryUpdates, ToolListNodes, ToolListScheduling, ToolGetScheduling, @@ -135,6 +136,7 @@ var _ = Describe("Tool dispatch", func() { {ToolListGalleries, struct{}{}, "ListGalleries"}, {ToolListBackends, struct{}{}, "ListBackends"}, {ToolListKnownBackends, struct{}{}, "ListKnownBackends"}, + {ToolListBackendLibraryUpdates, struct{}{}, "ListBackendLibraryUpdates"}, {ToolSystemInfo, struct{}{}, "SystemInfo"}, {ToolListNodes, struct{}{}, "ListNodes"}, {ToolListScheduling, struct{}{}, "ListScheduling"}, diff --git a/pkg/mcp/localaitools/tools.go b/pkg/mcp/localaitools/tools.go index da1afb5d54d6..5bcfe9400e8a 100644 --- a/pkg/mcp/localaitools/tools.go +++ b/pkg/mcp/localaitools/tools.go @@ -8,25 +8,26 @@ package localaitools // that the mutating names stay aligned with the confirmation rule. const ( // Read-only tools. - ToolGallerySearch = "gallery_search" - ToolListInstalledModels = "list_installed_models" - ToolListGalleries = "list_galleries" - ToolGetJobStatus = "get_job_status" - ToolGetModelConfig = "get_model_config" - ToolListBackends = "list_backends" - ToolListKnownBackends = "list_known_backends" - ToolSystemInfo = "system_info" - ToolListNodes = "list_nodes" - ToolListScheduling = "list_scheduling" - ToolGetScheduling = "get_scheduling" - ToolVRAMEstimate = "vram_estimate" - ToolGetBranding = "get_branding" - ToolGetUsageStats = "get_usage_stats" - ToolGetPIIEvents = "get_pii_events" - ToolGetMiddlewareStatus = "get_middleware_status" - ToolGetRouterDecisions = "get_router_decisions" - ToolGetRouterCorpusStats = "get_router_corpus_stats" - ToolListVoiceProfiles = "list_voice_profiles" + ToolGallerySearch = "gallery_search" + ToolListInstalledModels = "list_installed_models" + ToolListGalleries = "list_galleries" + ToolGetJobStatus = "get_job_status" + ToolGetModelConfig = "get_model_config" + ToolListBackends = "list_backends" + ToolListKnownBackends = "list_known_backends" + ToolListBackendLibraryUpdates = "list_backend_library_updates" + ToolSystemInfo = "system_info" + ToolListNodes = "list_nodes" + ToolListScheduling = "list_scheduling" + ToolGetScheduling = "get_scheduling" + ToolVRAMEstimate = "vram_estimate" + ToolGetBranding = "get_branding" + ToolGetUsageStats = "get_usage_stats" + ToolGetPIIEvents = "get_pii_events" + ToolGetMiddlewareStatus = "get_middleware_status" + ToolGetRouterDecisions = "get_router_decisions" + ToolGetRouterCorpusStats = "get_router_corpus_stats" + ToolListVoiceProfiles = "list_voice_profiles" // Mutating tools — guarded by Options.DisableMutating and the // LLM-side safety prompt (see prompts/10_safety.md). diff --git a/pkg/mcp/localaitools/tools_backends.go b/pkg/mcp/localaitools/tools_backends.go index 3b721ed3c55c..61990d750182 100644 --- a/pkg/mcp/localaitools/tools_backends.go +++ b/pkg/mcp/localaitools/tools_backends.go @@ -29,6 +29,17 @@ func registerBackendTools(s *mcp.Server, client LocalAIClient, opts Options) { return jsonResult(backends), nil, nil }) + mcp.AddTool(s, &mcp.Tool{ + Name: ToolListBackendLibraryUpdates, + Description: "List installed library backends that have a newer build in their library. Reads the cached library index and installs nothing.", + }, func(ctx context.Context, _ *mcp.CallToolRequest, _ struct{}) (*mcp.CallToolResult, any, error) { + updates, err := client.ListBackendLibraryUpdates(ctx) + if err != nil { + return errorResult(err), nil, nil + } + return jsonResult(map[string]any{"updates": updates}), nil, nil + }) + if opts.DisableMutating { return } diff --git a/pkg/system/capabilities.go b/pkg/system/capabilities.go index ffe9a4c96594..23de67a2781e 100644 --- a/pkg/system/capabilities.go +++ b/pkg/system/capabilities.go @@ -304,8 +304,17 @@ func (s *SystemState) CapabilityFilterDisabled() bool { } func (s *SystemState) Capability(capMap map[string]string) string { - reportedCapability := s.getSystemCapabilities() + return PickCapability(s.getSystemCapabilities(), capMap) +} + +// ReportedCapability is the capability this host reports, before it is matched +// against the capabilities a meta backend lists. A process that resolves a meta +// backend for another host sends it, and that host calls PickCapability. +func (s *SystemState) ReportedCapability() string { return s.getSystemCapabilities() } +// PickCapability chooses the key of capMap that serves a host which reports +// reportedCapability. Capability uses it for the local host. +func PickCapability(reportedCapability string, capMap map[string]string) string { // Check if the reported capability is in the map if _, exists := capMap[reportedCapability]; exists { xlog.Debug("Using reported capability", "reportedCapability", reportedCapability, "capMap", capMap) diff --git a/swagger/docs.go b/swagger/docs.go index 9f441f67bd1b..56abb07f3e63 100644 --- a/swagger/docs.go +++ b/swagger/docs.go @@ -3941,6 +3941,34 @@ const docTemplate = `{ } } }, + "/api/fine-tuning/status": { + "get": { + "description": "Returns model_exports=false when this instance does not share the models path of the add-on owner. The value is the same as the model_exports key of GET /api/features.", + "produces": [ + "application/json" + ], + "tags": [ + "fine-tuning", + "quantization" + ], + "summary": "Report whether model exports are allowed", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/localai.ModelExportsStatus" + } + }, + "503": { + "description": "add-ons are off or the owner is not reachable", + "schema": { + "type": "object", + "additionalProperties": true + } + } + } + } + }, "/api/instructions": { "get": { "description": "Returns a compact list of instruction areas with descriptions and URLs for detailed guides", @@ -4023,6 +4051,12 @@ const docTemplate = `{ "schema": { "$ref": "#/definitions/localai.LibrarySourcesResponse" } + }, + "503": { + "description": "on a frontend: the add-on owner is not reachable, does not serve library calls, or runs on another platform; the message says which", + "schema": { + "$ref": "#/definitions/schema.ErrorResponse" + } } } }, @@ -4044,11 +4078,53 @@ const docTemplate = `{ "schema": { "$ref": "#/definitions/library.Info" } + }, + "503": { + "description": "on a frontend: the add-on owner is not reachable, does not serve library calls, or runs on another platform; the message says which", + "schema": { + "$ref": "#/definitions/schema.ErrorResponse" + } } } } }, "/api/library/sources/{id}": { + "put": { + "description": "Replaces the mirrors of a model or backend source; an empty list clears them. Each mirror is an http(s), github:, oci:// or file:// address. Administrators only.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "library" + ], + "summary": "Change the mirrors of a library source", + "parameters": [ + { + "type": "string", + "description": "Source id", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/library.Info" + } + }, + "503": { + "description": "on a frontend: the add-on owner is not reachable, does not serve library calls, or runs on another platform; the message says which", + "schema": { + "$ref": "#/definitions/schema.ErrorResponse" + } + } + } + }, "delete": { "description": "A source of your own is deleted. A shipped default is not: it stays as a tombstone (removed true), so it does not come back at the next start; enabling it brings it back. Administrators only.", "produces": [ @@ -4074,6 +4150,12 @@ const docTemplate = `{ "type": "object", "additionalProperties": true } + }, + "503": { + "description": "on a frontend: the add-on owner is not reachable, does not serve library calls, or runs on another platform; the message says which", + "schema": { + "$ref": "#/definitions/schema.ErrorResponse" + } } } } @@ -4103,6 +4185,12 @@ const docTemplate = `{ "schema": { "$ref": "#/definitions/library.Info" } + }, + "503": { + "description": "on a frontend: the add-on owner is not reachable, does not serve library calls, or runs on another platform; the message says which", + "schema": { + "$ref": "#/definitions/schema.ErrorResponse" + } } } } @@ -4132,6 +4220,12 @@ const docTemplate = `{ "schema": { "$ref": "#/definitions/library.Info" } + }, + "503": { + "description": "on a frontend: the add-on owner is not reachable, does not serve library calls, or runs on another platform; the message says which", + "schema": { + "$ref": "#/definitions/schema.ErrorResponse" + } } } } @@ -4920,6 +5014,34 @@ const docTemplate = `{ } } }, + "/api/quantization/status": { + "get": { + "description": "Returns model_exports=false when this instance does not share the models path of the add-on owner. The value is the same as the model_exports key of GET /api/features.", + "produces": [ + "application/json" + ], + "tags": [ + "fine-tuning", + "quantization" + ], + "summary": "Report whether model imports are allowed", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/localai.ModelExportsStatus" + } + }, + "503": { + "description": "add-ons are off or the owner is not reachable", + "schema": { + "type": "object", + "additionalProperties": true + } + } + } + } + }, "/api/router/decide": { "post": { "consumes": [ @@ -5880,6 +6002,12 @@ const docTemplate = `{ "schema": { "$ref": "#/definitions/schema.ErrorResponse" } + }, + "503": { + "description": "on a frontend: the add-on owner is not reachable, does not serve library calls, or runs on another platform; the message says which", + "schema": { + "$ref": "#/definitions/schema.ErrorResponse" + } } } } @@ -5902,6 +6030,12 @@ const docTemplate = `{ } } } + }, + "503": { + "description": "on a frontend: the add-on owner is not reachable, does not serve library calls, or runs on another platform; the message says which", + "schema": { + "$ref": "#/definitions/schema.ErrorResponse" + } } } } @@ -9917,6 +10051,13 @@ const docTemplate = `{ "kind": { "$ref": "#/definitions/catalog.Kind" }, + "mirrors": { + "description": "Mirrors are the fallback addresses of the source, in order.", + "type": "array", + "items": { + "type": "string" + } + }, "name": { "type": "string" }, @@ -10134,6 +10275,15 @@ const docTemplate = `{ } } }, + "localai.ModelExportsStatus": { + "type": "object", + "properties": { + "model_exports": { + "description": "ModelExports is false only when this instance does not see the models\npath of the add-on owner. It is the same value as the model_exports key\nof GET /api/features.", + "type": "boolean" + } + } + }, "localai.ModelResponse": { "type": "object", "properties": { diff --git a/swagger/swagger.json b/swagger/swagger.json index aaf7e31d7ade..b7313248ca97 100644 --- a/swagger/swagger.json +++ b/swagger/swagger.json @@ -3938,6 +3938,34 @@ } } }, + "/api/fine-tuning/status": { + "get": { + "description": "Returns model_exports=false when this instance does not share the models path of the add-on owner. The value is the same as the model_exports key of GET /api/features.", + "produces": [ + "application/json" + ], + "tags": [ + "fine-tuning", + "quantization" + ], + "summary": "Report whether model exports are allowed", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/localai.ModelExportsStatus" + } + }, + "503": { + "description": "add-ons are off or the owner is not reachable", + "schema": { + "type": "object", + "additionalProperties": true + } + } + } + } + }, "/api/instructions": { "get": { "description": "Returns a compact list of instruction areas with descriptions and URLs for detailed guides", @@ -4020,6 +4048,12 @@ "schema": { "$ref": "#/definitions/localai.LibrarySourcesResponse" } + }, + "503": { + "description": "on a frontend: the add-on owner is not reachable, does not serve library calls, or runs on another platform; the message says which", + "schema": { + "$ref": "#/definitions/schema.ErrorResponse" + } } } }, @@ -4041,11 +4075,53 @@ "schema": { "$ref": "#/definitions/library.Info" } + }, + "503": { + "description": "on a frontend: the add-on owner is not reachable, does not serve library calls, or runs on another platform; the message says which", + "schema": { + "$ref": "#/definitions/schema.ErrorResponse" + } } } } }, "/api/library/sources/{id}": { + "put": { + "description": "Replaces the mirrors of a model or backend source; an empty list clears them. Each mirror is an http(s), github:, oci:// or file:// address. Administrators only.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "library" + ], + "summary": "Change the mirrors of a library source", + "parameters": [ + { + "type": "string", + "description": "Source id", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/library.Info" + } + }, + "503": { + "description": "on a frontend: the add-on owner is not reachable, does not serve library calls, or runs on another platform; the message says which", + "schema": { + "$ref": "#/definitions/schema.ErrorResponse" + } + } + } + }, "delete": { "description": "A source of your own is deleted. A shipped default is not: it stays as a tombstone (removed true), so it does not come back at the next start; enabling it brings it back. Administrators only.", "produces": [ @@ -4071,6 +4147,12 @@ "type": "object", "additionalProperties": true } + }, + "503": { + "description": "on a frontend: the add-on owner is not reachable, does not serve library calls, or runs on another platform; the message says which", + "schema": { + "$ref": "#/definitions/schema.ErrorResponse" + } } } } @@ -4100,6 +4182,12 @@ "schema": { "$ref": "#/definitions/library.Info" } + }, + "503": { + "description": "on a frontend: the add-on owner is not reachable, does not serve library calls, or runs on another platform; the message says which", + "schema": { + "$ref": "#/definitions/schema.ErrorResponse" + } } } } @@ -4129,6 +4217,12 @@ "schema": { "$ref": "#/definitions/library.Info" } + }, + "503": { + "description": "on a frontend: the add-on owner is not reachable, does not serve library calls, or runs on another platform; the message says which", + "schema": { + "$ref": "#/definitions/schema.ErrorResponse" + } } } } @@ -4917,6 +5011,34 @@ } } }, + "/api/quantization/status": { + "get": { + "description": "Returns model_exports=false when this instance does not share the models path of the add-on owner. The value is the same as the model_exports key of GET /api/features.", + "produces": [ + "application/json" + ], + "tags": [ + "fine-tuning", + "quantization" + ], + "summary": "Report whether model imports are allowed", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/localai.ModelExportsStatus" + } + }, + "503": { + "description": "add-ons are off or the owner is not reachable", + "schema": { + "type": "object", + "additionalProperties": true + } + } + } + } + }, "/api/router/decide": { "post": { "consumes": [ @@ -5877,6 +5999,12 @@ "schema": { "$ref": "#/definitions/schema.ErrorResponse" } + }, + "503": { + "description": "on a frontend: the add-on owner is not reachable, does not serve library calls, or runs on another platform; the message says which", + "schema": { + "$ref": "#/definitions/schema.ErrorResponse" + } } } } @@ -5899,6 +6027,12 @@ } } } + }, + "503": { + "description": "on a frontend: the add-on owner is not reachable, does not serve library calls, or runs on another platform; the message says which", + "schema": { + "$ref": "#/definitions/schema.ErrorResponse" + } } } } @@ -9914,6 +10048,13 @@ "kind": { "$ref": "#/definitions/catalog.Kind" }, + "mirrors": { + "description": "Mirrors are the fallback addresses of the source, in order.", + "type": "array", + "items": { + "type": "string" + } + }, "name": { "type": "string" }, @@ -10131,6 +10272,15 @@ } } }, + "localai.ModelExportsStatus": { + "type": "object", + "properties": { + "model_exports": { + "description": "ModelExports is false only when this instance does not see the models\npath of the add-on owner. It is the same value as the model_exports key\nof GET /api/features.", + "type": "boolean" + } + } + }, "localai.ModelResponse": { "type": "object", "properties": { diff --git a/swagger/swagger.yaml b/swagger/swagger.yaml index 51b58a0c0843..c75e7169ca44 100644 --- a/swagger/swagger.yaml +++ b/swagger/swagger.yaml @@ -1631,6 +1631,11 @@ definitions: type: string kind: $ref: '#/definitions/catalog.Kind' + mirrors: + description: Mirrors are the fallback addresses of the source, in order. + items: + type: string + type: array name: type: string policy: @@ -1798,6 +1803,15 @@ definitions: $ref: '#/definitions/library.Info' type: array type: object + localai.ModelExportsStatus: + properties: + model_exports: + description: |- + ModelExports is false only when this instance does not see the models + path of the add-on owner. It is the same value as the model_exports key + of GET /api/features. + type: boolean + type: object localai.ModelResponse: properties: config: {} @@ -8446,6 +8460,27 @@ paths: summary: Stream failover events (server-sent events) tags: - failover + /api/fine-tuning/status: + get: + description: Returns model_exports=false when this instance does not share the + models path of the add-on owner. The value is the same as the model_exports + key of GET /api/features. + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/localai.ModelExportsStatus' + "503": + description: add-ons are off or the owner is not reachable + schema: + additionalProperties: true + type: object + summary: Report whether model exports are allowed + tags: + - fine-tuning + - quantization /api/instructions: get: description: Returns a compact list of instruction areas with descriptions and @@ -8507,6 +8542,11 @@ paths: description: OK schema: $ref: '#/definitions/localai.LibrarySourcesResponse' + "503": + description: 'on a frontend: the add-on owner is not reachable, does not + serve library calls, or runs on another platform; the message says which' + schema: + $ref: '#/definitions/schema.ErrorResponse' summary: List the library sources tags: - library @@ -8524,6 +8564,11 @@ paths: description: Created schema: $ref: '#/definitions/library.Info' + "503": + description: 'on a frontend: the add-on owner is not reachable, does not + serve library calls, or runs on another platform; the message says which' + schema: + $ref: '#/definitions/schema.ErrorResponse' summary: Add a library source tags: - library @@ -8546,9 +8591,41 @@ paths: schema: additionalProperties: true type: object + "503": + description: 'on a frontend: the add-on owner is not reachable, does not + serve library calls, or runs on another platform; the message says which' + schema: + $ref: '#/definitions/schema.ErrorResponse' summary: Remove a library source tags: - library + put: + consumes: + - application/json + description: Replaces the mirrors of a model or backend source; an empty list + clears them. Each mirror is an http(s), github:, oci:// or file:// address. + Administrators only. + parameters: + - description: Source id + in: path + name: id + required: true + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/library.Info' + "503": + description: 'on a frontend: the add-on owner is not reachable, does not + serve library calls, or runs on another platform; the message says which' + schema: + $ref: '#/definitions/schema.ErrorResponse' + summary: Change the mirrors of a library source + tags: + - library /api/library/sources/{id}/disable: post: description: Enabling a removed default brings it back. Administrators only. @@ -8565,6 +8642,11 @@ paths: description: OK schema: $ref: '#/definitions/library.Info' + "503": + description: 'on a frontend: the add-on owner is not reachable, does not + serve library calls, or runs on another platform; the message says which' + schema: + $ref: '#/definitions/schema.ErrorResponse' summary: Turn a library source on or off tags: - library @@ -8584,6 +8666,11 @@ paths: description: OK schema: $ref: '#/definitions/library.Info' + "503": + description: 'on a frontend: the add-on owner is not reachable, does not + serve library calls, or runs on another platform; the message says which' + schema: + $ref: '#/definitions/schema.ErrorResponse' summary: Turn a library source on or off tags: - library @@ -9133,6 +9220,27 @@ paths: summary: Redact PII in a string by applying the configured policy. tags: - pii + /api/quantization/status: + get: + description: Returns model_exports=false when this instance does not share the + models path of the add-on owner. The value is the same as the model_exports + key of GET /api/features. + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/localai.ModelExportsStatus' + "503": + description: add-ons are off or the owner is not reachable + schema: + additionalProperties: true + type: object + summary: Report whether model imports are allowed + tags: + - fine-tuning + - quantization /api/router/{name}/corpus: delete: parameters: @@ -9793,6 +9901,11 @@ paths: description: the same backend is already being installed schema: $ref: '#/definitions/schema.ErrorResponse' + "503": + description: 'on a frontend: the add-on owner is not reachable, does not + serve library calls, or runs on another platform; the message says which' + schema: + $ref: '#/definitions/schema.ErrorResponse' summary: Install a backend from a library entry on this node. tags: - backends @@ -9807,6 +9920,11 @@ paths: $ref: '#/definitions/backendlib.Update' type: array type: object + "503": + description: 'on a frontend: the add-on owner is not reachable, does not + serve library calls, or runs on another platform; the message says which' + schema: + $ref: '#/definitions/schema.ErrorResponse' summary: List the installed library backends with a newer build in their library. tags: - backends