From 5569814409dbb02233cc082f377847896faf9a53 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> Date: Mon, 28 Sep 2026 02:29:55 +0000 Subject: [PATCH 1/5] feat(3d): add Pixal3D multiview generation Support four ordered RGBA views through the native Pixal3D CLI. Wire request validation, distributed staging, discovery, packaging, and a dedicated UI form while preserving single-image TRELLIS. Generate native output in a temporary directory and copy only the validated GLB so texture and mesh sidecars do not accumulate. Assisted-by: Codex:GPT-6 Signed-off-by: Ettore Di Giacinto --- .github/backend-matrix.yml | 86 ++++ .github/workflows/bump_deps.yaml | 4 + Makefile | 11 +- README.md | 4 + addons/studio/ui/src/pages/ThreeDGen.jsx | 39 +- backend/backend.proto | 2 + backend/go/pixal3dcpp/Makefile | 53 +++ backend/go/pixal3dcpp/main.go | 15 + backend/go/pixal3dcpp/package.sh | 26 ++ backend/go/pixal3dcpp/pixal3d.go | 209 ++++++++++ backend/go/pixal3dcpp/pixal3d_test.go | 106 +++++ backend/go/pixal3dcpp/run.sh | 9 + backend/index.yaml | 282 +++++++++++++ core/backend/model3d.go | 4 + core/config/backend_capabilities.go | 6 + core/config/model_3d.go | 14 + core/config/model_config.go | 2 +- core/config/pixal3d_test.go | 17 + core/gallery/importers/importers.go | 1 + core/gallery/importers/pixal3d.go | 67 ++++ core/gallery/importers/pixal3d_LICENSE | 22 ++ core/gallery/importers/pixal3d_models.json | 71 ++++ core/gallery/importers/pixal3d_test.go | 24 ++ core/http/endpoints/localai/model3d.go | 54 ++- .../localai/model3d_pixal3d_internal_test.go | 97 +++++ core/http/react-ui/e2e/threed-gen.spec.js | 30 ++ core/schema/localai.go | 4 +- core/services/nodes/file_staging_client.go | 10 + .../nodes/file_staging_pixal3d_test.go | 37 ++ docs/content/features/3d-generation.md | 81 +++- docs/content/features/backends.md | 2 +- gallery/backends/index.yaml | 372 ++++++++++++++++++ pkg/pixal3d/inputs.go | 53 +++ pkg/pixal3d/inputs_test.go | 62 +++ scripts/lib/backend-filter.mjs | 5 +- scripts/lib/backend-filter_test.mjs | 6 + 36 files changed, 1864 insertions(+), 23 deletions(-) create mode 100644 backend/go/pixal3dcpp/Makefile create mode 100644 backend/go/pixal3dcpp/main.go create mode 100755 backend/go/pixal3dcpp/package.sh create mode 100644 backend/go/pixal3dcpp/pixal3d.go create mode 100644 backend/go/pixal3dcpp/pixal3d_test.go create mode 100755 backend/go/pixal3dcpp/run.sh create mode 100644 core/config/pixal3d_test.go create mode 100644 core/gallery/importers/pixal3d.go create mode 100644 core/gallery/importers/pixal3d_LICENSE create mode 100644 core/gallery/importers/pixal3d_models.json create mode 100644 core/gallery/importers/pixal3d_test.go create mode 100644 core/http/endpoints/localai/model3d_pixal3d_internal_test.go create mode 100644 core/services/nodes/file_staging_pixal3d_test.go create mode 100644 pkg/pixal3d/inputs.go create mode 100644 pkg/pixal3d/inputs_test.go diff --git a/.github/backend-matrix.yml b/.github/backend-matrix.yml index 293bc6978444..186790bc7c5b 100644 --- a/.github/backend-matrix.yml +++ b/.github/backend-matrix.yml @@ -808,6 +808,19 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2404' + - build-type: 'cublas' + cuda-major-version: "12" + cuda-minor-version: "8" + platforms: 'linux/amd64' + tag-latest: 'auto' + tag-suffix: '-gpu-nvidia-cuda-12-pixal3dcpp' + runs-on: 'ubuntu-latest' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "pixal3dcpp" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' - build-type: 'cublas' cuda-major-version: "12" cuda-minor-version: "8" @@ -1820,6 +1833,19 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2404' + - build-type: 'cublas' + cuda-major-version: "13" + cuda-minor-version: "0" + platforms: 'linux/amd64' + tag-latest: 'auto' + tag-suffix: '-gpu-nvidia-cuda-13-pixal3dcpp' + runs-on: 'ubuntu-latest' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "pixal3dcpp" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' - build-type: 'cublas' cuda-major-version: "13" cuda-minor-version: "0" @@ -3688,6 +3714,20 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2404' + - build-type: '' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/amd64' + platform-tag: 'amd64' + tag-latest: 'auto' + tag-suffix: '-cpu-pixal3dcpp' + runs-on: 'ubuntu-latest' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "pixal3dcpp" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' - build-type: '' cuda-major-version: "" cuda-minor-version: "" @@ -3702,6 +3742,20 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2404' + - build-type: '' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/arm64' + platform-tag: 'arm64' + tag-latest: 'auto' + tag-suffix: '-cpu-pixal3dcpp' + runs-on: 'ubuntu-24.04-arm' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "pixal3dcpp" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' # sam3-cpp - build-type: '' cuda-major-version: "" @@ -4054,6 +4108,20 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2404' + - build-type: 'vulkan' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/amd64' + platform-tag: 'amd64' + tag-latest: 'auto' + tag-suffix: '-gpu-vulkan-pixal3dcpp' + runs-on: 'ubuntu-latest' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "pixal3dcpp" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' - build-type: 'vulkan' cuda-major-version: "" cuda-minor-version: "" @@ -4068,6 +4136,20 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2404' + - build-type: 'vulkan' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/arm64' + platform-tag: 'arm64' + tag-latest: 'auto' + tag-suffix: '-gpu-vulkan-pixal3dcpp' + runs-on: 'ubuntu-24.04-arm' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "pixal3dcpp" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' - build-type: 'cublas' cuda-major-version: "12" cuda-minor-version: "0" @@ -6676,6 +6758,10 @@ includeDarwin: tag-suffix: "-metal-darwin-arm64-trellis2cpp" build-type: "metal" lang: "go" + - backend: "pixal3dcpp" + tag-suffix: "-metal-darwin-arm64-pixal3dcpp" + build-type: "metal" + lang: "go" - backend: "whisper" tag-suffix: "-metal-darwin-arm64-whisper" build-type: "metal" diff --git a/.github/workflows/bump_deps.yaml b/.github/workflows/bump_deps.yaml index de3a8b74d515..0875f2432c68 100644 --- a/.github/workflows/bump_deps.yaml +++ b/.github/workflows/bump_deps.yaml @@ -86,6 +86,10 @@ jobs: variable: "STABLEDIFFUSION_GGML_VERSION" branch: "master" file: "backend/go/stablediffusion-ggml/Makefile" + - repository: "raven38/pixal3d.cpp" + variable: "PIXAL3D_VERSION" + branch: "main" + file: "backend/go/pixal3dcpp/Makefile" - repository: "localai-org/trellis2cpp" variable: "TRELLIS2CPP_VERSION" branch: "pbr-textures" diff --git a/Makefile b/Makefile index 8739e1dac11a..873caa946022 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Disable parallel execution for backend builds -.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/bonsai backends/outetts backends/piper backends/stablediffusion-ggml backends/trellis2cpp backends/trellis2cpp-darwin backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/nemo-speech-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/valkey-store backends/qdrant-store backends/cloud-proxy backends/localai-proxy backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/mlx-video backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/moss-tts-cpp backends/magpie-tts-cpp backends/vllm-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin backends/audio-cpp backends/audio-cpp-darwin +.NOTPARALLEL: backends/pixal3dcpp backends/pixal3dcpp-darwin backends/diffusers backends/llama-cpp backends/turboquant backends/bonsai backends/outetts backends/piper backends/stablediffusion-ggml backends/trellis2cpp backends/trellis2cpp-darwin backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/nemo-speech-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/valkey-store backends/qdrant-store backends/cloud-proxy backends/localai-proxy backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/mlx-video backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/moss-tts-cpp backends/magpie-tts-cpp backends/vllm-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin backends/audio-cpp backends/audio-cpp-darwin .NOTPARALLEL: backends/whisper-medusa .NOTPARALLEL: backends/funasr @@ -659,6 +659,7 @@ prepare-test-extra: protogen-python $(MAKE) -C backend/go/rfdetr-cpp $(MAKE) -C backend/go/locate-anything-cpp $(MAKE) -C backend/go/trellis2cpp + $(MAKE) -C backend/go/pixal3dcpp $(MAKE) -C backend/go/kimodocpp $(MAKE) -C backend/go/gemxcpp $(MAKE) -C backend/go/valkey-store @@ -700,6 +701,7 @@ test-extra: prepare-test-extra $(MAKE) -C backend/go/vllm-cpp test $(MAKE) -C backend/go/nemo-speech-cpp test $(MAKE) -C backend/go/trellis2cpp test + $(MAKE) -C backend/go/pixal3dcpp test $(MAKE) -C backend/go/kimodocpp test $(MAKE) -C backend/go/gemxcpp test $(MAKE) -C backend/go/valkey-store test @@ -1359,6 +1361,7 @@ BACKEND_LOCALAI_PROXY = localai-proxy|golang|.|false|true BACKEND_HUGGINGFACE = huggingface|golang|.|false|true BACKEND_SILERO_VAD = silero-vad|golang|.|false|true BACKEND_STABLEDIFFUSION_GGML = stablediffusion-ggml|golang|.|--progress=plain|true +BACKEND_PIXAL3DCPP = pixal3dcpp|golang|.|--progress=plain|true BACKEND_TRELLIS2CPP = trellis2cpp|golang|.|--progress=plain|true BACKEND_GEMXCPP = gemxcpp|golang|.|--progress=plain|true BACKEND_KIMODOCPP = kimodocpp|golang|.|--progress=plain|true @@ -1469,10 +1472,11 @@ $(eval $(call generate-docker-build-target,$(BACKEND_HUGGINGFACE))) $(eval $(call generate-docker-build-target,$(BACKEND_SILERO_VAD))) $(eval $(call generate-docker-build-target,$(BACKEND_STABLEDIFFUSION_GGML))) $(eval $(call generate-docker-build-target,$(BACKEND_TRELLIS2CPP))) +$(eval $(call generate-docker-build-target,$(BACKEND_PIXAL3DCPP))) $(eval $(call generate-docker-build-target,$(BACKEND_KIMODOCPP))) $(eval $(call generate-docker-build-target,$(BACKEND_GEMXCPP))) .NOTPARALLEL: backends/kimodocpp backends/kimodocpp-darwin -docker-build-backends: docker-build-kimodocpp +docker-build-backends: docker-build-pixal3dcpp docker-build-kimodocpp backends/kimodocpp-darwin: BACKEND=kimodocpp BUILD_TYPE=cpu $(MAKE) build-darwin-go-backend @@ -1743,3 +1747,6 @@ build-launcher-appimage: test-launcher-appimage: python3 -m unittest discover -s contrib/appimage -p 'test_*.py' -v +backends/pixal3dcpp-darwin: + BACKEND=pixal3dcpp BUILD_TYPE=metal $(MAKE) build-darwin-go-backend + ./local-ai backends install "ocifile://$(abspath ./backend-images/pixal3dcpp.tar)" diff --git a/README.md b/README.md index 7548f2c4fa83..5053d4b60f92 100644 --- a/README.md +++ b/README.md @@ -235,6 +235,10 @@ LocalAI supports **60+ backends** including llama.cpp, vLLM, SGLang, transformer See the full [Backend & Model Compatibility Table](https://localai.io/model-compatibility/) and [GPU Acceleration guide](https://localai.io/features/gpu-acceleration/). +| Additional native backend | What it does | +|---------------------------|--------------| +| [pixal3d.cpp](https://github.com/raven38/pixal3d.cpp) (`pixal3dcpp`) | Four canonical RGBA views to a textured 3D GLB, with an explicit mesh scale. [Setup and API](https://localai.io/features/3d-generation/) | + ### Backends built by us Most backends wrap a best-in-class upstream engine. A handful of them are native C/C++/GGML engines (no Python at inference) developed and maintained by the LocalAI project itself: diff --git a/addons/studio/ui/src/pages/ThreeDGen.jsx b/addons/studio/ui/src/pages/ThreeDGen.jsx index 7c1336588c16..f786f3a6dcd7 100644 --- a/addons/studio/ui/src/pages/ThreeDGen.jsx +++ b/addons/studio/ui/src/pages/ThreeDGen.jsx @@ -77,6 +77,9 @@ export default function ThreeDGen() { const modelNames = useMemo(() => models.filter(item => item.capabilities?.some(cap => cap === CAP_3D || cap === CAP_3D_ANIMATION)).map(item => item.id), [models]) const selectedModel = models.find(item => item.id === model) const animation = selectedModel?.three_d_operations?.find(operation => operation.id === 'animate') + const multiview = selectedModel?.three_d_operations?.some(operation => operation.id === 'generate_multiview') + const [views, setViews] = useState({}) + const [meshScale, setMeshScale] = useState('') const [animationInputs, setAnimationInputs] = useState({}) const [animationParams, setAnimationParams] = useState({}) const [requestEndpoint, setRequestEndpoint] = useState('/3d/generations') @@ -125,6 +128,8 @@ export default function ThreeDGen() { setAnimationInputs({}) setAnimationParams({}) setImage(null) + setViews({}) + setMeshScale('') setQuality('auto') setBackground('auto') setSteps('') @@ -144,7 +149,10 @@ export default function ThreeDGen() { } const run = async () => { - if (!animation && !image?.base64) { addToast(t('threed.toasts.noImage'), 'warning'); return } + if (multiview && (['front', 'right', 'back', 'left'].some(name => !views[name]?.base64) || !Number.isFinite(Number(meshScale)) || Number(meshScale) <= 0)) { + addToast('Provide all four RGBA PNG views and a positive mesh scale.', 'warning'); return + } + if (!animation && !multiview && !image?.base64) { addToast(t('threed.toasts.noImage'), 'warning'); return } if (!model) { addToast(t('threed.toasts.noModel'), 'warning'); return } setLoading(true) @@ -156,8 +164,9 @@ export default function ThreeDGen() { const params = Object.fromEntries(Object.entries(animationParams).filter(([key, value]) => value !== '' && animation?.parameters.some(parameter => parameter.name === key))) const body = animation ? { model, inputs: animationInputs, params, response_format: 'url' } + : multiview ? { model, images: ['front', 'right', 'back', 'left'].map(name => views[name].base64), mesh_scale: Number(meshScale), quality: '1024', response_format: 'url' } : { model, image: image.base64, quality, background, response_format: 'url' } - if (!animation) { + if (!animation && !multiview) { if (steps) body.step = parseInt(steps) if (textureSteps) body.texture_steps = parseInt(textureSteps) if (guidance) body.cfg_scale = parseFloat(guidance) @@ -166,7 +175,7 @@ export default function ThreeDGen() { // RequestPanel renders and copies its body. Keeping a multi-megabyte image // there duplicates the upload in React and can starve the result render. - setLastRequest(animation ? body : { ...body, image: `` }) + setLastRequest(animation ? body : multiview ? { ...body, images: ['front', 'right', 'back', 'left'].map(name => `<${name} RGBA PNG omitted>`) } : { ...body, image: `` }) setRequestEndpoint(animation ? '/3d/animate' : '/3d/generations') try { @@ -186,7 +195,7 @@ export default function ThreeDGen() { const inputThumb = !animation && image?.dataUrl ? await makeThumb(image.dataUrl) : null const saved = await addEntry({ model, - params: animation ? params : { quality, background, steps, textureSteps, guidance, seed }, + params: animation ? params : multiview ? { mesh_scale: Number(meshScale), quality: '1024' } : { quality, background, steps, textureSteps, guidance, seed }, inputs: animation ? animationInputs : undefined, operation: animation ? 'animate' : 'generate', outputType, @@ -233,7 +242,7 @@ export default function ThreeDGen() { const installed = ws.installed.byType.threed const noModel = !ws.installed.loading && !ws.installed.error && installed.length === 0 const why = noModel ? t('studio.composer.whyModel', { type: t('studio.tabs.threed') }) - : (!animation && !image?.base64) ? t('studio.workspace.threed.whyImage') : '' + : (!animation && !multiview && !image?.base64) ? t('studio.workspace.threed.whyImage') : '' const f = (key) => t(`studio.workspace.fields.${key}`) const advancedSet = [steps && `${t('threed.labels.steps')} ${steps}`, textureSteps && `${t('threed.labels.textureSteps')} ${textureSteps}`, guidance && `${f('cfg')} ${guidance}`, seed && `${f('seed')} ${seed}`].filter(Boolean) const qualityOptions = QUALITIES.map(q => ({ value: q, label: t(`threed.labels.quality_${q}`) })) @@ -252,12 +261,12 @@ export default function ThreeDGen() { noModel={noModel ? { type: 'threed', label: t('studio.tabs.threed'), onChanged: ws.installed.refetch } : null} options={<> - {!animation && <> + {!animation && !multiview && <> } } - fold={animation ? null : ( + fold={animation || multiview ? null : ( )} - submit={{ label: t('threed.actions.generate'), busyLabel: t('threed.actions.generating'), busy: loading, disabled: !model || noModel || (!animation && !image?.base64), why, icon: 'cube' }} + submit={{ label: t('threed.actions.generate'), busyLabel: t('threed.actions.generating'), busy: loading, disabled: !model || noModel || (!animation && !multiview && !image?.base64), why, icon: 'cube' }} > {animation ? (
+ ) : multiview ? ( + <> +

Upload four pre-matted RGBA PNGs of the same object in the canonical turntable framing. Resolution: 1024.

+ {['front', 'right', 'back', 'left'].map(name => ( + setViews(current => ({ ...current, [name]: value }))} + onError={err => addToast(err.message, 'error')} maxBytes={MAX_3D_INPUT_BYTES} idPrefix={`threed-${name}`} /> + ))} +
+ + setMeshScale(e.target.value)} /> +

Enter the positive scale used to frame these views. Pixal3D does not estimate it.

+
+ ) : (
\/|^[[:space:]]*\// {for (i=1;i<=NF;i++) if ($i ~ /^\//) print $i}' | while read -r library; do + case "$library" in + */libcuda.so*|*/libnvidia-*) continue ;; + esac + cp -Lf "$library" "$CURDIR/package/lib/" + done + for loader in /lib64/ld-linux-x86-64.so.2 /lib/ld-linux-aarch64.so.1; do + if [ -f "$loader" ]; then cp -Lf "$loader" "$CURDIR/package/lib/ld.so"; break; fi + done +fi +source "$CURDIR/../../../scripts/build/package-gpu-libs.sh" "$CURDIR/package/lib" +package_gpu_libs diff --git a/backend/go/pixal3dcpp/pixal3d.go b/backend/go/pixal3dcpp/pixal3d.go new file mode 100644 index 000000000000..a9a83eba5e5b --- /dev/null +++ b/backend/go/pixal3dcpp/pixal3d.go @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: MIT +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strconv" + "sync" + + "github.com/mudler/LocalAI/pkg/grpc/base" + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + "github.com/mudler/LocalAI/pkg/pixal3d" +) + +var modelFiles = []string{"dinov3.gguf", "pixal3d_naf.gguf", "pixal3d_ss_flow_mv.gguf", "ss_dec.gguf", "pixal3d_shape_flow_512_mv.gguf", "shape_dec.gguf", "pixal3d_shape_flow_1024_mv.gguf", "pixal3d_tex_flow_1024_mv.gguf", "tex_dec.gguf"} + +// Pixal3D registers a validated model directory in the normal backend lifecycle. +// This adapter invokes the native CLI, so each request reloads +// weights and releases native memory when the child exits. +type Pixal3D struct { + base.Base + mu sync.Mutex + models string + cli string + cancel context.CancelFunc +} + +func validateModels(dir string) error { + f, err := os.Open(filepath.Join(dir, "pixal3d-models.json")) + if err != nil { + return fmt.Errorf("Pixal3D model manifest: %w", err) + } + var manifest struct { + Schema int `json:"schema_version"` + Family string `json:"model_family"` + Files []struct { + Name string `json:"name"` + Size int64 `json:"size_bytes"` + } `json:"files"` + } + err = json.NewDecoder(io.LimitReader(f, 1<<20)).Decode(&manifest) + f.Close() + if err != nil || manifest.Schema != 1 || (manifest.Family != "" && manifest.Family != "mv") { + return fmt.Errorf("invalid Pixal3D MV model manifest") + } + sizes := map[string]int64{} + for _, file := range manifest.Files { + if _, exists := sizes[file.Name]; exists { + return fmt.Errorf("duplicate manifest component %s", file.Name) + } + sizes[file.Name] = file.Size + } + for _, name := range modelFiles { + f, err := os.Open(filepath.Join(dir, name)) + if err != nil { + return fmt.Errorf("not a Pixal3D MV model set: %s: %w", name, err) + } + stat, err := f.Stat() + var magic [4]byte + _, readErr := io.ReadFull(f, magic[:]) + f.Close() + if err != nil || !stat.Mode().IsRegular() || readErr != nil || string(magic[:]) != "GGUF" || sizes[name] <= 0 || stat.Size() != sizes[name] { + return fmt.Errorf("invalid Pixal3D component %s (check manifest and file size)", name) + } + } + return nil +} + +func (p *Pixal3D) Load(opts *pb.ModelOptions) error { + p.mu.Lock() + defer p.mu.Unlock() + if p.cancel != nil { + return fmt.Errorf("cannot load Pixal3D while generation is running") + } + dir := opts.ModelFile + if !filepath.IsAbs(dir) { + dir = filepath.Join(opts.ModelPath, dir) + } + if err := validateModels(dir); err != nil { + return err + } + exe, err := os.Executable() + if err != nil { + return err + } + cli := filepath.Join(filepath.Dir(exe), "trellis-cli") + if st, err := os.Stat(cli); err != nil || st.IsDir() { + return fmt.Errorf("packaged Pixal3D trellis-cli is missing") + } + p.models, p.cli = dir, cli + return nil +} + +func (p *Pixal3D) Free() error { + p.mu.Lock() + defer p.mu.Unlock() + if p.cancel != nil { + p.cancel() + } + p.models = "" + return nil +} + +func commandArgs(models, views, output string, req *pb.Generate3DRequest) ([]string, error) { + if err := pixal3d.ValidateRequest(req.Src, req.Images, req.MeshScale, req.Quality, req.Background); err != nil { + return nil, err + } + if req.Step != 0 || req.CfgScale != 0 || req.TextureSteps != 0 || req.Seed != 0 || len(req.Params) != 0 { + return nil, fmt.Errorf("Pixal3D sampling overrides and params are unsupported") + } + if req.Dst == "" { + return nil, fmt.Errorf("output path is required") + } + return []string{"--views", views, "--models", models, "--res", "1024", "--mesh-scale", strconv.FormatFloat(req.MeshScale, 'g', -1, 64), "--output", output}, nil +} + +func (p *Pixal3D) Generate3D(req *pb.Generate3DRequest) error { + p.mu.Lock() + if p.models == "" { + p.mu.Unlock() + return fmt.Errorf("Pixal3D model is not loaded") + } + if p.cancel != nil { + p.mu.Unlock() + return fmt.Errorf("Pixal3D generation already running") + } + models, cli := p.models, p.cli + ctx, cancel := context.WithCancel(context.Background()) + p.cancel = cancel + p.mu.Unlock() + defer func() { cancel(); p.mu.Lock(); p.cancel = nil; p.mu.Unlock() }() + dir, err := os.MkdirTemp("", "pixal3d-views-") + if err != nil { + return err + } + defer os.RemoveAll(dir) + // The native CLI writes texture and mesh sidecars beside its GLB. + outputDir, err := os.MkdirTemp("", "pixal3d-output-") + if err != nil { + return err + } + defer os.RemoveAll(outputDir) + output := filepath.Join(outputDir, "output.glb") + args, err := commandArgs(models, dir, output, req) + if err != nil { + return err + } + names := []string{"0_front.png", "1_right.png", "2_back.png", "3_left.png"} + for i, path := range req.Images { + f, err := os.Open(path) + if err != nil { + return err + } + data, err := io.ReadAll(io.LimitReader(f, pixal3d.MaxInputBytes+1)) + f.Close() + if err != nil { + return err + } + if err = pixal3d.ValidatePNG(data); err != nil { + return fmt.Errorf("view %d: %w", i, err) + } + if err = os.WriteFile(filepath.Join(dir, names[i]), data, 0600); err != nil { + return err + } + } + loader := filepath.Join(filepath.Dir(cli), "lib", "ld.so") + if _, err := os.Stat(loader); err == nil { + args = append([]string{cli}, args...) + cli = loader + } + cmd := exec.CommandContext(ctx, cli, args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("Pixal3D generation failed: %w", err) + } + f, err := os.Open(output) + if err != nil { + return err + } + defer f.Close() + var header [4]byte + if _, err := io.ReadFull(f, header[:]); err != nil || string(header[:]) != "glTF" { + return fmt.Errorf("Pixal3D did not produce a GLB") + } + if _, err := f.Seek(0, io.SeekStart); err != nil { + return err + } + // Copy because the destination may be on a different filesystem. + dst, err := os.Create(req.Dst) + if err != nil { + return err + } + _, copyErr := io.Copy(dst, f) + closeErr := dst.Close() + if copyErr != nil || closeErr != nil { + os.Remove(req.Dst) + if copyErr != nil { + return copyErr + } + return closeErr + } + return nil +} diff --git a/backend/go/pixal3dcpp/pixal3d_test.go b/backend/go/pixal3dcpp/pixal3d_test.go new file mode 100644 index 000000000000..5e529f6d5668 --- /dev/null +++ b/backend/go/pixal3dcpp/pixal3d_test.go @@ -0,0 +1,106 @@ +package main + +import ( + "bytes" + "fmt" + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "image" + "image/png" + "os" + "path/filepath" + "testing" +) + +func TestPixal3D(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "Pixal3D backend") } + +var _ = Describe("Pixal3D backend", func() { + It("rejects missing or TRELLIS-only weights", func() { + p := &Pixal3D{} + Expect(p.Load(&pb.ModelOptions{ModelFile: GinkgoT().TempDir()})).NotTo(Succeed()) + }) + It("uses the canonical multiview CLI without shell interpretation", func() { + req := &pb.Generate3DRequest{Images: []string{"a", "b", "c", "d"}, MeshScale: 0.8, Dst: "/tmp/out; touch bad.glb"} + args, err := commandArgs("/models", "/views", req.Dst, req) + Expect(err).NotTo(HaveOccurred()) + Expect(args).To(Equal([]string{"--views", "/views", "--models", "/models", "--res", "1024", "--mesh-scale", "0.8", "--output", req.Dst})) + req.Src = "single.png" + _, err = commandArgs("/models", "/views", req.Dst, req) + Expect(err).To(HaveOccurred()) + }) + It("requires every MV model component", func() { + dir := GinkgoT().TempDir() + for _, name := range modelFiles { + Expect(os.WriteFile(filepath.Join(dir, name), []byte("GGUF"), 0600)).To(Succeed()) + } + Expect(validateModels(dir)).NotTo(Succeed()) + manifest := `{"schema_version":1,"model_family":"mv","files":[` + for i, name := range modelFiles { + if i > 0 { + manifest += "," + } + manifest += fmt.Sprintf(`{"name":%q,"size_bytes":4}`, name) + } + manifest += "]}" + Expect(os.WriteFile(filepath.Join(dir, "pixal3d-models.json"), []byte(manifest), 0600)).To(Succeed()) + Expect(validateModels(dir)).To(Succeed()) + Expect(os.Remove(filepath.Join(dir, "pixal3d_naf.gguf"))).To(Succeed()) + Expect(validateModels(dir)).NotTo(Succeed()) + }) +}) + +var _ = Describe("Pixal3D native process", func() { + DescribeTable("checks the process status and GLB output, then removes all staged files", func(script string, wantError bool) { + dir := GinkgoT().TempDir() + var data bytes.Buffer + Expect(png.Encode(&data, image.NewNRGBA(image.Rect(0, 0, 2, 2)))).To(Succeed()) + views := []string{} + for i := range 4 { + path := filepath.Join(dir, fmt.Sprintf("view-%d.png", i)) + Expect(os.WriteFile(path, data.Bytes(), 0600)).To(Succeed()) + views = append(views, path) + } + cli := filepath.Join(dir, "trellis-cli") + // Native generation emits sidecars even if the process later fails. + script = `#!/bin/sh +printf '%s' "$2" > "$PIXAL_CAPTURE" +for arg do dst=$arg; done +printf '%s' "$dst" > "$PIXAL_CAPTURE.output" +printf texture > "${dst%.glb}_base.png" +printf mesh > "${dst%.glb}.ply" +` + script + GinkgoT().Setenv("PIXAL_CAPTURE", filepath.Join(dir, "capture")) + Expect(os.WriteFile(cli, []byte(script), 0700)).To(Succeed()) + p := &Pixal3D{models: dir, cli: cli} + destination := filepath.Join(dir, "destination") + Expect(os.Mkdir(destination, 0700)).To(Succeed()) + err := p.Generate3D(&pb.Generate3DRequest{Images: views, MeshScale: 1, Dst: filepath.Join(destination, "out.glb")}) + Expect(err != nil).To(Equal(wantError)) + staged, readErr := os.ReadFile(filepath.Join(dir, "capture")) + Expect(readErr).NotTo(HaveOccurred()) + _, statErr := os.Stat(string(staged)) + Expect(os.IsNotExist(statErr)).To(BeTrue()) + entries, readErr := os.ReadDir(destination) + Expect(readErr).NotTo(HaveOccurred()) + if wantError { + Expect(entries).To(BeEmpty()) + } else { + Expect(entries).To(HaveLen(1)) + Expect(entries[0].Name()).To(Equal("out.glb")) + output, readErr := os.ReadFile(filepath.Join(destination, "out.glb")) + Expect(readErr).NotTo(HaveOccurred()) + Expect(string(output)).To(Equal("glTFpayload")) + } + stagedOutput, readErr := os.ReadFile(filepath.Join(dir, "capture.output")) + Expect(readErr).NotTo(HaveOccurred()) + _, statErr = os.Stat(filepath.Dir(string(stagedOutput))) + Expect(os.IsNotExist(statErr)).To(BeTrue()) + Expect(p.Free()).To(Succeed()) + Expect(p.models).To(BeEmpty()) + }, + Entry("valid output", "test -f \"$2/0_front.png\" && test -f \"$2/3_left.png\" || exit 9\nprintf glTFpayload > \"$dst\"\n", false), + Entry("process failure", "printf glTFpartial > \"$dst\"\nexit 7\n", true), + Entry("bad output", "for arg do dst=$arg; done\nprintf nope > \"$dst\"\n", true), + ) +}) diff --git a/backend/go/pixal3dcpp/run.sh b/backend/go/pixal3dcpp/run.sh new file mode 100755 index 000000000000..73138c7d56e0 --- /dev/null +++ b/backend/go/pixal3dcpp/run.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -euo pipefail +CURDIR=$(cd "$(dirname "$0")" && pwd) +if [ "$(uname -s)" = Darwin ]; then + export DYLD_LIBRARY_PATH="$CURDIR/lib:${DYLD_LIBRARY_PATH:-}" +else + export LD_LIBRARY_PATH="$CURDIR/lib:${LD_LIBRARY_PATH:-}" +fi +exec "$CURDIR/pixal3dcpp" "$@" diff --git a/backend/index.yaml b/backend/index.yaml index 1bf606e65ea8..1d5f5d170e87 100644 --- a/backend/index.yaml +++ b/backend/index.yaml @@ -29389,3 +29389,285 @@ uri: quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-funasr mirrors: - localai/localai-backends:master-metal-darwin-arm64-funasr +- name: pixal3dcpp + alias: pixal3dcpp + license: mit + description: Pixal3D canonical four-view RGBA input to textured GLB via the native C++/ggml CLI + urls: + - https://github.com/raven38/pixal3d.cpp + tags: + - image-to-3d + - 3d-generation + - CPU + - GPU + - CUDA + - Vulkan + - Metal + capabilities: + default: cpu-pixal3dcpp + metal: metal-pixal3dcpp + nvidia: cuda12-pixal3dcpp + nvidia-cuda-12: cuda12-pixal3dcpp + nvidia-cuda-13: cuda13-pixal3dcpp + vulkan: vulkan-pixal3dcpp +- name: pixal3dcpp-development + alias: pixal3dcpp + license: mit + description: Pixal3D canonical four-view RGBA input to textured GLB via the native C++/ggml CLI + urls: + - https://github.com/raven38/pixal3d.cpp + tags: + - image-to-3d + - 3d-generation + - CPU + - GPU + - CUDA + - Vulkan + - Metal + capabilities: + default: cpu-pixal3dcpp-development + metal: metal-pixal3dcpp-development + nvidia: cuda12-pixal3dcpp-development + nvidia-cuda-12: cuda12-pixal3dcpp-development + nvidia-cuda-13: cuda13-pixal3dcpp-development + vulkan: vulkan-pixal3dcpp-development +- name: cpu-pixal3dcpp + alias: pixal3dcpp + license: mit + description: Pixal3D canonical four-view RGBA input to textured GLB via the native C++/ggml CLI + urls: + - https://github.com/raven38/pixal3d.cpp + tags: + - image-to-3d + - 3d-generation + - CPU + - GPU + - CUDA + - Vulkan + - Metal + capabilities: + default: cpu-pixal3dcpp + metal: metal-pixal3dcpp + nvidia: cuda12-pixal3dcpp + nvidia-cuda-12: cuda12-pixal3dcpp + nvidia-cuda-13: cuda13-pixal3dcpp + vulkan: vulkan-pixal3dcpp + uri: quay.io/go-skynet/local-ai-backends:latest-cpu-pixal3dcpp + mirrors: + - localai/localai-backends:latest-cpu-pixal3dcpp +- name: cpu-pixal3dcpp-development + alias: pixal3dcpp + license: mit + description: Pixal3D canonical four-view RGBA input to textured GLB via the native C++/ggml CLI + urls: + - https://github.com/raven38/pixal3d.cpp + tags: + - image-to-3d + - 3d-generation + - CPU + - GPU + - CUDA + - Vulkan + - Metal + capabilities: + default: cpu-pixal3dcpp + metal: metal-pixal3dcpp + nvidia: cuda12-pixal3dcpp + nvidia-cuda-12: cuda12-pixal3dcpp + nvidia-cuda-13: cuda13-pixal3dcpp + vulkan: vulkan-pixal3dcpp + uri: quay.io/go-skynet/local-ai-backends:master-cpu-pixal3dcpp + mirrors: + - localai/localai-backends:master-cpu-pixal3dcpp +- name: cuda12-pixal3dcpp + alias: pixal3dcpp + license: mit + description: Pixal3D canonical four-view RGBA input to textured GLB via the native C++/ggml CLI + urls: + - https://github.com/raven38/pixal3d.cpp + tags: + - image-to-3d + - 3d-generation + - CPU + - GPU + - CUDA + - Vulkan + - Metal + capabilities: + default: cpu-pixal3dcpp + metal: metal-pixal3dcpp + nvidia: cuda12-pixal3dcpp + nvidia-cuda-12: cuda12-pixal3dcpp + nvidia-cuda-13: cuda13-pixal3dcpp + vulkan: vulkan-pixal3dcpp + uri: quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-pixal3dcpp + mirrors: + - localai/localai-backends:latest-gpu-nvidia-cuda-12-pixal3dcpp +- name: cuda12-pixal3dcpp-development + alias: pixal3dcpp + license: mit + description: Pixal3D canonical four-view RGBA input to textured GLB via the native C++/ggml CLI + urls: + - https://github.com/raven38/pixal3d.cpp + tags: + - image-to-3d + - 3d-generation + - CPU + - GPU + - CUDA + - Vulkan + - Metal + capabilities: + default: cpu-pixal3dcpp + metal: metal-pixal3dcpp + nvidia: cuda12-pixal3dcpp + nvidia-cuda-12: cuda12-pixal3dcpp + nvidia-cuda-13: cuda13-pixal3dcpp + vulkan: vulkan-pixal3dcpp + uri: quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-12-pixal3dcpp + mirrors: + - localai/localai-backends:master-gpu-nvidia-cuda-12-pixal3dcpp +- name: cuda13-pixal3dcpp + alias: pixal3dcpp + license: mit + description: Pixal3D canonical four-view RGBA input to textured GLB via the native C++/ggml CLI + urls: + - https://github.com/raven38/pixal3d.cpp + tags: + - image-to-3d + - 3d-generation + - CPU + - GPU + - CUDA + - Vulkan + - Metal + capabilities: + default: cpu-pixal3dcpp + metal: metal-pixal3dcpp + nvidia: cuda12-pixal3dcpp + nvidia-cuda-12: cuda12-pixal3dcpp + nvidia-cuda-13: cuda13-pixal3dcpp + vulkan: vulkan-pixal3dcpp + uri: quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-13-pixal3dcpp + mirrors: + - localai/localai-backends:latest-gpu-nvidia-cuda-13-pixal3dcpp +- name: cuda13-pixal3dcpp-development + alias: pixal3dcpp + license: mit + description: Pixal3D canonical four-view RGBA input to textured GLB via the native C++/ggml CLI + urls: + - https://github.com/raven38/pixal3d.cpp + tags: + - image-to-3d + - 3d-generation + - CPU + - GPU + - CUDA + - Vulkan + - Metal + capabilities: + default: cpu-pixal3dcpp + metal: metal-pixal3dcpp + nvidia: cuda12-pixal3dcpp + nvidia-cuda-12: cuda12-pixal3dcpp + nvidia-cuda-13: cuda13-pixal3dcpp + vulkan: vulkan-pixal3dcpp + uri: quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-pixal3dcpp + mirrors: + - localai/localai-backends:master-gpu-nvidia-cuda-13-pixal3dcpp +- name: vulkan-pixal3dcpp + alias: pixal3dcpp + license: mit + description: Pixal3D canonical four-view RGBA input to textured GLB via the native C++/ggml CLI + urls: + - https://github.com/raven38/pixal3d.cpp + tags: + - image-to-3d + - 3d-generation + - CPU + - GPU + - CUDA + - Vulkan + - Metal + capabilities: + default: cpu-pixal3dcpp + metal: metal-pixal3dcpp + nvidia: cuda12-pixal3dcpp + nvidia-cuda-12: cuda12-pixal3dcpp + nvidia-cuda-13: cuda13-pixal3dcpp + vulkan: vulkan-pixal3dcpp + uri: quay.io/go-skynet/local-ai-backends:latest-gpu-vulkan-pixal3dcpp + mirrors: + - localai/localai-backends:latest-gpu-vulkan-pixal3dcpp +- name: vulkan-pixal3dcpp-development + alias: pixal3dcpp + license: mit + description: Pixal3D canonical four-view RGBA input to textured GLB via the native C++/ggml CLI + urls: + - https://github.com/raven38/pixal3d.cpp + tags: + - image-to-3d + - 3d-generation + - CPU + - GPU + - CUDA + - Vulkan + - Metal + capabilities: + default: cpu-pixal3dcpp + metal: metal-pixal3dcpp + nvidia: cuda12-pixal3dcpp + nvidia-cuda-12: cuda12-pixal3dcpp + nvidia-cuda-13: cuda13-pixal3dcpp + vulkan: vulkan-pixal3dcpp + uri: quay.io/go-skynet/local-ai-backends:master-gpu-vulkan-pixal3dcpp + mirrors: + - localai/localai-backends:master-gpu-vulkan-pixal3dcpp +- name: metal-pixal3dcpp + alias: pixal3dcpp + license: mit + description: Pixal3D canonical four-view RGBA input to textured GLB via the native C++/ggml CLI + urls: + - https://github.com/raven38/pixal3d.cpp + tags: + - image-to-3d + - 3d-generation + - CPU + - GPU + - CUDA + - Vulkan + - Metal + capabilities: + default: cpu-pixal3dcpp + metal: metal-pixal3dcpp + nvidia: cuda12-pixal3dcpp + nvidia-cuda-12: cuda12-pixal3dcpp + nvidia-cuda-13: cuda13-pixal3dcpp + vulkan: vulkan-pixal3dcpp + uri: quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-pixal3dcpp + mirrors: + - localai/localai-backends:latest-metal-darwin-arm64-pixal3dcpp +- name: metal-pixal3dcpp-development + alias: pixal3dcpp + license: mit + description: Pixal3D canonical four-view RGBA input to textured GLB via the native C++/ggml CLI + urls: + - https://github.com/raven38/pixal3d.cpp + tags: + - image-to-3d + - 3d-generation + - CPU + - GPU + - CUDA + - Vulkan + - Metal + capabilities: + default: cpu-pixal3dcpp + metal: metal-pixal3dcpp + nvidia: cuda12-pixal3dcpp + nvidia-cuda-12: cuda12-pixal3dcpp + nvidia-cuda-13: cuda13-pixal3dcpp + vulkan: vulkan-pixal3dcpp + uri: quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-pixal3dcpp + mirrors: + - localai/localai-backends:master-metal-darwin-arm64-pixal3dcpp diff --git a/core/backend/model3d.go b/core/backend/model3d.go index 7cccbc7db6c5..a7cee80ce0f7 100644 --- a/core/backend/model3d.go +++ b/core/backend/model3d.go @@ -15,6 +15,8 @@ import ( // this layer. type Model3DGenerationOptions struct { Image string + Images []string + MeshScale float64 Destination string Seed int32 Step int32 @@ -38,6 +40,8 @@ func Model3DGeneration(options Model3DGenerationOptions, loader *model.ModelLoad appConfig.Context, &proto.Generate3DRequest{ Src: options.Image, + Images: append([]string(nil), options.Images...), + MeshScale: options.MeshScale, Dst: options.Destination, Seed: options.Seed, Step: options.Step, diff --git a/core/config/backend_capabilities.go b/core/config/backend_capabilities.go index cfaa7cec1f1f..d2851a7a5275 100644 --- a/core/config/backend_capabilities.go +++ b/core/config/backend_capabilities.go @@ -455,6 +455,12 @@ var BackendCapabilities = map[string]BackendCapability{ DefaultUsecases: []string{Usecase3DAnimation}, Description: "kimodo.cpp — text-to-motion on CPU/Vulkan, exported as animated skeleton GLB", }, + "pixal3dcpp": { + GRPCMethods: []GRPCMethod{MethodGenerate3D}, + PossibleUsecases: []string{Usecase3D}, + DefaultUsecases: []string{Usecase3D}, + Description: "Pixal3D — four pre-matted RGBA views to textured 3D mesh (GLB)", + }, "trellis2cpp": { GRPCMethods: []GRPCMethod{MethodGenerate3D}, PossibleUsecases: []string{Usecase3D}, diff --git a/core/config/model_3d.go b/core/config/model_3d.go index 5cbcd909009f..6178a3761c55 100644 --- a/core/config/model_3d.go +++ b/core/config/model_3d.go @@ -34,6 +34,20 @@ func (c *ModelConfig) ThreeDOperations() []schema.ThreeDOperation { Inputs: []schema.ThreeDInput{{Name: "prompt", Type: "text", Label: "Motion prompt", Required: true, MaxBytes: 4096}}, Parameters: params, }} + case "pixal3dcpp": + if !c.HasUsecases(FLAG_3D) { + return nil + } + return []schema.ThreeDOperation{{ + ID: "generate_multiview", Label: "Generate from four views", Endpoint: "/3d/generations", Output: "mesh", + Inputs: []schema.ThreeDInput{ + {Name: "front", Type: "image", Label: "Front RGBA PNG", Required: true}, + {Name: "right", Type: "image", Label: "Right RGBA PNG", Required: true}, + {Name: "back", Type: "image", Label: "Back RGBA PNG", Required: true}, + {Name: "left", Type: "image", Label: "Left RGBA PNG", Required: true}, + }, + Parameters: []schema.ThreeDParameter{{Name: "mesh_scale", Label: "Mesh scale (required, positive)", Type: "number", Min: 0}}, + }} case "trellis2cpp": if !c.HasUsecases(FLAG_3D) { return nil diff --git a/core/config/model_config.go b/core/config/model_config.go index 3b56213ea147..315b689eb912 100644 --- a/core/config/model_config.go +++ b/core/config/model_config.go @@ -2646,7 +2646,7 @@ func (c *ModelConfig) GuessUsecases(u ModelConfigUsecase) bool { } if (u & FLAG_3D) == FLAG_3D { - threeDBackends := []string{"trellis2cpp"} + threeDBackends := []string{"trellis2cpp", "pixal3dcpp"} if !slices.Contains(threeDBackends, c.Backend) { return false } diff --git a/core/config/pixal3d_test.go b/core/config/pixal3d_test.go new file mode 100644 index 000000000000..038b018efc88 --- /dev/null +++ b/core/config/pixal3d_test.go @@ -0,0 +1,17 @@ +package config + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Pixal3D discovery", func() { + It("advertises four views and explicit mesh scale", func() { + c := &ModelConfig{Backend: "pixal3dcpp"} + Expect(c.HasUsecases(FLAG_3D)).To(BeTrue()) + ops := c.ThreeDOperations() + Expect(ops).To(HaveLen(1)) + Expect(ops[0].Inputs).To(HaveLen(4)) + Expect(ops[0].Parameters[0].Name).To(Equal("mesh_scale")) + }) +}) diff --git a/core/gallery/importers/importers.go b/core/gallery/importers/importers.go index ce1a14055c78..9efab36b3ad5 100644 --- a/core/gallery/importers/importers.go +++ b/core/gallery/importers/importers.go @@ -148,6 +148,7 @@ var defaultImporters = []Importer{ // run before LlamaCPPImporter so its GGUF sets aren't claimed by the // generic .gguf importer; matches only trellis-named URIs/repos or the // distinctive component filenames, so arbitrary GGUFs are never claimed. + &Pixal3DImporter{}, &Trellis2CppImporter{}, &GemXCppImporter{}, &KimodoCppImporter{}, diff --git a/core/gallery/importers/pixal3d.go b/core/gallery/importers/pixal3d.go new file mode 100644 index 000000000000..84641f8ba23e --- /dev/null +++ b/core/gallery/importers/pixal3d.go @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: MIT +package importers + +import ( + "crypto/sha256" + _ "embed" + "encoding/json" + "fmt" + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/gallery" + "github.com/mudler/LocalAI/core/schema" + "go.yaml.in/yaml/v2" +) + +// Inventory from raven38/pixal3d.cpp d1b4926452f9e09702b891db5f05acc845e153ed (MIT). +// +//go:embed pixal3d_models.json +var pixal3dManifest []byte + +type Pixal3DImporter struct{} + +func (*Pixal3DImporter) Name() string { return "pixal3dcpp" } +func (*Pixal3DImporter) Modality() string { return "3d" } +func (*Pixal3DImporter) AutoDetects() bool { return false } +func (*Pixal3DImporter) Match(d Details) bool { + var p struct { + Backend string `json:"backend"` + } + return json.Unmarshal(d.Preferences, &p) == nil && p.Backend == "pixal3dcpp" +} +func (*Pixal3DImporter) Import(d Details) (gallery.ModelConfig, error) { + const directory = "pixal3d-q8_0-v1" + owner, repo, ok := HFOwnerRepoFromURI(d.URI) + if !ok || owner != "raven38" || repo != directory { + return gallery.ModelConfig{}, fmt.Errorf("pixal3dcpp importer supports https://huggingface.co/raven38/pixal3d-q8_0-v1; configure other complete MV directories manually") + } + var p struct { + Name string `json:"name"` + } + if err := json.Unmarshal(d.Preferences, &p); err != nil { + return gallery.ModelConfig{}, err + } + if p.Name == "" { + p.Name = "pixal3d" + } + var manifest struct { + Files []struct { + Name string `json:"name"` + SHA256 string `json:"sha256"` + } `json:"files"` + } + if err := json.Unmarshal(pixal3dManifest, &manifest); err != nil { + return gallery.ModelConfig{}, err + } + result := gallery.ModelConfig{Name: p.Name, Description: "Pixal3D multiview Q8_0: four RGBA views to GLB"} + for _, file := range manifest.Files { + result.Files = append(result.Files, gallery.File{URI: "https://huggingface.co/raven38/" + directory + "/resolve/main/" + file.Name, Filename: directory + "/" + file.Name, SHA256: file.SHA256}) + } + result.Files = append(result.Files, gallery.File{URI: "https://raw.githubusercontent.com/raven38/pixal3d.cpp/d1b4926452f9e09702b891db5f05acc845e153ed/models/" + directory + "/pixal3d-models.json", Filename: directory + "/pixal3d-models.json", SHA256: fmt.Sprintf("%x", sha256.Sum256(pixal3dManifest))}) + cfg := config.ModelConfig{Name: p.Name, Backend: "pixal3dcpp", KnownUsecaseStrings: []string{"FLAG_3D"}, PredictionOptions: schema.PredictionOptions{BasicModelRequest: schema.BasicModelRequest{Model: directory}}} + data, err := yaml.Marshal(cfg) + if err != nil { + return gallery.ModelConfig{}, err + } + result.ConfigFile = string(data) + return result, nil +} diff --git a/core/gallery/importers/pixal3d_LICENSE b/core/gallery/importers/pixal3d_LICENSE new file mode 100644 index 000000000000..f8fd8bd2f507 --- /dev/null +++ b/core/gallery/importers/pixal3d_LICENSE @@ -0,0 +1,22 @@ +MIT License + +Copyright (c) 2026 Piotr Wilkin (trellis.cpp, the base of this fork) +Copyright (c) 2026 raven (pixal3d.cpp changes) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/core/gallery/importers/pixal3d_models.json b/core/gallery/importers/pixal3d_models.json new file mode 100644 index 000000000000..c3625bcaa0c7 --- /dev/null +++ b/core/gallery/importers/pixal3d_models.json @@ -0,0 +1,71 @@ +{ + "schema_version": 1, + "model_set": "pixal3d-q8_0", + "version": "v1", + "files": [ + { + "name": "dinov3.gguf", + "role": "image_encoder", + "required": true, + "size_bytes": 323657920, + "sha256": "0dd4ffd4b46a248f5b7d49c35275d68461fbf73f57ddb4c1fa8afb4f7bb45a0d" + }, + { + "name": "pixal3d_naf.gguf", + "role": "naf", + "required": true, + "size_bytes": 1334656, + "sha256": "c4f6cd80e94c8d120360ba50a3633322e6ecce0163966e3b7650b24f99c18569" + }, + { + "name": "pixal3d_ss_flow_mv.gguf", + "role": "ss_flow", + "required": true, + "size_bytes": 1426559744, + "sha256": "5211bb54a3f9ec77ab9451cb37031543f516bbb3cf592026210b69a2468adf4c" + }, + { + "name": "ss_dec.gguf", + "role": "ss_decoder", + "required": true, + "size_bytes": 147379392, + "sha256": "2790b5eecb261cc877d9bf175ce2bd6dd48cd65be8c042c5f5bc023dfca01cf7" + }, + { + "name": "pixal3d_shape_flow_512_mv.gguf", + "role": "shape_flow_512", + "required": true, + "size_bytes": 1476761760, + "sha256": "79a8ff77d0da194a8d6fc348914549aa0442d07a32de7a10f065f35d50116152" + }, + { + "name": "shape_dec.gguf", + "role": "shape_decoder", + "required": true, + "size_bytes": 881361568, + "sha256": "0de7c7a675022dd8696d526a9279e5a50b2a35c8a79452f434a72dc53d40f169" + }, + { + "name": "pixal3d_shape_flow_1024_mv.gguf", + "role": "shape_flow_1024", + "required": true, + "size_bytes": 1476761760, + "sha256": "50cd892247d73d4750b720d5612cef03cc0c02ec806b9e37d0d54d7ec579a94b" + }, + { + "name": "pixal3d_tex_flow_1024_mv.gguf", + "role": "texture_flow_1024", + "required": true, + "size_bytes": 1476813984, + "sha256": "e14a590dc6c77211bd591582159252ec473165b9401c4813df6e4f1a6bf69456" + }, + { + "name": "tex_dec.gguf", + "role": "texture_decoder", + "required": true, + "size_bytes": 881344576, + "sha256": "88b4fced46455e02f316664d5c43584a311921dd9a1cdc1b7b7d981cca9214d4" + } + ], + "source": "tools/quantize_gguf.py Q8_0 applied to every file of pixal3d-f16 v1; flow: TencentARC/Pixal3D ckpts/{ss_flow_img_dit_1_3B_64_bf16_mv,slat_flow_img2shape_dit_1_3B_{512,1024}_bf16_mv,slat_flow_imgshape2tex_dit_1_3B_1024_bf16_mv}.safetensors; decoders: ckpts/{shape_dec,tex_dec}_next_dc_f16c32_fp16.safetensors + ss_dec ckpt; image_encoder: timm/vit_large_patch16_dinov3.lvd1689m model.safetensors; naf: valeoai/NAF naf_release.pth converted to safetensors; all converted with tools/convert.py" +} diff --git a/core/gallery/importers/pixal3d_test.go b/core/gallery/importers/pixal3d_test.go new file mode 100644 index 000000000000..43597eb87fc6 --- /dev/null +++ b/core/gallery/importers/pixal3d_test.go @@ -0,0 +1,24 @@ +package importers + +import ( + "encoding/json" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Pixal3D importer", func() { + It("requires explicit selection and only imports the supported multiview bundle", func() { + imp := &Pixal3DImporter{} + details := Details{URI: "https://huggingface.co/raven38/pixal3d-q8_0-v1", Preferences: json.RawMessage(`{}`)} + Expect(imp.Match(details)).To(BeFalse()) + details.Preferences = json.RawMessage(`{"backend":"pixal3dcpp"}`) + Expect(imp.Match(details)).To(BeTrue()) + cfg, err := imp.Import(details) + Expect(err).NotTo(HaveOccurred()) + Expect(cfg.Files).To(HaveLen(10)) + Expect(cfg.ConfigFile).To(ContainSubstring("model: pixal3d-q8_0-v1")) + details.URI = "https://huggingface.co/microsoft/TRELLIS.2-4B" + _, err = imp.Import(details) + Expect(err).To(HaveOccurred()) + }) +}) diff --git a/core/http/endpoints/localai/model3d.go b/core/http/endpoints/localai/model3d.go index fd514c7db686..34137a401f4f 100644 --- a/core/http/endpoints/localai/model3d.go +++ b/core/http/endpoints/localai/model3d.go @@ -18,6 +18,7 @@ import ( "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/http/middleware" "github.com/mudler/LocalAI/core/schema" + "github.com/mudler/LocalAI/pkg/pixal3d" "github.com/mudler/xlog" @@ -53,7 +54,17 @@ func Model3DEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appCon return echo.ErrBadRequest } - if input.Image == "" { + isPixal := config.Backend == "pixal3dcpp" + if isPixal { + if err := pixal3d.ValidateRequest(input.Image, input.Images, input.MeshScale, input.Quality, input.Background); err != nil { + return echo.NewHTTPError(http.StatusBadRequest, err.Error()) + } + if input.Step != 0 || input.CFGScale != 0 || input.TextureSteps != 0 || input.Seed != 0 || len(input.Params) != 0 { + return echo.NewHTTPError(http.StatusBadRequest, "Pixal3D supports images, mesh_scale and quality 1024; sampling overrides and params are not supported") + } + } else if len(input.Images) != 0 || input.MeshScale != 0 { + return echo.NewHTTPError(http.StatusBadRequest, "images and mesh_scale require the pixal3dcpp backend") + } else if input.Image == "" { return echo.NewHTTPError(http.StatusBadRequest, "image is required: 3D generation is image-conditioned") } // Reject unknown enum values here rather than surfacing an opaque @@ -65,11 +76,38 @@ func Model3DEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appCon return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("invalid background %q: must be one of auto, keep, black, white", input.Background)) } - src, err := stageVideoMediaWithLimit(c.Request().Context(), appConfig.GeneratedContentDir, input.Image, max3DInputBytes) - if err != nil { - return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("invalid image: %v", err)) + var src string + var images []string + var err error + defer func() { + if src != "" { + _ = os.Remove(src) + } + for _, path := range images { + _ = os.Remove(path) + } + }() + if isPixal { + for i, view := range input.Images { + path, stageErr := stageVideoMediaWithLimit(c.Request().Context(), appConfig.GeneratedContentDir, view, max3DInputBytes) + if stageErr != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("invalid view %d: %v", i, stageErr)) + } + images = append(images, path) + data, readErr := os.ReadFile(path) + if readErr != nil { + return readErr + } + if err := pixal3d.ValidatePNG(data); err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("invalid view %d: %v", i, err)) + } + } + } else { + src, err = stageVideoMediaWithLimit(c.Request().Context(), appConfig.GeneratedContentDir, input.Image, max3DInputBytes) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("invalid image: %v", err)) + } } - defer func() { _ = os.Remove(src) }() xlog.Debug("Parameter Config", "config", config) @@ -78,11 +116,11 @@ func Model3DEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appCon } step := input.Step - if step == 0 && config.Step != 0 { + if !isPixal && step == 0 && config.Step != 0 { step = int32(config.Step) } cfgScale := input.CFGScale - if cfgScale == 0 && config.CFGScale != 0 { + if !isPixal && cfgScale == 0 && config.CFGScale != 0 { cfgScale = config.CFGScale } @@ -133,6 +171,8 @@ func Model3DEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appCon fn, err := backend.Model3DGeneration( backend.Model3DGenerationOptions{ Image: src, + Images: images, + MeshScale: input.MeshScale, Destination: output, Seed: input.Seed, Step: step, diff --git a/core/http/endpoints/localai/model3d_pixal3d_internal_test.go b/core/http/endpoints/localai/model3d_pixal3d_internal_test.go new file mode 100644 index 000000000000..fe3e0da83484 --- /dev/null +++ b/core/http/endpoints/localai/model3d_pixal3d_internal_test.go @@ -0,0 +1,97 @@ +package localai + +import ( + "bytes" + "context" + "encoding/base64" + "github.com/labstack/echo/v4" + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/http/middleware" + "github.com/mudler/LocalAI/core/schema" + grpcPkg "github.com/mudler/LocalAI/pkg/grpc" + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + "github.com/mudler/LocalAI/pkg/model" + "github.com/mudler/LocalAI/pkg/system" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + ggrpc "google.golang.org/grpc" + "image" + "image/png" + "net/http" + "net/http/httptest" + "os" +) + +type pixalEndpointBackend struct { + grpcPkg.Backend + seen *pb.Generate3DRequest +} + +func (*pixalEndpointBackend) HealthCheck(context.Context) (bool, error) { return true, nil } +func (*pixalEndpointBackend) IsBusy() bool { return false } +func (b *pixalEndpointBackend) Generate3D(_ context.Context, r *pb.Generate3DRequest, _ ...ggrpc.CallOption) (*pb.Result, error) { + b.seen = r + for _, path := range r.Images { + _, err := os.Stat(path) + Expect(err).NotTo(HaveOccurred()) + } + return &pb.Result{Success: true}, os.WriteFile(r.Dst, []byte("glTF fixture"), 0600) +} + +var _ = Describe("Pixal3D HTTP generation", func() { + It("forwards four staged images and scale, ignores TRELLIS config defaults, and cleans inputs", func() { + state := &system.SystemState{} + loader := model.NewModelLoader(state) + fixture := &pixalEndpointBackend{} + loader.SetModelRouter(func(_ context.Context, id string, _, _, _, _ string, _ *pb.ModelOptions, _ bool) (*model.Model, error) { + return model.NewModelWithClient(id, "test://pixal", fixture), nil + }) + cfg := &config.ModelConfig{Backend: "pixal3dcpp", Name: "pixal"} + cfg.SetDefaults() + cfg.Model = "pixal3d" + cfg.Step = 12 + cfg.CFGScale = 7.5 + app := config.NewApplicationConfig(config.WithSystemState(state)) + app.GeneratedContentDir = GinkgoT().TempDir() + var buf bytes.Buffer + Expect(png.Encode(&buf, image.NewNRGBA(image.Rect(0, 0, 2, 2)))).To(Succeed()) + data := base64.StdEncoding.EncodeToString(buf.Bytes()) + req := &schema.Model3DRequest{BasicModelRequest: schema.BasicModelRequest{Model: "pixal"}, Images: []string{data, data, data, data}, MeshScale: 0.8, ResponseFormat: "b64_json"} + e := echo.New() + rec := httptest.NewRecorder() + c := e.NewContext(httptest.NewRequest(http.MethodPost, "/3d/generations", nil), rec) + c.Set(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST, req) + c.Set(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG, cfg) + Expect(Model3DEndpoint(nil, loader, app)(c)).To(Succeed()) + Expect(rec.Code).To(Equal(http.StatusOK)) + Expect(fixture.seen.Images).To(HaveLen(4)) + Expect(fixture.seen.Src).To(BeEmpty()) + Expect(fixture.seen.MeshScale).To(Equal(0.8)) + Expect(fixture.seen.Step).To(BeZero()) + Expect(fixture.seen.CfgScale).To(BeZero()) + for _, path := range fixture.seen.Images { + _, err := os.Stat(path) + Expect(os.IsNotExist(err)).To(BeTrue()) + } + entries, err := os.ReadDir(app.GeneratedContentDir) + Expect(err).NotTo(HaveOccurred()) + Expect(entries).To(BeEmpty()) + }) + It("rejects bad views and cleans partially staged input before loading", func() { + var buf bytes.Buffer + Expect(png.Encode(&buf, image.NewNRGBA(image.Rect(0, 0, 2, 2)))).To(Succeed()) + data := base64.StdEncoding.EncodeToString(buf.Bytes()) + app := &config.ApplicationConfig{GeneratedContentDir: GinkgoT().TempDir()} + req := &schema.Model3DRequest{BasicModelRequest: schema.BasicModelRequest{Model: "pixal"}, Images: []string{data, "bad%%%", data, data}, MeshScale: 1} + e := echo.New() + c := e.NewContext(httptest.NewRequest(http.MethodPost, "/3d/generations", nil), httptest.NewRecorder()) + c.Set(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST, req) + c.Set(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG, &config.ModelConfig{Backend: "pixal3dcpp"}) + err := Model3DEndpoint(nil, nil, app)(c) + Expect(err).To(BeAssignableToTypeOf(&echo.HTTPError{})) + Expect(err.(*echo.HTTPError).Code).To(Equal(http.StatusBadRequest)) + entries, err := os.ReadDir(app.GeneratedContentDir) + Expect(err).NotTo(HaveOccurred()) + Expect(entries).To(BeEmpty()) + }) +}) diff --git a/core/http/react-ui/e2e/threed-gen.spec.js b/core/http/react-ui/e2e/threed-gen.spec.js index 9fdcbc92fd84..2df1cc3ffe9a 100644 --- a/core/http/react-ui/e2e/threed-gen.spec.js +++ b/core/http/react-ui/e2e/threed-gen.spec.js @@ -404,3 +404,33 @@ test.describe('3D generation', () => { }) }) }) + +test('Pixal3D requires four ordered views and a positive mesh scale', async ({ page }) => { + await page.route('**/api/models/capabilities', route => route.fulfill({ + contentType: 'application/json', + body: JSON.stringify({ data: [{ id: 'pixal-test', capabilities: ['FLAG_3D'], three_d_operations: [{ + id: 'generate_multiview', endpoint: '/3d/generations', output: 'mesh', + inputs: ['front', 'right', 'back', 'left'].map(name => ({ name, type: 'image', required: true })), + parameters: [{ name: 'mesh_scale', type: 'number' }], + }] }] }), + })) + let request + await mockGeneration(page, body => { request = body }) + await mockGlbDownload(page) + await page.goto('/app/3d') + await expect(page.getByRole('button', { name: 'pixal-test' })).toBeVisible() + await expect(page.locator('#threed-image-file')).toHaveCount(0) + await expect(page.getByLabel('Mesh scale')).toBeVisible() + for (const name of ['front', 'right', 'back', 'left']) { + await page.locator(`#threed-${name}-image-file`).setInputFiles({ name: `${name}.png`, mimeType: 'image/png', buffer: TINY_PNG }) + } + await page.getByLabel('Mesh scale').fill('0.8') + await page.locator('button[type="submit"]').click() + await expect(page.getByTestId('glb-download')).toBeVisible() + expect(request.images).toHaveLength(4) + expect(request.mesh_scale).toBe(0.8) + expect(request.quality).toBe('1024') + expect(request.image).toBeUndefined() + expect(request.background).toBeUndefined() + await expect(page.getByTestId('glb-remesh')).toHaveCount(0) +}) diff --git a/core/schema/localai.go b/core/schema/localai.go index 479cdfbc389c..43f363a3918d 100644 --- a/core/schema/localai.go +++ b/core/schema/localai.go @@ -84,7 +84,9 @@ type VideoRequest struct { // (TRELLIS.2 has no text-prompt path); the response is a binary glTF (.glb). type Model3DRequest struct { BasicModelRequest - Image string `json:"image" yaml:"image"` // conditioning image: URL, base64, or data URI (required) + Images []string `json:"images,omitempty" yaml:"images,omitempty"` // Pixal3D RGBA views: front, right, back, left + MeshScale float64 `json:"mesh_scale,omitempty" yaml:"mesh_scale,omitempty"` // required positive Pixal3D object scale + Image string `json:"image" yaml:"image"` // TRELLIS conditioning image: URL, base64, or data URI Seed int32 `json:"seed,omitempty" yaml:"seed,omitempty"` // random seed; <=0 picks a random seed Step int32 `json:"step,omitempty" yaml:"step,omitempty"` // flow sampling steps (backend default 12) CFGScale float32 `json:"cfg_scale,omitempty" yaml:"cfg_scale,omitempty"` // classifier-free guidance scale (backend default 7.5) diff --git a/core/services/nodes/file_staging_client.go b/core/services/nodes/file_staging_client.go index a80de9fa1091..f387a315f2b4 100644 --- a/core/services/nodes/file_staging_client.go +++ b/core/services/nodes/file_staging_client.go @@ -349,6 +349,16 @@ func (f *FileStagingClient) Generate3D(ctx context.Context, in *pb.Generate3DReq lifecycle := f.newStagedInputLifecycle() defer lifecycle.release() in = proto.Clone(in).(*pb.Generate3DRequest) + for i, path := range in.Images { + if path == "" || !isFilePath(path) { + return nil, fmt.Errorf("3D view %d must be a staged file", i) + } + backendPath, err := f.stageInputFile(ctx, lifecycle, path, "inputs") + if err != nil { + return nil, fmt.Errorf("staging 3D view %d: %w", i, err) + } + in.Images[i] = backendPath + } // Stage the conditioning image or existing GLB used by 3D post-processing. if in.Src != "" && isFilePath(in.Src) { diff --git a/core/services/nodes/file_staging_pixal3d_test.go b/core/services/nodes/file_staging_pixal3d_test.go new file mode 100644 index 000000000000..42696c26072b --- /dev/null +++ b/core/services/nodes/file_staging_pixal3d_test.go @@ -0,0 +1,37 @@ +package nodes + +import ( + "errors" + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Pixal3D distributed views", func() { + It("stages each view, preserves order and releases all inputs", func(ctx SpecContext) { + backend := &capturing3DBackend{} + stager := &lifecycleStager{} + client := NewFileStagingClient(backend, stager, "worker") + paths := []string{"/tmp/front.png", "/tmp/right.png", "/tmp/back.png", "/tmp/left.png"} + request := &pb.Generate3DRequest{Images: paths, MeshScale: 0.8} + _, err := client.Generate3D(ctx, request) + Expect(err).NotTo(HaveOccurred()) + Expect(request.Images).To(Equal(paths)) + Expect(stager.ensureCalls).To(HaveLen(4)) + Expect(stager.releasedKeys).To(HaveLen(4)) + for i, path := range backend.request.Images { + Expect(path).To(ContainSubstring("/remote/ephemeral/inputs/")) + Expect(path).To(HaveSuffix(paths[i][5:])) + } + Expect(backend.request.MeshScale).To(Equal(0.8)) + }) + It("releases partial uploads without invoking the backend", func(ctx SpecContext) { + backend := &capturing3DBackend{} + stager := &lifecycleStager{ensureErr: errors.New("offline"), ensureErrAt: 3} + client := NewFileStagingClient(backend, stager, "worker") + _, err := client.Generate3D(ctx, &pb.Generate3DRequest{Images: []string{"/tmp/a", "/tmp/b", "/tmp/c", "/tmp/d"}}) + Expect(err).To(MatchError(ContainSubstring("offline"))) + Expect(stager.releasedKeys).To(HaveLen(3)) + Expect(backend.request).To(BeNil()) + }) +}) diff --git a/docs/content/features/3d-generation.md b/docs/content/features/3d-generation.md index 8c71f37bc19c..9af3784e5a6b 100644 --- a/docs/content/features/3d-generation.md +++ b/docs/content/features/3d-generation.md @@ -5,7 +5,7 @@ weight = 19 url = "/features/3d-generation/" +++ -LocalAI can generate textured 3D meshes from a single conditioning image via the `/3d/generations` endpoint, powered by the `trellis2cpp` backend — a C++/GGML port of [Microsoft TRELLIS.2](https://github.com/microsoft/TRELLIS.2) ([trellis2.cpp](https://github.com/localai-org/trellis2cpp)). The output is a binary glTF (`.glb`) asset with PBR materials. +LocalAI supports single-image TRELLIS.2 and four-view Pixal3D generation. The TRELLIS.2 workflow below generates textured 3D meshes from a single conditioning image via the `/3d/generations` endpoint, powered by the `trellis2cpp` backend — a C++/GGML port of [Microsoft TRELLIS.2](https://github.com/microsoft/TRELLIS.2) ([trellis2.cpp](https://github.com/localai-org/trellis2cpp)). The output is a binary glTF (`.glb`) asset with PBR materials. Generation is image-conditioned only — there is no text-prompt path. Provide a photo or rendering of a single object (ideally on a plain background) and TRELLIS.2 reconstructs a full 3D mesh from it. @@ -122,3 +122,82 @@ The React UI includes a 3D tab in the Studio (and a `/3d` page) with an interact - The 512³ pipeline takes roughly two minutes on a modern GPU; the 1024³ cascade takes around five minutes and needs about 10 GB VRAM plus a temporary host-RAM spike. - `TRELLIS2_DEVICE=cpu` forces CPU inference (slow; mainly for debugging). - The generated mesh has unoriented winding (faithful to TRELLIS.2) and is exported Y-up with vertex-PBR materials; a UV-atlas texture bake can be enabled in the backend via the `T2GLB_XATLAS` environment variable. + +## Pixal3D: four views to a mesh + +The `pixal3dcpp` backend runs [raven38/pixal3d.cpp](https://github.com/raven38/pixal3d.cpp) in its explicit Pixal3D multiview mode. It produces a textured GLB with PNG textures at resolution 1024. + +Use four **8-bit RGBA PNGs**, ordered **front, right, back, left**. Prepare the alpha mattes before uploading. Each view can contain at most 32 MiB and 4096 × 4096 pixels. URLs, base64 strings, and data URIs use the same input staging as single-image generation; client-local file paths are not accepted. + +The server upload limit also applies to the complete JSON request. Set `--upload-limit` to accommodate all four images and their base64 overhead. + +These are canonical turntable views, not arbitrary camera photographs. Match the upstream canonical rig: 20° field of view, zero elevation, and camera distance 3.1192049980163574. Set `mesh_scale` to the finite positive object scale used for those views. The backend does not estimate cameras, create alpha mattes, or accept `transforms.json` uploads. + +### Install the model + +In **Import model**, select `pixal3dcpp` explicitly and enter: + +```text +https://huggingface.co/raven38/pixal3d-q8_0-v1 +``` + +The importer downloads the complete Q8_0 multiview bundle with SHA256 checks. It does not automatically claim other GGUF or TRELLIS repositories. + +For a manually installed bundle, place the following files in `models/pixal3d/`: + +```text +pixal3d-models.json +dinov3.gguf +pixal3d_naf.gguf +pixal3d_ss_flow_mv.gguf +ss_dec.gguf +pixal3d_shape_flow_512_mv.gguf +shape_dec.gguf +pixal3d_shape_flow_1024_mv.gguf +pixal3d_tex_flow_1024_mv.gguf +tex_dec.gguf +``` + +Use the matching upstream multiview manifest. Loading checks the manifest family, each required filename, the GGUF header, and the declared file size. Do not mix single-view, multiview, or TRELLIS components. + +Create `models/pixal3d.yaml`: + +```yaml +name: pixal3d +backend: pixal3dcpp +parameters: + model: pixal3d +``` + +The model path names the **directory**, so distributed workers receive the complete model set. LocalAI also stages all four request images on the selected worker and removes them after the request. + +### Generate + +Select the Pixal3D model on Studio's **3D** page. Upload the four labelled views and enter the mesh scale. The form offers only supported generation controls. + +The same request works through `POST /3d/generations`: + +```json +{ + "model": "pixal3d", + "images": [ + "data:image/png;base64,", + "data:image/png;base64,", + "data:image/png;base64,", + "data:image/png;base64," + ], + "mesh_scale": 0.8, + "quality": "1024", + "response_format": "url" +} +``` + +Replace the placeholders with the image data. `mesh_scale: 0.8` is an example, not an estimated default. `quality` can be omitted or set to `1024`; `background` can be omitted or set to `keep`. Both `url` and `b64_json` responses use the envelope described above. + +Pixal3D rejects `image`, sampling overrides, nonempty `params`, and other resolutions or background modes. Other backends reject the new `images` and `mesh_scale` fields. The existing TRELLIS single-image request remains unchanged. + +### Runtime and platforms + +The adapter registers and unloads models through LocalAI's normal backend lifecycle. It starts the native CLI for each generation, so each request reloads weights. Unloading cancels an active child process. This adapter does not expose upstream's persistent HTTP server or its single-view mode. + +Build definitions cover Linux amd64 and arm64 CPU/Vulkan, Linux amd64 CUDA 12/13, and Apple Silicon Metal. Some upstream operations use CPU fallbacks on Metal. CPU inference can be slow and requires substantial memory. Native builds and inference with real weights require platform validation; no throughput or memory guarantee is implied by the build definitions. diff --git a/docs/content/features/backends.md b/docs/content/features/backends.md index 8591b6250dd3..1abaf576b7cc 100644 --- a/docs/content/features/backends.md +++ b/docs/content/features/backends.md @@ -439,7 +439,7 @@ LocalAI supports various types of backends: - **Sound Generation Backends**: For music and audio generation (e.g., ACE-Step, [audio.cpp]({{%relref "features/audio-cpp" %}})) - **Sound Classification Backends**: For sound-event classification / audio tagging - identifying everyday sounds like baby cry, glass breaking, alarms (e.g., ced.cpp) - **Image & Video Generation Backends**: For diffusion and audio-conditioned avatar models (e.g., stable-diffusion.cpp, diffusers, vLLM-Omni, [MLX-Video on Apple Silicon]({{%relref "features/video-generation" %}}), [LongCat-Video]({{%relref "features/video-generation" %}}), [vllm.cpp / MiniMax-H3]({{%relref "features/video-generation" %}})) -- **3D Generation Backends**: For image-to-3D mesh generation ([trellis2.cpp]({{%relref "features/3d-generation" %}}) — Microsoft TRELLIS.2, producing GLB assets with PBR textures) +- **3D Generation Backends**: `trellis2cpp` converts a single image into a textured GLB. `pixal3dcpp` takes four canonical RGBA views. See [3D generation]({{%relref "features/3d-generation" %}}) for setup and requests. - **Vision & Detection Backends**: For object detection, segmentation, depth, and face/voice recognition (e.g., rf-detr.cpp, locate-anything.cpp, sam3.cpp, insightface) - **Audio Processing Backends**: For voice activity detection and audio enhancement (e.g., Silero VAD, LocalVQE, [audio.cpp]({{%relref "features/audio-cpp" %}})) - **Source Separation & Voice Conversion Backends**: For splitting a mix into named stems (vocals, drums, bass) and for converting speech or singing to a target voice (e.g., [audio.cpp]({{%relref "features/audio-cpp" %}})) diff --git a/gallery/backends/index.yaml b/gallery/backends/index.yaml index 383d88dd45b3..e9e10adda72e 100644 --- a/gallery/backends/index.yaml +++ b/gallery/backends/index.yaml @@ -43107,3 +43107,375 @@ urls: - https://github.com/modelscope/FunASR version: 0.0.0-development +- alias: pixal3dcpp + capabilities: + default: cpu-pixal3dcpp + metal: metal-pixal3dcpp + nvidia: cuda12-pixal3dcpp + nvidia-cuda-12: cuda12-pixal3dcpp + nvidia-cuda-13: cuda13-pixal3dcpp + vulkan: vulkan-pixal3dcpp + channel: latest + description: Pixal3D canonical four-view RGBA input to textured GLB via the native C++/ggml CLI + id: pixal3dcpp + kind: backend + license: mit + name: pixal3dcpp + schemaVersion: 1 + tags: + - image-to-3d + - 3d-generation + - CPU + - GPU + - CUDA + - Vulkan + - Metal + urls: + - https://github.com/raven38/pixal3d.cpp + version: 0.0.0-latest +- alias: pixal3dcpp + capabilities: + default: cpu-pixal3dcpp-development + metal: metal-pixal3dcpp-development + nvidia: cuda12-pixal3dcpp-development + nvidia-cuda-12: cuda12-pixal3dcpp-development + nvidia-cuda-13: cuda13-pixal3dcpp-development + vulkan: vulkan-pixal3dcpp-development + channel: development + description: Pixal3D canonical four-view RGBA input to textured GLB via the native C++/ggml CLI + id: pixal3dcpp-development + kind: backend + license: mit + name: pixal3dcpp-development + schemaVersion: 1 + tags: + - image-to-3d + - 3d-generation + - CPU + - GPU + - CUDA + - Vulkan + - Metal + urls: + - https://github.com/raven38/pixal3d.cpp + version: 0.0.0-development +- alias: pixal3dcpp + arch: amd64 + artifact: oci://quay.io/go-skynet/local-ai-backends:latest-cpu-pixal3dcpp + capabilities: + default: cpu-pixal3dcpp + metal: metal-pixal3dcpp + nvidia: cuda12-pixal3dcpp + nvidia-cuda-12: cuda12-pixal3dcpp + nvidia-cuda-13: cuda13-pixal3dcpp + vulkan: vulkan-pixal3dcpp + channel: latest + description: Pixal3D canonical four-view RGBA input to textured GLB via the native C++/ggml CLI + id: cpu-pixal3dcpp + kind: backend + license: mit + mirrors: + - oci://localai/localai-backends:latest-cpu-pixal3dcpp + mutable: true + name: cpu-pixal3dcpp + os: linux + schemaVersion: 1 + tags: + - image-to-3d + - 3d-generation + - CPU + - GPU + - CUDA + - Vulkan + - Metal + urls: + - https://github.com/raven38/pixal3d.cpp + version: 0.0.0-latest +- alias: pixal3dcpp + arch: amd64 + artifact: oci://quay.io/go-skynet/local-ai-backends:master-cpu-pixal3dcpp + capabilities: + default: cpu-pixal3dcpp + metal: metal-pixal3dcpp + nvidia: cuda12-pixal3dcpp + nvidia-cuda-12: cuda12-pixal3dcpp + nvidia-cuda-13: cuda13-pixal3dcpp + vulkan: vulkan-pixal3dcpp + channel: development + description: Pixal3D canonical four-view RGBA input to textured GLB via the native C++/ggml CLI + id: cpu-pixal3dcpp-development + kind: backend + license: mit + mirrors: + - oci://localai/localai-backends:master-cpu-pixal3dcpp + mutable: true + name: cpu-pixal3dcpp-development + os: linux + schemaVersion: 1 + tags: + - image-to-3d + - 3d-generation + - CPU + - GPU + - CUDA + - Vulkan + - Metal + urls: + - https://github.com/raven38/pixal3d.cpp + version: 0.0.0-development +- alias: pixal3dcpp + arch: amd64 + artifact: oci://quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-pixal3dcpp + capabilities: + default: cpu-pixal3dcpp + metal: metal-pixal3dcpp + nvidia: cuda12-pixal3dcpp + nvidia-cuda-12: cuda12-pixal3dcpp + nvidia-cuda-13: cuda13-pixal3dcpp + vulkan: vulkan-pixal3dcpp + channel: latest + description: Pixal3D canonical four-view RGBA input to textured GLB via the native C++/ggml CLI + id: cuda12-pixal3dcpp + kind: backend + license: mit + mirrors: + - oci://localai/localai-backends:latest-gpu-nvidia-cuda-12-pixal3dcpp + mutable: true + name: cuda12-pixal3dcpp + os: linux + schemaVersion: 1 + tags: + - image-to-3d + - 3d-generation + - CPU + - GPU + - CUDA + - Vulkan + - Metal + urls: + - https://github.com/raven38/pixal3d.cpp + version: 0.0.0-latest +- alias: pixal3dcpp + arch: amd64 + artifact: oci://quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-12-pixal3dcpp + capabilities: + default: cpu-pixal3dcpp + metal: metal-pixal3dcpp + nvidia: cuda12-pixal3dcpp + nvidia-cuda-12: cuda12-pixal3dcpp + nvidia-cuda-13: cuda13-pixal3dcpp + vulkan: vulkan-pixal3dcpp + channel: development + description: Pixal3D canonical four-view RGBA input to textured GLB via the native C++/ggml CLI + id: cuda12-pixal3dcpp-development + kind: backend + license: mit + mirrors: + - oci://localai/localai-backends:master-gpu-nvidia-cuda-12-pixal3dcpp + mutable: true + name: cuda12-pixal3dcpp-development + os: linux + schemaVersion: 1 + tags: + - image-to-3d + - 3d-generation + - CPU + - GPU + - CUDA + - Vulkan + - Metal + urls: + - https://github.com/raven38/pixal3d.cpp + version: 0.0.0-development +- alias: pixal3dcpp + arch: amd64 + artifact: oci://quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-13-pixal3dcpp + capabilities: + default: cpu-pixal3dcpp + metal: metal-pixal3dcpp + nvidia: cuda12-pixal3dcpp + nvidia-cuda-12: cuda12-pixal3dcpp + nvidia-cuda-13: cuda13-pixal3dcpp + vulkan: vulkan-pixal3dcpp + channel: latest + description: Pixal3D canonical four-view RGBA input to textured GLB via the native C++/ggml CLI + id: cuda13-pixal3dcpp + kind: backend + license: mit + mirrors: + - oci://localai/localai-backends:latest-gpu-nvidia-cuda-13-pixal3dcpp + mutable: true + name: cuda13-pixal3dcpp + os: linux + schemaVersion: 1 + tags: + - image-to-3d + - 3d-generation + - CPU + - GPU + - CUDA + - Vulkan + - Metal + urls: + - https://github.com/raven38/pixal3d.cpp + version: 0.0.0-latest +- alias: pixal3dcpp + arch: amd64 + artifact: oci://quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-pixal3dcpp + capabilities: + default: cpu-pixal3dcpp + metal: metal-pixal3dcpp + nvidia: cuda12-pixal3dcpp + nvidia-cuda-12: cuda12-pixal3dcpp + nvidia-cuda-13: cuda13-pixal3dcpp + vulkan: vulkan-pixal3dcpp + channel: development + description: Pixal3D canonical four-view RGBA input to textured GLB via the native C++/ggml CLI + id: cuda13-pixal3dcpp-development + kind: backend + license: mit + mirrors: + - oci://localai/localai-backends:master-gpu-nvidia-cuda-13-pixal3dcpp + mutable: true + name: cuda13-pixal3dcpp-development + os: linux + schemaVersion: 1 + tags: + - image-to-3d + - 3d-generation + - CPU + - GPU + - CUDA + - Vulkan + - Metal + urls: + - https://github.com/raven38/pixal3d.cpp + version: 0.0.0-development +- alias: pixal3dcpp + arch: amd64 + artifact: oci://quay.io/go-skynet/local-ai-backends:latest-gpu-vulkan-pixal3dcpp + capabilities: + default: cpu-pixal3dcpp + metal: metal-pixal3dcpp + nvidia: cuda12-pixal3dcpp + nvidia-cuda-12: cuda12-pixal3dcpp + nvidia-cuda-13: cuda13-pixal3dcpp + vulkan: vulkan-pixal3dcpp + channel: latest + description: Pixal3D canonical four-view RGBA input to textured GLB via the native C++/ggml CLI + id: vulkan-pixal3dcpp + kind: backend + license: mit + mirrors: + - oci://localai/localai-backends:latest-gpu-vulkan-pixal3dcpp + mutable: true + name: vulkan-pixal3dcpp + os: linux + schemaVersion: 1 + tags: + - image-to-3d + - 3d-generation + - CPU + - GPU + - CUDA + - Vulkan + - Metal + urls: + - https://github.com/raven38/pixal3d.cpp + version: 0.0.0-latest +- alias: pixal3dcpp + arch: amd64 + artifact: oci://quay.io/go-skynet/local-ai-backends:master-gpu-vulkan-pixal3dcpp + capabilities: + default: cpu-pixal3dcpp + metal: metal-pixal3dcpp + nvidia: cuda12-pixal3dcpp + nvidia-cuda-12: cuda12-pixal3dcpp + nvidia-cuda-13: cuda13-pixal3dcpp + vulkan: vulkan-pixal3dcpp + channel: development + description: Pixal3D canonical four-view RGBA input to textured GLB via the native C++/ggml CLI + id: vulkan-pixal3dcpp-development + kind: backend + license: mit + mirrors: + - oci://localai/localai-backends:master-gpu-vulkan-pixal3dcpp + mutable: true + name: vulkan-pixal3dcpp-development + os: linux + schemaVersion: 1 + tags: + - image-to-3d + - 3d-generation + - CPU + - GPU + - CUDA + - Vulkan + - Metal + urls: + - https://github.com/raven38/pixal3d.cpp + version: 0.0.0-development +- alias: pixal3dcpp + arch: arm64 + artifact: oci://quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-pixal3dcpp + capabilities: + default: cpu-pixal3dcpp + metal: metal-pixal3dcpp + nvidia: cuda12-pixal3dcpp + nvidia-cuda-12: cuda12-pixal3dcpp + nvidia-cuda-13: cuda13-pixal3dcpp + vulkan: vulkan-pixal3dcpp + channel: latest + description: Pixal3D canonical four-view RGBA input to textured GLB via the native C++/ggml CLI + id: metal-pixal3dcpp + kind: backend + license: mit + mirrors: + - oci://localai/localai-backends:latest-metal-darwin-arm64-pixal3dcpp + mutable: true + name: metal-pixal3dcpp + os: darwin + schemaVersion: 1 + tags: + - image-to-3d + - 3d-generation + - CPU + - GPU + - CUDA + - Vulkan + - Metal + urls: + - https://github.com/raven38/pixal3d.cpp + version: 0.0.0-latest +- alias: pixal3dcpp + arch: arm64 + artifact: oci://quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-pixal3dcpp + capabilities: + default: cpu-pixal3dcpp + metal: metal-pixal3dcpp + nvidia: cuda12-pixal3dcpp + nvidia-cuda-12: cuda12-pixal3dcpp + nvidia-cuda-13: cuda13-pixal3dcpp + vulkan: vulkan-pixal3dcpp + channel: development + description: Pixal3D canonical four-view RGBA input to textured GLB via the native C++/ggml CLI + id: metal-pixal3dcpp-development + kind: backend + license: mit + mirrors: + - oci://localai/localai-backends:master-metal-darwin-arm64-pixal3dcpp + mutable: true + name: metal-pixal3dcpp-development + os: darwin + schemaVersion: 1 + tags: + - image-to-3d + - 3d-generation + - CPU + - GPU + - CUDA + - Vulkan + - Metal + urls: + - https://github.com/raven38/pixal3d.cpp + version: 0.0.0-development diff --git a/pkg/pixal3d/inputs.go b/pkg/pixal3d/inputs.go new file mode 100644 index 000000000000..0636415541d3 --- /dev/null +++ b/pkg/pixal3d/inputs.go @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: MIT +// Package pixal3d defines the canonical multiview contract shared by HTTP and +// native backend entry points. +package pixal3d + +import ( + "bytes" + "fmt" + "image/png" + "math" + "strings" +) + +const MaxInputBytes = 32 << 20 + +func ValidateRequest(single string, views []string, scale float64, quality, background string) error { + if single != "" || len(views) != 4 { + return fmt.Errorf("Pixal3D requires exactly four images in front/right/back/left order and no image field") + } + for _, v := range views { + if strings.TrimSpace(v) == "" { + return fmt.Errorf("Pixal3D images must not be empty") + } + } + if scale <= 0 || math.IsNaN(scale) || math.IsInf(scale, 0) || float32(scale) == 0 || math.IsInf(float64(float32(scale)), 0) { + return fmt.Errorf("mesh_scale must be explicitly set to a finite positive number") + } + if quality != "" && quality != "1024" { + return fmt.Errorf("Pixal3D supports only quality 1024") + } + if background != "" && background != "keep" { + return fmt.Errorf("Pixal3D requires pre-matted RGBA PNG images; background must be keep") + } + return nil +} + +func ValidatePNG(data []byte) error { + // Checking IHDR before decoding bounds memory use even for compressed bombs. + if len(data) > MaxInputBytes || len(data) < 33 || !bytes.Equal(data[:8], []byte("\x89PNG\r\n\x1a\n")) || string(data[12:16]) != "IHDR" || data[24] != 8 || data[25] != 6 { + return fmt.Errorf("each Pixal3D view must be an 8-bit RGBA PNG of at most 32 MiB") + } + cfg, err := png.DecodeConfig(bytes.NewReader(data)) + if err != nil { + return fmt.Errorf("invalid PNG: %w", err) + } + if cfg.Width > 4096 || cfg.Height > 4096 { + return fmt.Errorf("Pixal3D views must be at most 4096 by 4096 pixels") + } + if _, err := png.Decode(bytes.NewReader(data)); err != nil { + return fmt.Errorf("invalid PNG: %w", err) + } + return nil +} diff --git a/pkg/pixal3d/inputs_test.go b/pkg/pixal3d/inputs_test.go new file mode 100644 index 000000000000..7af01fe709c5 --- /dev/null +++ b/pkg/pixal3d/inputs_test.go @@ -0,0 +1,62 @@ +package pixal3d + +import ( + "bytes" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "image" + "image/color" + "image/png" + "math" + "testing" +) + +func TestPixal3D(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "Pixal3D input contract") } + +var _ = Describe("canonical Pixal3D inputs", func() { + It("validates all required inputs before model loading", func() { + good := []string{"front", "right", "back", "left"} + cases := []struct { + name string + image string + views []string + scale float64 + quality, background string + bad bool + }{ + {"canonical", "", good, 1, "1024", "keep", false}, + {"defaults", "", good, 0.5, "", "", false}, + {"missing views", "", nil, 1, "", "", true}, + {"three views", "", good[:3], 1, "", "", true}, + {"ambiguous", "one", good, 1, "", "", true}, + {"empty view", "", []string{"front", "", "back", "left"}, 1, "", "", true}, + {"missing scale", "", good, 0, "", "", true}, + {"negative scale", "", good, -1, "", "", true}, + {"nan scale", "", good, math.NaN(), "", "", true}, + {"infinite scale", "", good, math.Inf(1), "", "", true}, + {"wrong resolution", "", good, 1, "512", "", true}, + {"remove background", "", good, 1, "", "auto", true}, + } + for _, c := range cases { + err := ValidateRequest(c.image, c.views, c.scale, c.quality, c.background) + Expect(err != nil).To(Equal(c.bad), c.name) + } + }) + It("requires decodable RGBA PNGs", func() { + for _, transparent := range []bool{false, true} { + img := image.NewNRGBA(image.Rect(0, 0, 2, 2)) + for y := 0; y < 2; y++ { + for x := 0; x < 2; x++ { + img.SetNRGBA(x, y, color.NRGBA{R: 255, A: 255}) + } + } + if transparent { + img.SetNRGBA(0, 0, color.NRGBA{A: 0}) + } + var b bytes.Buffer + Expect(png.Encode(&b, img)).To(Succeed()) + Expect(ValidatePNG(b.Bytes()) == nil).To(Equal(transparent)) + } + Expect(ValidatePNG([]byte("not PNG"))).NotTo(Succeed()) + }) +}) diff --git a/scripts/lib/backend-filter.mjs b/scripts/lib/backend-filter.mjs index 069433d3087a..db16b64fe829 100644 --- a/scripts/lib/backend-filter.mjs +++ b/scripts/lib/backend-filter.mjs @@ -188,14 +188,14 @@ const always = () => true; // go list -deps ./backend/go/... | grep LocalAI/pkg // // returns exactly pkg/audio, pkg/grpc (+ base, grpcerrors, proto), pkg/httpclient, -// pkg/sound, pkg/store and pkg/utils — identical for GOOS/GOARCH in +// pkg/pixal3d, pkg/sound, pkg/store and pkg/utils — identical for GOOS/GOARCH in // {linux,darwin} x {amd64,arm64}, so one list covers both matrices. Notably // absent: pkg/model, pkg/downloader, pkg/functions and the other ~21 pkg // subtrees, which are core-server-only. // // Enumerating rather than taking all of pkg/ is the whole point. All of pkg/ // changes in ~8.6% of commits and would fire a 199-entry Go matrix that often; -// these six subtrees change in 2.0%, which is the same order as the already +// the original six subtrees changed in 2.0%, the same order as the already // accepted scripts/build/ rule (1.9%). If the enumeration ever drifts from the // `go list` output the tests pin both directions — a listed subtree must // trigger, an unlisted one must not. @@ -203,6 +203,7 @@ const GO_BACKEND_PKG_PREFIXES = [ "pkg/audio/", "pkg/grpc/", // covers base/, grpcerrors/ and the generated proto/ "pkg/httpclient/", + "pkg/pixal3d/", "pkg/sound/", "pkg/store/", "pkg/utils/", diff --git a/scripts/lib/backend-filter_test.mjs b/scripts/lib/backend-filter_test.mjs index 25d942ae642d..ca58cfad7d1f 100644 --- a/scripts/lib/backend-filter_test.mjs +++ b/scripts/lib/backend-filter_test.mjs @@ -302,6 +302,7 @@ test("every enumerated Go-backend pkg subtree triggers a Go rebuild", () => { "pkg/audio/convert.go", "pkg/grpc/client.go", "pkg/httpclient/client.go", + "pkg/pixal3d/inputs.go", "pkg/sound/sound.go", "pkg/store/client.go", "pkg/utils/path.go", @@ -612,3 +613,8 @@ test("public motion schema and validation changes rebuild GEM-X on both OSes", ( assert.ok(!rules.some(rule => rule.linux({ backend: "kimodocpp", dockerfile: "./backend/Dockerfile.golang" }))); } }); + +test("pixal3dcpp maps to its Go wrapper on Linux and Darwin", () => { + assert.equal(inferBackendPath({ backend: "pixal3dcpp", dockerfile: "./backend/Dockerfile.golang" }), "backend/go/pixal3dcpp/"); + assert.equal(inferBackendPathDarwin({ backend: "pixal3dcpp", lang: "go" }), "backend/go/pixal3dcpp/"); +}); From f8d3ac9b05bb0e768ef6abeaaa59d016a1aec723 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> Date: Mon, 28 Sep 2026 08:06:50 +0000 Subject: [PATCH 2/5] fix(pixal3d): handle cleanup errors Check file-close errors and report failed temporary-file cleanup. Document trusted paths and the packaged CLI for security analysis. Assisted-by: Codex:gpt-6 golangci-lint gosec Signed-off-by: Ettore Di Giacinto --- backend/go/pixal3dcpp/pixal3d.go | 48 +++++++++++++++++++------- core/http/endpoints/localai/model3d.go | 2 +- 2 files changed, 37 insertions(+), 13 deletions(-) diff --git a/backend/go/pixal3dcpp/pixal3d.go b/backend/go/pixal3dcpp/pixal3d.go index a9a83eba5e5b..ec081a8d3ec8 100644 --- a/backend/go/pixal3dcpp/pixal3d.go +++ b/backend/go/pixal3dcpp/pixal3d.go @@ -15,6 +15,7 @@ import ( "github.com/mudler/LocalAI/pkg/grpc/base" pb "github.com/mudler/LocalAI/pkg/grpc/proto" "github.com/mudler/LocalAI/pkg/pixal3d" + "github.com/mudler/xlog" ) var modelFiles = []string{"dinov3.gguf", "pixal3d_naf.gguf", "pixal3d_ss_flow_mv.gguf", "ss_dec.gguf", "pixal3d_shape_flow_512_mv.gguf", "shape_dec.gguf", "pixal3d_shape_flow_1024_mv.gguf", "pixal3d_tex_flow_1024_mv.gguf", "tex_dec.gguf"} @@ -31,7 +32,7 @@ type Pixal3D struct { } func validateModels(dir string) error { - f, err := os.Open(filepath.Join(dir, "pixal3d-models.json")) + f, err := os.Open(filepath.Join(dir, "pixal3d-models.json")) // #nosec G304 -- fixed manifest name in the administrator-configured model directory if err != nil { return fmt.Errorf("Pixal3D model manifest: %w", err) } @@ -44,7 +45,10 @@ func validateModels(dir string) error { } `json:"files"` } err = json.NewDecoder(io.LimitReader(f, 1<<20)).Decode(&manifest) - f.Close() + closeErr := f.Close() + if closeErr != nil { + return fmt.Errorf("close Pixal3D manifest: %w", closeErr) + } if err != nil || manifest.Schema != 1 || (manifest.Family != "" && manifest.Family != "mv") { return fmt.Errorf("invalid Pixal3D MV model manifest") } @@ -56,14 +60,17 @@ func validateModels(dir string) error { sizes[file.Name] = file.Size } for _, name := range modelFiles { - f, err := os.Open(filepath.Join(dir, name)) + f, err := os.Open(filepath.Join(dir, name)) // #nosec G304 -- name comes from the fixed modelFiles list, not manifest input if err != nil { return fmt.Errorf("not a Pixal3D MV model set: %s: %w", name, err) } stat, err := f.Stat() var magic [4]byte _, readErr := io.ReadFull(f, magic[:]) - f.Close() + closeErr := f.Close() + if closeErr != nil { + return fmt.Errorf("close Pixal3D component %s: %w", name, closeErr) + } if err != nil || !stat.Mode().IsRegular() || readErr != nil || string(magic[:]) != "GGUF" || sizes[name] <= 0 || stat.Size() != sizes[name] { return fmt.Errorf("invalid Pixal3D component %s (check manifest and file size)", name) } @@ -138,13 +145,21 @@ func (p *Pixal3D) Generate3D(req *pb.Generate3DRequest) error { if err != nil { return err } - defer os.RemoveAll(dir) + defer func() { + if err := os.RemoveAll(dir); err != nil { + xlog.Warn("Failed to remove Pixal3D staged views", "error", err) + } + }() // The native CLI writes texture and mesh sidecars beside its GLB. outputDir, err := os.MkdirTemp("", "pixal3d-output-") if err != nil { return err } - defer os.RemoveAll(outputDir) + defer func() { + if err := os.RemoveAll(outputDir); err != nil { + xlog.Warn("Failed to remove Pixal3D staged output", "error", err) + } + }() output := filepath.Join(outputDir, "output.glb") args, err := commandArgs(models, dir, output, req) if err != nil { @@ -152,12 +167,15 @@ func (p *Pixal3D) Generate3D(req *pb.Generate3DRequest) error { } names := []string{"0_front.png", "1_right.png", "2_back.png", "3_left.png"} for i, path := range req.Images { - f, err := os.Open(path) + f, err := os.Open(path) // #nosec G304 -- images are local files staged by the server, not raw HTTP input if err != nil { return err } data, err := io.ReadAll(io.LimitReader(f, pixal3d.MaxInputBytes+1)) - f.Close() + closeErr := f.Close() + if closeErr != nil { + return closeErr + } if err != nil { return err } @@ -173,17 +191,21 @@ func (p *Pixal3D) Generate3D(req *pb.Generate3DRequest) error { args = append([]string{cli}, args...) cli = loader } - cmd := exec.CommandContext(ctx, cli, args...) + cmd := exec.CommandContext(ctx, cli, args...) // #nosec G204 -- Load resolves the packaged CLI and loader beside this executable; arguments never pass through a shell cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { return fmt.Errorf("Pixal3D generation failed: %w", err) } - f, err := os.Open(output) + f, err := os.Open(output) // #nosec G304 -- fixed output.glb name inside the private directory created above if err != nil { return err } - defer f.Close() + defer func() { + if err := f.Close(); err != nil { + xlog.Warn("Failed to close Pixal3D output", "error", err) + } + }() var header [4]byte if _, err := io.ReadFull(f, header[:]); err != nil || string(header[:]) != "glTF" { return fmt.Errorf("Pixal3D did not produce a GLB") @@ -199,7 +221,9 @@ func (p *Pixal3D) Generate3D(req *pb.Generate3DRequest) error { _, copyErr := io.Copy(dst, f) closeErr := dst.Close() if copyErr != nil || closeErr != nil { - os.Remove(req.Dst) + if err := os.Remove(req.Dst); err != nil { + xlog.Warn("Failed to remove incomplete Pixal3D output", "error", err) + } if copyErr != nil { return copyErr } diff --git a/core/http/endpoints/localai/model3d.go b/core/http/endpoints/localai/model3d.go index 34137a401f4f..d355487cd9a6 100644 --- a/core/http/endpoints/localai/model3d.go +++ b/core/http/endpoints/localai/model3d.go @@ -94,7 +94,7 @@ func Model3DEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appCon return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("invalid view %d: %v", i, stageErr)) } images = append(images, path) - data, readErr := os.ReadFile(path) + data, readErr := os.ReadFile(path) // #nosec G304 -- stageVideoMediaWithLimit returns a server-created temporary file if readErr != nil { return readErr } From d4efdcb7a9f715edc83ddabf06fffac98b85df83 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:04:43 +0000 Subject: [PATCH 3/5] fix(mcp): update assistant test client stub Implement InstallLibraryBackend so the assistant tests satisfy the LocalAIClient interface after the library integration. Assisted-by: Codex:GPT-6 --- core/http/endpoints/mcp/localai_assistant_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/http/endpoints/mcp/localai_assistant_test.go b/core/http/endpoints/mcp/localai_assistant_test.go index 36407565b198..e0c59ac88bb5 100644 --- a/core/http/endpoints/mcp/localai_assistant_test.go +++ b/core/http/endpoints/mcp/localai_assistant_test.go @@ -79,6 +79,10 @@ func (stubClient) InstallBackend(_ context.Context, _ localaitools.InstallBacken return "stub-backend-job", nil } +func (stubClient) InstallLibraryBackend(_ context.Context, _ localaitools.InstallLibraryBackendRequest) (*localaitools.InstallLibraryBackendResult, error) { + return &localaitools.InstallLibraryBackendResult{Status: "installed", Name: "stub-backend", SourceID: "stub-source"}, nil +} + func (stubClient) UpgradeBackend(_ context.Context, _ string) (string, error) { return "stub-upgrade-job", nil } From fa58029eb60001eb6ff586a18c80334f623ebfb2 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:07:31 +0000 Subject: [PATCH 4/5] test(gallery): count current library backends The shipped library grows when a backend is added. Compare the reader against the current library instead of the frozen migration fixture. Assisted-by: Codex:GPT-6 --- core/gallery/backend_library_generated_test.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/core/gallery/backend_library_generated_test.go b/core/gallery/backend_library_generated_test.go index 57676a6b3aaf..564f90982433 100644 --- a/core/gallery/backend_library_generated_test.go +++ b/core/gallery/backend_library_generated_test.go @@ -104,8 +104,12 @@ func TestLibraryGalleryReadsTheBackendLibrary(t *testing.T) { official.Library = true official.URL = srv.URL + "/backends/index.yaml" backends := names(official) - if len(backends) != goldenGalleryNames { - t.Fatalf("the library gave %d backends, want %d", len(backends), goldenGalleryNames) + expected, err := BackendsFromLibrary(loadRealBackendLibrary(t)) + if err != nil { + t.Fatal(err) + } + if len(backends) != len(expected) { + t.Fatalf("the library gave %d backends, want %d", len(backends), len(expected)) } llama := backends.FindByName("llama-cpp") if llama == nil || !llama.IsMeta() || llama.CapabilitiesMap["nvidia"] != "cuda12-llama-cpp" || llama.Alias != "llama-cpp" { From bc4bb1692cc07b975ce3c042cc55807e3c5d1eeb Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> Date: Fri, 23 Oct 2026 14:06:15 +0000 Subject: [PATCH 5/5] test(gallery): use Gomega for backend counts Keep the library count assertions within the test lint policy. Assisted-by: Codex:gpt-6 --- core/gallery/backend_library_generated_test.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/core/gallery/backend_library_generated_test.go b/core/gallery/backend_library_generated_test.go index 564f90982433..2e088aaa05c0 100644 --- a/core/gallery/backend_library_generated_test.go +++ b/core/gallery/backend_library_generated_test.go @@ -9,6 +9,7 @@ import ( "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/pkg/system" + "github.com/onsi/gomega" "gopkg.in/yaml.v3" ) @@ -105,12 +106,9 @@ func TestLibraryGalleryReadsTheBackendLibrary(t *testing.T) { official.URL = srv.URL + "/backends/index.yaml" backends := names(official) expected, err := BackendsFromLibrary(loadRealBackendLibrary(t)) - if err != nil { - t.Fatal(err) - } - if len(backends) != len(expected) { - t.Fatalf("the library gave %d backends, want %d", len(backends), len(expected)) - } + g := gomega.NewWithT(t) + g.Expect(err).To(gomega.Succeed()) + g.Expect(backends).To(gomega.HaveLen(len(expected))) llama := backends.FindByName("llama-cpp") if llama == nil || !llama.IsMeta() || llama.CapabilitiesMap["nvidia"] != "cuda12-llama-cpp" || llama.Alias != "llama-cpp" { t.Fatalf("the meta backend lost its body: %+v", llama)