feat(3d): add Pixal3D multiview generation

.github/backend-matrix.yml

@@ -808,6 +808,19 @@ include:
     dockerfile: "./backend/Dockerfile.golang"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: 'cublas'
+    cuda-major-version: "12"
+    cuda-minor-version: "8"
+    platforms: 'linux/amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-nvidia-cuda-12-pixal3dcpp'
+    runs-on: 'ubuntu-latest'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "pixal3dcpp"
+    dockerfile: "./backend/Dockerfile.golang"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: 'cublas'
     cuda-major-version: "12"
     cuda-minor-version: "8"
@@ -1820,6 +1833,19 @@ include:
     dockerfile: "./backend/Dockerfile.golang"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: 'cublas'
+    cuda-major-version: "13"
+    cuda-minor-version: "0"
+    platforms: 'linux/amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-nvidia-cuda-13-pixal3dcpp'
+    runs-on: 'ubuntu-latest'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "pixal3dcpp"
+    dockerfile: "./backend/Dockerfile.golang"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: 'cublas'
     cuda-major-version: "13"
     cuda-minor-version: "0"
@@ -3688,6 +3714,20 @@ include:
     dockerfile: "./backend/Dockerfile.golang"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: ''
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/amd64'
+    platform-tag: 'amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-cpu-pixal3dcpp'
+    runs-on: 'ubuntu-latest'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "pixal3dcpp"
+    dockerfile: "./backend/Dockerfile.golang"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: ''
     cuda-major-version: ""
     cuda-minor-version: ""
@@ -3702,6 +3742,20 @@ include:
     dockerfile: "./backend/Dockerfile.golang"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: ''
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/arm64'
+    platform-tag: 'arm64'
+    tag-latest: 'auto'
+    tag-suffix: '-cpu-pixal3dcpp'
+    runs-on: 'ubuntu-24.04-arm'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "pixal3dcpp"
+    dockerfile: "./backend/Dockerfile.golang"
+    context: "./"
+    ubuntu-version: '2404'
   # sam3-cpp
   - build-type: ''
     cuda-major-version: ""
@@ -4054,6 +4108,20 @@ include:
     dockerfile: "./backend/Dockerfile.golang"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: 'vulkan'
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/amd64'
+    platform-tag: 'amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-vulkan-pixal3dcpp'
+    runs-on: 'ubuntu-latest'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "pixal3dcpp"
+    dockerfile: "./backend/Dockerfile.golang"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: 'vulkan'
     cuda-major-version: ""
     cuda-minor-version: ""
@@ -4068,6 +4136,20 @@ include:
     dockerfile: "./backend/Dockerfile.golang"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: 'vulkan'
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/arm64'
+    platform-tag: 'arm64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-vulkan-pixal3dcpp'
+    runs-on: 'ubuntu-24.04-arm'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "pixal3dcpp"
+    dockerfile: "./backend/Dockerfile.golang"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: 'cublas'
     cuda-major-version: "12"
     cuda-minor-version: "0"
@@ -6676,6 +6758,10 @@ includeDarwin:
     tag-suffix: "-metal-darwin-arm64-trellis2cpp"
     build-type: "metal"
     lang: "go"
+  - backend: "pixal3dcpp"
+    tag-suffix: "-metal-darwin-arm64-pixal3dcpp"
+    build-type: "metal"
+    lang: "go"
   - backend: "whisper"
     tag-suffix: "-metal-darwin-arm64-whisper"
     build-type: "metal"

.github/workflows/bump_deps.yaml

@@ -86,6 +86,10 @@ jobs:
             variable: "STABLEDIFFUSION_GGML_VERSION"
             branch: "master"
             file: "backend/go/stablediffusion-ggml/Makefile"
+          - repository: "raven38/pixal3d.cpp"
+            variable: "PIXAL3D_VERSION"
+            branch: "main"
+            file: "backend/go/pixal3dcpp/Makefile"
           - repository: "localai-org/trellis2cpp"
             variable: "TRELLIS2CPP_VERSION"
             branch: "pbr-textures"

Makefile

@@ -1,5 +1,5 @@
 # Disable parallel execution for backend builds
-.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/bonsai backends/outetts backends/piper backends/stablediffusion-ggml backends/trellis2cpp backends/trellis2cpp-darwin backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/nemo-speech-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/valkey-store backends/qdrant-store backends/cloud-proxy backends/localai-proxy backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/mlx-video backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/moss-tts-cpp backends/magpie-tts-cpp backends/vllm-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin backends/audio-cpp backends/audio-cpp-darwin
+.NOTPARALLEL: backends/pixal3dcpp backends/pixal3dcpp-darwin backends/diffusers backends/llama-cpp backends/turboquant backends/bonsai backends/outetts backends/piper backends/stablediffusion-ggml backends/trellis2cpp backends/trellis2cpp-darwin backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/nemo-speech-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/valkey-store backends/qdrant-store backends/cloud-proxy backends/localai-proxy backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/mlx-video backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/moss-tts-cpp backends/magpie-tts-cpp backends/vllm-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin backends/audio-cpp backends/audio-cpp-darwin
 .NOTPARALLEL: backends/whisper-medusa
 .NOTPARALLEL: backends/funasr
 
@@ -659,6 +659,7 @@ prepare-test-extra: protogen-python
 	$(MAKE) -C backend/go/rfdetr-cpp
 	$(MAKE) -C backend/go/locate-anything-cpp
 	$(MAKE) -C backend/go/trellis2cpp
+	$(MAKE) -C backend/go/pixal3dcpp
 	$(MAKE) -C backend/go/kimodocpp
 	$(MAKE) -C backend/go/gemxcpp
 	$(MAKE) -C backend/go/valkey-store
@@ -700,6 +701,7 @@ test-extra: prepare-test-extra
 	$(MAKE) -C backend/go/vllm-cpp test
 	$(MAKE) -C backend/go/nemo-speech-cpp test
 	$(MAKE) -C backend/go/trellis2cpp test
+	$(MAKE) -C backend/go/pixal3dcpp test
 	$(MAKE) -C backend/go/kimodocpp test
 	$(MAKE) -C backend/go/gemxcpp test
 	$(MAKE) -C backend/go/valkey-store test
@@ -1359,6 +1361,7 @@ BACKEND_LOCALAI_PROXY = localai-proxy|golang|.|false|true
 BACKEND_HUGGINGFACE = huggingface|golang|.|false|true
 BACKEND_SILERO_VAD = silero-vad|golang|.|false|true
 BACKEND_STABLEDIFFUSION_GGML = stablediffusion-ggml|golang|.|--progress=plain|true
+BACKEND_PIXAL3DCPP = pixal3dcpp|golang|.|--progress=plain|true
 BACKEND_TRELLIS2CPP = trellis2cpp|golang|.|--progress=plain|true
 BACKEND_GEMXCPP = gemxcpp|golang|.|--progress=plain|true
 BACKEND_KIMODOCPP = kimodocpp|golang|.|--progress=plain|true
@@ -1469,10 +1472,11 @@ $(eval $(call generate-docker-build-target,$(BACKEND_HUGGINGFACE)))
 $(eval $(call generate-docker-build-target,$(BACKEND_SILERO_VAD)))
 $(eval $(call generate-docker-build-target,$(BACKEND_STABLEDIFFUSION_GGML)))
 $(eval $(call generate-docker-build-target,$(BACKEND_TRELLIS2CPP)))
+$(eval $(call generate-docker-build-target,$(BACKEND_PIXAL3DCPP)))
 $(eval $(call generate-docker-build-target,$(BACKEND_KIMODOCPP)))
 $(eval $(call generate-docker-build-target,$(BACKEND_GEMXCPP)))
 .NOTPARALLEL: backends/kimodocpp backends/kimodocpp-darwin
-docker-build-backends: docker-build-kimodocpp
+docker-build-backends: docker-build-pixal3dcpp docker-build-kimodocpp
 
 backends/kimodocpp-darwin:
 	BACKEND=kimodocpp BUILD_TYPE=cpu $(MAKE) build-darwin-go-backend
@@ -1743,3 +1747,6 @@ build-launcher-appimage:
 
 test-launcher-appimage:
 	python3 -m unittest discover -s contrib/appimage -p 'test_*.py' -v
+backends/pixal3dcpp-darwin:
+	BACKEND=pixal3dcpp BUILD_TYPE=metal $(MAKE) build-darwin-go-backend
+	./local-ai backends install "ocifile://$(abspath ./backend-images/pixal3dcpp.tar)"

README.md

@@ -235,6 +235,10 @@ LocalAI supports **60+ backends** including llama.cpp, vLLM, SGLang, transformer
 
 See the full [Backend & Model Compatibility Table](https://localai.io/model-compatibility/) and [GPU Acceleration guide](https://localai.io/features/gpu-acceleration/).
 
+| Additional native backend | What it does |
+|---------------------------|--------------|
+| [pixal3d.cpp](https://github.com/raven38/pixal3d.cpp) (`pixal3dcpp`) | Four canonical RGBA views to a textured 3D GLB, with an explicit mesh scale. [Setup and API](https://localai.io/features/3d-generation/) |
+
 ### Backends built by us
 
 Most backends wrap a best-in-class upstream engine. A handful of them are native C/C++/GGML engines (no Python at inference) developed and maintained by the LocalAI project itself:

addons/studio/ui/src/pages/ThreeDGen.jsx

@@ -77,6 +77,9 @@ export default function ThreeDGen() {
   const modelNames = useMemo(() => models.filter(item => item.capabilities?.some(cap => cap === CAP_3D || cap === CAP_3D_ANIMATION)).map(item => item.id), [models])
   const selectedModel = models.find(item => item.id === model)
   const animation = selectedModel?.three_d_operations?.find(operation => operation.id === 'animate')
+  const multiview = selectedModel?.three_d_operations?.some(operation => operation.id === 'generate_multiview')
+  const [views, setViews] = useState({})
+  const [meshScale, setMeshScale] = useState('')
   const [animationInputs, setAnimationInputs] = useState({})
   const [animationParams, setAnimationParams] = useState({})
   const [requestEndpoint, setRequestEndpoint] = useState('/3d/generations')
@@ -125,6 +128,8 @@ export default function ThreeDGen() {
     setAnimationInputs({})
     setAnimationParams({})
     setImage(null)
+    setViews({})
+    setMeshScale('')
     setQuality('auto')
     setBackground('auto')
     setSteps('')
@@ -144,7 +149,10 @@ export default function ThreeDGen() {
   }
 
   const run = async () => {
-    if (!animation && !image?.base64) { addToast(t('threed.toasts.noImage'), 'warning'); return }
+    if (multiview && (['front', 'right', 'back', 'left'].some(name => !views[name]?.base64) || !Number.isFinite(Number(meshScale)) || Number(meshScale) <= 0)) {
+      addToast('Provide all four RGBA PNG views and a positive mesh scale.', 'warning'); return
+    }
+    if (!animation && !multiview && !image?.base64) { addToast(t('threed.toasts.noImage'), 'warning'); return }
     if (!model) { addToast(t('threed.toasts.noModel'), 'warning'); return }
 
     setLoading(true)
@@ -156,8 +164,9 @@ export default function ThreeDGen() {
     const params = Object.fromEntries(Object.entries(animationParams).filter(([key, value]) => value !== '' && animation?.parameters.some(parameter => parameter.name === key)))
     const body = animation
       ? { model, inputs: animationInputs, params, response_format: 'url' }
+      : multiview ? { model, images: ['front', 'right', 'back', 'left'].map(name => views[name].base64), mesh_scale: Number(meshScale), quality: '1024', response_format: 'url' }
       : { model, image: image.base64, quality, background, response_format: 'url' }
-    if (!animation) {
+    if (!animation && !multiview) {
       if (steps) body.step = parseInt(steps)
       if (textureSteps) body.texture_steps = parseInt(textureSteps)
       if (guidance) body.cfg_scale = parseFloat(guidance)
@@ -166,7 +175,7 @@ export default function ThreeDGen() {
 
     // RequestPanel renders and copies its body. Keeping a multi-megabyte image
     // there duplicates the upload in React and can starve the result render.
-    setLastRequest(animation ? body : { ...body, image: `<base64 ${image.mime || 'image'} omitted>` })
+    setLastRequest(animation ? body : multiview ? { ...body, images: ['front', 'right', 'back', 'left'].map(name => `<${name} RGBA PNG omitted>`) } : { ...body, image: `<base64 ${image.mime || 'image'} omitted>` })
     setRequestEndpoint(animation ? '/3d/animate' : '/3d/generations')
 
     try {
@@ -186,7 +195,7 @@ export default function ThreeDGen() {
       const inputThumb = !animation && image?.dataUrl ? await makeThumb(image.dataUrl) : null
       const saved = await addEntry({
         model,
-        params: animation ? params : { quality, background, steps, textureSteps, guidance, seed },
+        params: animation ? params : multiview ? { mesh_scale: Number(meshScale), quality: '1024' } : { quality, background, steps, textureSteps, guidance, seed },
         inputs: animation ? animationInputs : undefined,
         operation: animation ? 'animate' : 'generate',
         outputType,
@@ -233,7 +242,7 @@ export default function ThreeDGen() {
   const installed = ws.installed.byType.threed
   const noModel = !ws.installed.loading && !ws.installed.error && installed.length === 0
   const why = noModel ? t('studio.composer.whyModel', { type: t('studio.tabs.threed') })
-    : (!animation && !image?.base64) ? t('studio.workspace.threed.whyImage') : ''
+    : (!animation && !multiview && !image?.base64) ? t('studio.workspace.threed.whyImage') : ''
   const f = (key) => t(`studio.workspace.fields.${key}`)
   const advancedSet = [steps && `${t('threed.labels.steps')} ${steps}`, textureSteps && `${t('threed.labels.textureSteps')} ${textureSteps}`, guidance && `${f('cfg')} ${guidance}`, seed && `${f('seed')} ${seed}`].filter(Boolean)
   const qualityOptions = QUALITIES.map(q => ({ value: q, label: t(`threed.labels.quality_${q}`) }))
@@ -252,12 +261,12 @@ export default function ThreeDGen() {
         noModel={noModel ? { type: 'threed', label: t('studio.tabs.threed'), onChanged: ws.installed.refetch } : null}
         options={<>
           <ModelChip value={model} onChange={changeModel} options={modelNames} loading={modelsLoading} disabled={loading} capability={CAP_3D} />
-          {!animation && <>
+          {!animation && !multiview && <>
             <ChipSelect label={t('threed.labels.quality')} value={quality} onChange={setQuality} options={qualityOptions} mono={false} testId="ws-quality" />
             <ChipSelect label={t('threed.labels.background')} value={background} onChange={setBackground} options={backgroundOptions} mono={false} testId="ws-background" />
           </>}
         </>}
-        fold={animation ? null : (
+        fold={animation || multiview ? null : (
           <Fold
             label={t('threed.labels.advanced')}
             summary={foldSummary(advancedSet, [t('threed.labels.steps'), t('threed.labels.textureSteps'), t('threed.labels.guidance'), t('threed.labels.seed')])}
@@ -273,12 +282,26 @@ export default function ThreeDGen() {
             </div>
           </Fold>
         )}
-        submit={{ label: t('threed.actions.generate'), busyLabel: t('threed.actions.generating'), busy: loading, disabled: !model || noModel || (!animation && !image?.base64), why, icon: 'cube' }}
+        submit={{ label: t('threed.actions.generate'), busyLabel: t('threed.actions.generating'), busy: loading, disabled: !model || noModel || (!animation && !multiview && !image?.base64), why, icon: 'cube' }}
       >
         {animation ? (
           <div className="ws-inputs ws-inputs--one">
             <AnimationOptions key={model} operation={animation} inputs={animationInputs} onInputsChange={setAnimationInputs} params={animationParams} onParamsChange={setAnimationParams} />
           </div>
+        ) : multiview ? (
+         <>
+           <p className="form-hint">Upload four pre-matted RGBA PNGs of the same object in the canonical turntable framing. Resolution: 1024.</p>
+           {['front', 'right', 'back', 'left'].map(name => (
+             <MediaInput key={name} mode="image" label={`${name[0].toUpperCase()}${name.slice(1)} RGBA PNG`} value={views[name] || null}
+               onChange={value => setViews(current => ({ ...current, [name]: value }))}
+               onError={err => addToast(err.message, 'error')} maxBytes={MAX_3D_INPUT_BYTES} idPrefix={`threed-${name}`} />
+           ))}
+           <div className="form-group">
+             <label className="form-label" htmlFor="threed-mesh-scale">Mesh scale</label>
+             <input id="threed-mesh-scale" className="input" type="number" step="any" min="0" required value={meshScale} onChange={e => setMeshScale(e.target.value)} />
+             <p className="form-hint">Enter the positive scale used to frame these views. Pixal3D does not estimate it.</p>
+           </div>
+         </>
         ) : (
           <div className="ws-inputs ws-inputs--one">
             <MediaInput

backend/backend.proto

@@ -766,6 +766,8 @@ message Animate3DRequest {
 }
 
 message Generate3DRequest {
+  repeated string images = 10; // Staged canonical views: front, right, back, left
+  double mesh_scale = 11; // Explicit finite positive Pixal3D object scale
   string src = 1;          // Path to the staged conditioning image (3D generation is image-conditioned)
   string dst = 2;          // Output path for the generated binary glTF (.glb) asset
   int32 seed = 3;          // <=0 lets the backend pick a random seed