diff --git a/Makefile b/Makefile index 691e140a5..33a8c68e8 100644 --- a/Makefile +++ b/Makefile @@ -25,8 +25,15 @@ DOCKER_BUILD_ARGS := \ # Test configuration BUILD_DMR ?= 1 -# Main targets -.PHONY: build run clean test integration-tests test-docker-ce-installation docker-build docker-build-multiplatform docker-run docker-build-vllm docker-run-vllm docker-build-sglang docker-run-sglang docker-run-impl help validate validate-all lint docker-build-diffusers docker-run-diffusers vllm-metal-build vllm-metal-install vllm-metal-dev vllm-metal-clean build-cli install-cli +# Phony targets grouped by category +.PHONY: build run clean test integration-tests build-cli install-cli +.PHONY: validate validate-all lint help +.PHONY: docker-build docker-build-multiplatform docker-run docker-run-impl +.PHONY: docker-build-vllm docker-run-vllm docker-build-sglang docker-run-sglang +.PHONY: docker-build-diffusers docker-run-diffusers +.PHONY: test-docker-ce-installation +.PHONY: vllm-metal-build vllm-metal-install vllm-metal-dev vllm-metal-clean +.PHONY: diffusers-build diffusers-install diffusers-dev diffusers-clean # Default target .DEFAULT_GOAL := build @@ -242,6 +249,75 @@ vllm-metal-clean: rm -f $(VLLM_METAL_TARBALL) @echo "vllm-metal cleaned!" +# diffusers (macOS ARM64 and Linux) +# The tarball is self-contained: includes a standalone Python 3.12 + all packages. +DIFFUSERS_RELEASE ?= v0.1.0-20260216-000000 +DIFFUSERS_INSTALL_DIR := $(HOME)/.docker/model-runner/diffusers +DIFFUSERS_OS := $(shell uname -s | tr '[:upper:]' '[:lower:]') +DIFFUSERS_ARCH := $(shell uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/') +DIFFUSERS_TARBALL := diffusers-$(DIFFUSERS_OS)-$(DIFFUSERS_ARCH)-$(DIFFUSERS_RELEASE).tar.gz + +diffusers-build: + @if [ -f "$(DIFFUSERS_TARBALL)" ]; then \ + echo "Tarball already exists: $(DIFFUSERS_TARBALL)"; \ + else \ + echo "Building diffusers tarball..."; \ + scripts/build-diffusers-tarball.sh $(DIFFUSERS_RELEASE) $(DIFFUSERS_TARBALL); \ + echo "Tarball created: $(DIFFUSERS_TARBALL)"; \ + fi + +diffusers-install: + @VERSION_FILE="$(DIFFUSERS_INSTALL_DIR)/.diffusers-version"; \ + if [ -f "$$VERSION_FILE" ] && [ "$$(cat "$$VERSION_FILE")" = "$(DIFFUSERS_RELEASE)" ]; then \ + echo "diffusers $(DIFFUSERS_RELEASE) already installed"; \ + exit 0; \ + fi; \ + if [ ! -f "$(DIFFUSERS_TARBALL)" ]; then \ + echo "Error: $(DIFFUSERS_TARBALL) not found. Run 'make diffusers-build' first."; \ + exit 1; \ + fi; \ + echo "Installing diffusers to $(DIFFUSERS_INSTALL_DIR)..."; \ + rm -rf "$(DIFFUSERS_INSTALL_DIR)"; \ + mkdir -p "$(DIFFUSERS_INSTALL_DIR)"; \ + tar -xzf "$(DIFFUSERS_TARBALL)" -C "$(DIFFUSERS_INSTALL_DIR)"; \ + echo "$(DIFFUSERS_RELEASE)" > "$$VERSION_FILE"; \ + echo "diffusers $(DIFFUSERS_RELEASE) installed successfully!" + +diffusers-dev: + @if [ -z "$(DIFFUSERS_PATH)" ]; then \ + echo "Usage: make diffusers-dev DIFFUSERS_PATH=../path-to-diffusers-server"; \ + exit 1; \ + fi + @PYTHON_BIN=""; \ + if command -v python3.12 >/dev/null 2>&1; then \ + PYTHON_BIN="python3.12"; \ + elif command -v python3 >/dev/null 2>&1; then \ + version=$$(python3 --version 2>&1 | grep -oE '[0-9]+\.[0-9]+'); \ + if [ "$$version" = "3.12" ]; then \ + PYTHON_BIN="python3"; \ + fi; \ + fi; \ + if [ -z "$$PYTHON_BIN" ]; then \ + echo "Error: Python 3.12 required"; \ + echo "Install with: brew install python@3.12"; \ + exit 1; \ + fi; \ + echo "Installing diffusers from $(DIFFUSERS_PATH)..."; \ + rm -rf "$(DIFFUSERS_INSTALL_DIR)"; \ + $$PYTHON_BIN -m venv "$(DIFFUSERS_INSTALL_DIR)"; \ + . "$(DIFFUSERS_INSTALL_DIR)/bin/activate" && \ + pip install "diffusers==0.36.0" "torch==2.9.1" "transformers==4.57.5" "accelerate==1.3.0" "safetensors==0.5.2" "huggingface_hub==0.34.0" "bitsandbytes==0.49.1" "fastapi==0.115.12" "uvicorn[standard]==0.34.1" "pillow==11.2.1" && \ + SITE_PACKAGES="$(DIFFUSERS_INSTALL_DIR)/lib/python3.12/site-packages" && \ + cp -Rp "$(DIFFUSERS_PATH)/python/diffusers_server" "$$SITE_PACKAGES/diffusers_server" && \ + echo "dev" > "$(DIFFUSERS_INSTALL_DIR)/.diffusers-version"; \ + echo "diffusers dev installed from $(DIFFUSERS_PATH)" + +diffusers-clean: + @echo "Removing diffusers installation and build artifacts..." + rm -rf "$(DIFFUSERS_INSTALL_DIR)" + rm -f $(DIFFUSERS_TARBALL) + @echo "diffusers cleaned!" + help: @echo "Available targets:" @echo " build - Build the Go application" @@ -269,6 +345,10 @@ help: @echo " vllm-metal-install - Install vllm-metal from local tarball" @echo " vllm-metal-dev - Install vllm-metal from local source (editable)" @echo " vllm-metal-clean - Clean vllm-metal installation and tarball" + @echo " diffusers-build - Build diffusers tarball locally" + @echo " diffusers-install - Install diffusers from local tarball" + @echo " diffusers-dev - Install diffusers from local source (editable)" + @echo " diffusers-clean - Clean diffusers installation and tarball" @echo " help - Show this help message" @echo "" @echo "Backend configuration options:" @@ -287,3 +367,11 @@ help: @echo " make vllm-metal-build && make vllm-metal-install && make run" @echo " 3. Install from local source (for development, requires Python 3.12):" @echo " make vllm-metal-dev VLLM_METAL_PATH=../vllm-metal && make run" + @echo "" + @echo "diffusers (macOS ARM64 and Linux):" + @echo " 1. Auto-pull from Docker Hub (clean dev installs first: make diffusers-clean):" + @echo " make run" + @echo " 2. Build and install from tarball:" + @echo " make diffusers-build && make diffusers-install && make run" + @echo " 3. Install from local source (for development, requires Python 3.12):" + @echo " make diffusers-dev DIFFUSERS_PATH=. && make run" diff --git a/main.go b/main.go index 717f54301..e7855768b 100644 --- a/main.go +++ b/main.go @@ -113,6 +113,9 @@ func main() { if mlxServerPath != "" { log.Info("MLX_SERVER_PATH", "path", mlxServerPath) } + if diffusersServerPath != "" { + log.Info("DIFFUSERS_SERVER_PATH", "path", diffusersServerPath) + } if vllmMetalServerPath != "" { log.Info("VLLM_METAL_SERVER_PATH", "path", vllmMetalServerPath) } @@ -145,6 +148,8 @@ func main() { IncludeVLLM: includeVLLM, VLLMPath: vllmServerPath, VLLMMetalPath: vllmMetalServerPath, + IncludeDiffusers: true, + DiffusersPath: diffusersServerPath, }), routing.BackendDef{Name: sglang.Name, Init: func(mm *models.Manager) (inference.Backend, error) { return sglang.New(log, mm, log.With("component", sglang.Name), nil, sglangServerPath) @@ -167,7 +172,7 @@ func main() { ), IncludeResponsesAPI: true, ExtraRoutes: func(r *routing.NormalizedServeMux, s *routing.Service) { - // Root handler - only catches exact "/" requests + // Root handler – only catches exact "/" requests r.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) { if req.URL.Path != "/" { http.NotFound(w, req) diff --git a/pkg/inference/backends/diffusers/diffusers.go b/pkg/inference/backends/diffusers/diffusers.go index d51e03850..40f6c04d6 100644 --- a/pkg/inference/backends/diffusers/diffusers.go +++ b/pkg/inference/backends/diffusers/diffusers.go @@ -8,28 +8,30 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "strings" - "github.com/docker/model-runner/pkg/diskusage" "github.com/docker/model-runner/pkg/inference" "github.com/docker/model-runner/pkg/inference/backends" "github.com/docker/model-runner/pkg/inference/models" "github.com/docker/model-runner/pkg/inference/platform" + "github.com/docker/model-runner/pkg/internal/dockerhub" "github.com/docker/model-runner/pkg/internal/utils" "github.com/docker/model-runner/pkg/logging" ) const ( // Name is the backend name. - Name = "diffusers" - diffusersDir = "/opt/diffusers-env" + Name = "diffusers" + defaultInstallDir = ".docker/model-runner/diffusers" + // diffusersVersion is the diffusers release tag to download from Docker Hub. + diffusersVersion = "v0.1.0-20260216-000000" ) var ( - ErrNotImplemented = errors.New("not implemented") - ErrDiffusersNotFound = errors.New("diffusers package not installed") - ErrPythonNotFound = errors.New("python3 not found in PATH") - ErrNoDDUFFile = errors.New("no DDUF file found in model bundle") + ErrNoDDUFFile = errors.New("no DDUF file found in model bundle") + // ErrPlatformNotSupported indicates the platform is not supported. + ErrPlatformNotSupported = errors.New("diffusers is not available on this platform") ) // diffusers is the diffusers-based backend implementation for image generation. @@ -44,20 +46,28 @@ type diffusers struct { config *Config // status is the state in which the diffusers backend is in. status string - // pythonPath is the path to the python3 binary. + // pythonPath is the path to the bundled python3 binary. pythonPath string - // customPythonPath is an optional custom path to the python3 binary. + // customPythonPath is an optional custom path to a python3 binary. customPythonPath string + // installDir is the directory where diffusers is installed. + installDir string } // New creates a new diffusers-based backend for image generation. -// customPythonPath is an optional path to a custom python3 binary; if empty, the default path is used. +// customPythonPath is an optional path to a custom python3 binary; if empty, the default installation is used. func New(log logging.Logger, modelManager *models.Manager, serverLog logging.Logger, conf *Config, customPythonPath string) (inference.Backend, error) { // If no config is provided, use the default configuration if conf == nil { conf = NewDefaultConfig() } + homeDir, err := os.UserHomeDir() + if err != nil { + return nil, fmt.Errorf("failed to get user home directory: %w", err) + } + installDir := filepath.Join(homeDir, defaultInstallDir) + return &diffusers{ log: log, modelManager: modelManager, @@ -65,6 +75,7 @@ func New(log logging.Logger, modelManager *models.Manager, serverLog logging.Log config: conf, status: inference.FormatNotInstalled(""), customPythonPath: customPythonPath, + installDir: installDir, }, nil } @@ -76,60 +87,124 @@ func (d *diffusers) Name() string { // UsesExternalModelManagement implements inference.Backend.UsesExternalModelManagement. // Diffusers uses the shared model manager with bundled DDUF files. func (d *diffusers) UsesExternalModelManagement() bool { - return false // Use the bundle system for DDUF files + return false } // UsesTCP implements inference.Backend.UsesTCP. -// Diffusers uses TCP for communication, like SGLang. +// Diffusers uses TCP for communication. func (d *diffusers) UsesTCP() bool { return true } // Install implements inference.Backend.Install. -func (d *diffusers) Install(_ context.Context, _ *http.Client) error { +func (d *diffusers) Install(ctx context.Context, httpClient *http.Client) error { if !platform.SupportsDiffusers() { - d.status = inference.FormatNotInstalled(inference.DetailOnlyLinux) - return ErrNotImplemented + return ErrPlatformNotSupported } - var pythonPath string - - // Use custom python path if specified if d.customPythonPath != "" { - pythonPath = d.customPythonPath - } else { - venvPython := filepath.Join(diffusersDir, "bin", "python3") - pythonPath = venvPython - - if _, err := os.Stat(venvPython); err != nil { - // Fall back to system Python - systemPython, err := exec.LookPath("python3") - if err != nil { - d.status = inference.FormatError(inference.DetailPythonNotFound) - return ErrPythonNotFound + d.pythonPath = d.customPythonPath + return d.verifyInstallation(ctx) + } + + pythonPath := filepath.Join(d.installDir, "bin", "python3") + versionFile := filepath.Join(d.installDir, ".diffusers-version") + + // Check if already installed with correct version + if _, err := os.Stat(pythonPath); err == nil { + if installedVersion, err := os.ReadFile(versionFile); err == nil { + installed := strings.TrimSpace(string(installedVersion)) + if installed == diffusersVersion || installed == "dev" { + d.pythonPath = pythonPath + return d.verifyInstallation(ctx) } - pythonPath = systemPython + d.log.Info("diffusers version mismatch", "installed", installed, "expected", diffusersVersion) } } - d.pythonPath = pythonPath + d.status = "installing" + if err := d.downloadAndExtract(ctx); err != nil { + return fmt.Errorf("failed to install diffusers: %w", err) + } - // Check if diffusers is installed - if err := d.pythonCmd("-c", "import diffusers").Run(); err != nil { - d.status = inference.FormatNotInstalled(inference.DetailPackageNotInstalled) - d.log.Warn("diffusers package not found. Install with: uv pip install diffusers torch") - return ErrDiffusersNotFound + // Save version file + if err := os.WriteFile(versionFile, []byte(diffusersVersion), 0644); err != nil { + d.log.Warn("failed to write version file", "error", err) } - // Get version - output, err := d.pythonCmd("-c", "import diffusers; print(diffusers.__version__)").Output() + d.pythonPath = pythonPath + return d.verifyInstallation(ctx) +} + +// downloadAndExtract downloads the diffusers image from Docker Hub and extracts it. +// The image contains a self-contained Python installation with all packages pre-installed. +func (d *diffusers) downloadAndExtract(ctx context.Context) error { + d.log.Info("Downloading diffusers from Docker Hub...", "version", diffusersVersion) + + // Create temp directory for download + downloadDir, err := os.MkdirTemp("", "diffusers-install") if err != nil { - d.log.Warn("could not get diffusers version", "error", err) - d.status = inference.FormatRunning(inference.DetailVersionUnknown) - } else { - d.status = inference.FormatRunning(fmt.Sprintf("diffusers %s", strings.TrimSpace(string(output)))) + return fmt.Errorf("failed to create temp dir: %w", err) } + defer os.RemoveAll(downloadDir) + // Pull the image + image := fmt.Sprintf("registry-1.docker.io/docker/model-runner:diffusers-%s", diffusersVersion) + if err := dockerhub.PullPlatform(ctx, image, filepath.Join(downloadDir, "image.tar"), runtime.GOOS, runtime.GOARCH); err != nil { + return fmt.Errorf("failed to pull image: %w", err) + } + + // Extract the image + extractDir := filepath.Join(downloadDir, "extracted") + if err := dockerhub.Extract(filepath.Join(downloadDir, "image.tar"), runtime.GOARCH, runtime.GOOS, extractDir); err != nil { + return fmt.Errorf("failed to extract image: %w", err) + } + + if err := os.MkdirAll(filepath.Dir(d.installDir), 0755); err != nil { + return fmt.Errorf("failed to create parent dir: %w", err) + } + + // Remove existing install dir if it exists (incomplete installation) + if err := os.RemoveAll(d.installDir); err != nil { + return fmt.Errorf("failed to remove existing install dir: %w", err) + } + + d.log.Info("Extracting self-contained Python environment...") + + // Copy the extracted self-contained Python installation directly to install dir + // (the image contains /diffusers/ with bin/, lib/, etc.) + diffusersDir := filepath.Join(extractDir, "diffusers") + if err := utils.CopyDir(diffusersDir, d.installDir); err != nil { + return fmt.Errorf("failed to copy to install dir: %w", err) + } + + // Docker COPY strips execute permissions in OCI image layers. + // Restore the execute bit on the bundled Python binary. + if err := os.Chmod(filepath.Join(d.installDir, "bin", "python3"), 0755); err != nil { + return fmt.Errorf("failed to make python3 executable: %w", err) + } + + d.log.Info("diffusers installed successfully", "version", diffusersVersion) + return nil +} + +// verifyInstallation checks that the diffusers Python package can be imported. +// Note: d.pythonPath is not user-controlled — it is set internally by Install() +// to the bundled Python binary path, so the exec.Command usage is safe. +func (d *diffusers) verifyInstallation(ctx context.Context) error { + cmd := exec.CommandContext(ctx, d.pythonPath, "-c", "import diffusers") //nolint:gosec // pythonPath is set internally by Install, not user input + if err := cmd.Run(); err != nil { + d.status = "import failed" + return fmt.Errorf("diffusers import failed: %w", err) + } + + versionFile := filepath.Join(d.installDir, ".diffusers-version") + versionBytes, err := os.ReadFile(versionFile) + if err != nil { + d.status = "running diffusers" + return nil + } + d.status = fmt.Sprintf("running diffusers %s", strings.TrimSpace(string(versionBytes))) return nil } @@ -137,7 +212,7 @@ func (d *diffusers) Install(_ context.Context, _ *http.Client) error { func (d *diffusers) Run(ctx context.Context, socket, model string, modelRef string, mode inference.BackendMode, backendConfig *inference.BackendConfiguration) error { if !platform.SupportsDiffusers() { d.log.Warn("diffusers backend is not yet supported on this platform") - return ErrNotImplemented + return ErrPlatformNotSupported } // For diffusers, we support image generation mode @@ -175,16 +250,11 @@ func (d *diffusers) Run(ctx context.Context, socket, model string, modelRef stri return fmt.Errorf("diffusers: python runtime not configured; did you forget to call Install") } - sandboxPath := "" - if _, err := os.Stat(diffusersDir); err == nil { - sandboxPath = diffusersDir - } - return backends.RunBackend(ctx, backends.RunnerConfig{ BackendName: "Diffusers", Socket: socket, BinaryPath: d.pythonPath, - SandboxPath: sandboxPath, + SandboxPath: "", SandboxConfig: "", Args: args, Logger: d.log, @@ -200,39 +270,23 @@ func (d *diffusers) Status() string { // GetDiskUsage implements inference.Backend.GetDiskUsage. func (d *diffusers) GetDiskUsage() (int64, error) { - // Check if Docker installation exists - if _, err := os.Stat(diffusersDir); err == nil { - size, err := diskusage.Size(diffusersDir) - if err != nil { - return 0, fmt.Errorf("error while getting diffusers dir size: %w", err) - } - return size, nil + // Return 0 if not installed + if _, err := os.Stat(d.installDir); os.IsNotExist(err) { + return 0, nil } - // Python installation doesn't have a dedicated installation directory - // It's installed via pip in the system Python environment - return 0, nil -} - -// GetRequiredMemoryForModel returns the estimated memory requirements for a model. -func (d *diffusers) GetRequiredMemoryForModel(_ context.Context, _ string, _ *inference.BackendConfiguration) (inference.RequiredMemory, error) { - if !platform.SupportsDiffusers() { - return inference.RequiredMemory{}, ErrNotImplemented - } - - // Stable Diffusion models typically require significant VRAM - // SD 1.5: ~4GB VRAM, SD 2.1: ~5GB VRAM, SDXL: ~8GB VRAM - return inference.RequiredMemory{ - RAM: 4 * 1024 * 1024 * 1024, // 4GB RAM - VRAM: 6 * 1024 * 1024 * 1024, // 6GB VRAM (average estimate) - }, nil -} -// pythonCmd creates an exec.Cmd that runs python with the given arguments. -// It uses the configured pythonPath if available, otherwise falls back to "python3". -func (d *diffusers) pythonCmd(args ...string) *exec.Cmd { - pythonBinary := "python3" - if d.pythonPath != "" { - pythonBinary = d.pythonPath + var size int64 + err := filepath.Walk(d.installDir, func(_ string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + size += info.Size() + } + return nil + }) + if err != nil { + return 0, fmt.Errorf("error while getting store size: %w", err) } - return exec.Command(pythonBinary, args...) + return size, nil } diff --git a/pkg/inference/backends/vllm/vllm_metal.go b/pkg/inference/backends/vllm/vllm_metal.go index af7535307..b65d490a0 100644 --- a/pkg/inference/backends/vllm/vllm_metal.go +++ b/pkg/inference/backends/vllm/vllm_metal.go @@ -18,6 +18,7 @@ import ( "github.com/docker/model-runner/pkg/inference/models" "github.com/docker/model-runner/pkg/inference/platform" "github.com/docker/model-runner/pkg/internal/dockerhub" + "github.com/docker/model-runner/pkg/internal/utils" "github.com/docker/model-runner/pkg/logging" ) @@ -164,7 +165,7 @@ func (v *vllmMetal) downloadAndExtract(ctx context.Context, _ *http.Client) erro // Copy the extracted self-contained Python installation directly to install dir // (the image contains /vllm-metal/ with bin/, lib/, etc.) vllmMetalDir := filepath.Join(extractDir, "vllm-metal") - if err := copyDir(vllmMetalDir, v.installDir); err != nil { + if err := utils.CopyDir(vllmMetalDir, v.installDir); err != nil { return fmt.Errorf("failed to copy to install dir: %w", err) } @@ -178,52 +179,6 @@ func (v *vllmMetal) downloadAndExtract(ctx context.Context, _ *http.Client) erro return nil } -// copyDir recursively copies a directory. -func copyDir(src, dst string) error { - return filepath.Walk(src, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - relPath, err := filepath.Rel(src, path) - if err != nil { - return err - } - dstPath := filepath.Join(dst, relPath) - - if info.IsDir() { - return os.MkdirAll(dstPath, info.Mode()) - } - - if info.Mode()&os.ModeSymlink == os.ModeSymlink { - link, err := os.Readlink(path) - if err != nil { - return err - } - return os.Symlink(link, dstPath) - } - - if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil { - return err - } - - srcFile, err := os.Open(path) - if err != nil { - return err - } - defer srcFile.Close() - - dstFile, err := os.OpenFile(dstPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, info.Mode()) - if err != nil { - return err - } - defer dstFile.Close() - - _, err = dstFile.ReadFrom(srcFile) - return err - }) -} - func (v *vllmMetal) verifyInstallation(ctx context.Context) error { cmd := exec.CommandContext(ctx, v.pythonPath, "-c", "import vllm_metal") if err := cmd.Run(); err != nil { diff --git a/pkg/inference/platform/platform.go b/pkg/inference/platform/platform.go index bb794ec97..b33a958e4 100644 --- a/pkg/inference/platform/platform.go +++ b/pkg/inference/platform/platform.go @@ -19,10 +19,10 @@ func SupportsSGLang() bool { } // SupportsDiffusers returns true if diffusers is supported on the current platform. -// Diffusers is supported on Linux (for Docker/CUDA) and macOS (for MPS/Apple Silicon). +// Diffusers is supported on Linux (for Docker/CUDA) and macOS ARM64 (for MPS/Apple Silicon). +// Distribution is handled via a self-contained Python environment downloaded from Docker Hub. func SupportsDiffusers() bool { - // return runtime.GOOS == "linux" || runtime.GOOS == "darwin" - return runtime.GOOS == "linux" // Support for macOS disabled for now until we design a solution to distribute it via Docker Desktop. + return runtime.GOOS == "linux" || (runtime.GOOS == "darwin" && runtime.GOARCH == "arm64") } // SupportsVLLMMetal returns true if vllm-metal is supported on the current platform. diff --git a/pkg/inference/scheduling/installer.go b/pkg/inference/scheduling/installer.go index a41448831..a31a975ed 100644 --- a/pkg/inference/scheduling/installer.go +++ b/pkg/inference/scheduling/installer.go @@ -113,13 +113,12 @@ func (i *installer) run(ctx context.Context) { // existing installation) when files are present, to avoid triggering // a download. if i.deferredBackends[name] { - status := i.statuses[name] + // If the backend is already on disk from a previous session, + // verify it via installBackend which properly serializes with + // on-demand installs from wait(). if diskUsage, err := backend.GetDiskUsage(); err == nil && diskUsage > 0 { - if err := backend.Install(ctx, i.httpClient); err != nil { - status.err = err - close(status.failed) - } else { - close(status.installed) + if err := i.installBackend(ctx, name); err != nil { + i.log.Warn("Backend installation failed", "backend", name, "error", err) } } // If not on disk, leave channels open so wait() can trigger diff --git a/pkg/internal/utils/copydir.go b/pkg/internal/utils/copydir.go new file mode 100644 index 000000000..734ebc2b5 --- /dev/null +++ b/pkg/internal/utils/copydir.go @@ -0,0 +1,53 @@ +package utils + +import ( + "os" + "path/filepath" +) + +// CopyDir recursively copies a directory tree from src to dst, +// preserving file modes and symlinks. +func CopyDir(src, dst string) error { + return filepath.Walk(src, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + relPath, err := filepath.Rel(src, path) + if err != nil { + return err + } + dstPath := filepath.Join(dst, relPath) + + if info.IsDir() { + return os.MkdirAll(dstPath, info.Mode()) + } + + if info.Mode()&os.ModeSymlink == os.ModeSymlink { + link, err := os.Readlink(path) + if err != nil { + return err + } + return os.Symlink(link, dstPath) + } + + if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil { + return err + } + + srcFile, err := os.Open(path) + if err != nil { + return err + } + defer srcFile.Close() + + dstFile, err := os.OpenFile(dstPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, info.Mode()) + if err != nil { + return err + } + defer dstFile.Close() + + _, err = dstFile.ReadFrom(srcFile) + return err + }) +} diff --git a/pkg/routing/backends.go b/pkg/routing/backends.go index 3c8c92e26..3d0231171 100644 --- a/pkg/routing/backends.go +++ b/pkg/routing/backends.go @@ -2,6 +2,7 @@ package routing import ( "github.com/docker/model-runner/pkg/inference" + "github.com/docker/model-runner/pkg/inference/backends/diffusers" "github.com/docker/model-runner/pkg/inference/backends/llamacpp" "github.com/docker/model-runner/pkg/inference/backends/mlx" "github.com/docker/model-runner/pkg/inference/backends/vllm" @@ -31,6 +32,9 @@ type BackendsConfig struct { IncludeVLLM bool VLLMPath string VLLMMetalPath string + + IncludeDiffusers bool + DiffusersPath string } // DefaultBackendDefs returns BackendDef entries for the configured backends. @@ -69,5 +73,15 @@ func DefaultBackendDefs(cfg BackendsConfig) []BackendDef { }) } + if cfg.IncludeDiffusers { + defs = append(defs, BackendDef{ + Name: diffusers.Name, + Deferred: true, + Init: func(mm *models.Manager) (inference.Backend, error) { + return diffusers.New(cfg.Log, mm, sl(diffusers.Name), nil, cfg.DiffusersPath) + }, + }) + } + return defs } diff --git a/scripts/build-diffusers-tarball.sh b/scripts/build-diffusers-tarball.sh new file mode 100755 index 000000000..9082bf620 --- /dev/null +++ b/scripts/build-diffusers-tarball.sh @@ -0,0 +1,106 @@ +#!/bin/bash +# Build script for diffusers macOS/Linux tarball distribution +# Creates a self-contained tarball with a standalone Python 3.12 + diffusers packages. +# The result can be extracted anywhere and run without any system Python dependency. +# +# Usage: ./scripts/build-diffusers-tarball.sh +# DIFFUSERS_RELEASE - diffusers release tag (required) +# TARBALL - Output tarball path (required) +# +# Requirements: +# - uv (will be installed if missing) + +set -e + +DIFFUSERS_RELEASE="${1:?Usage: $0 }" +TARBALL_ARG="${2:?Usage: $0 }" +WORK_DIR=$(mktemp -d) + +echo "Building diffusers tarball for release: $DIFFUSERS_RELEASE" + +# Convert tarball path to absolute before we cd elsewhere +TARBALL="$(cd "$(dirname "$TARBALL_ARG")" && pwd)/$(basename "$TARBALL_ARG")" + +# Directory containing this script (project root is one level up) +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +cleanup() { + rm -rf "$WORK_DIR" +} +trap cleanup EXIT + +if ! command -v uv &> /dev/null; then + echo "Installing uv..." + curl -LsSf https://astral.sh/uv/install.sh | sh + export PATH="$HOME/.local/bin:$PATH" +fi + +# Install standalone Python 3.12 via uv (from python-build-standalone, relocatable) +echo "Installing standalone Python 3.12 via uv..." +uv python install 3.12 + +PYTHON_BIN=$(uv python find 3.12) +PYTHON_PREFIX=$(cd "$(dirname "$PYTHON_BIN")/.." && pwd) +echo "Using standalone Python from: $PYTHON_PREFIX" + +# Copy the standalone Python to our work area +PYTHON_DIR="$WORK_DIR/python" +cp -Rp "$PYTHON_PREFIX" "$PYTHON_DIR" + +# Remove the externally-managed marker so we can install packages into it +rm -f "$PYTHON_DIR/lib/python3.12/EXTERNALLY-MANAGED" + +DIFFUSERS_VERSION="0.36.0" +TORCH_VERSION="2.9.1" +TRANSFORMERS_VERSION="4.57.5" +ACCELERATE_VERSION="1.3.0" +SAFETENSORS_VERSION="0.5.2" +HUGGINGFACE_HUB_VERSION="0.34.0" +BITSANDBYTES_VERSION="0.49.1" +FASTAPI_VERSION="0.115.12" +UVICORN_VERSION="0.34.1" +PILLOW_VERSION="11.2.1" + +echo "Installing diffusers and dependencies..." +uv pip install --python "$PYTHON_DIR/bin/python3" --system \ + "diffusers==${DIFFUSERS_VERSION}" \ + "torch==${TORCH_VERSION}" \ + "transformers==${TRANSFORMERS_VERSION}" \ + "accelerate==${ACCELERATE_VERSION}" \ + "safetensors==${SAFETENSORS_VERSION}" \ + "huggingface_hub==${HUGGINGFACE_HUB_VERSION}" \ + "bitsandbytes==${BITSANDBYTES_VERSION}" \ + "fastapi==${FASTAPI_VERSION}" \ + "uvicorn[standard]==${UVICORN_VERSION}" \ + "pillow==${PILLOW_VERSION}" + +# Install the diffusers_server module from the project +echo "Installing diffusers_server module..." +SITE_PACKAGES="$PYTHON_DIR/lib/python3.12/site-packages" +cp -Rp "$PROJECT_ROOT/python/diffusers_server" "$SITE_PACKAGES/diffusers_server" + +# Strip files not needed at runtime to reduce tarball size +echo "Stripping unnecessary files..." +rm -rf "$PYTHON_DIR/include" +rm -rf "$PYTHON_DIR/share" +PYLIB="$PYTHON_DIR/lib/python3.12" +rm -rf "$PYLIB/test" "$PYLIB/tests" +rm -rf "$PYLIB/idlelib" "$PYLIB/idle_test" +rm -rf "$PYLIB/tkinter" "$PYLIB/turtledemo" +rm -rf "$PYLIB/ensurepip" +# Remove Tcl/Tk native libraries (we don't need tkinter at runtime) +rm -f "$PYTHON_DIR"/lib/libtcl*.dylib "$PYTHON_DIR"/lib/libtk*.dylib +rm -rf "$PYTHON_DIR"/lib/tcl* "$PYTHON_DIR"/lib/tk* +# Remove dev tools not needed at runtime +rm -f "$PYTHON_DIR"/bin/*-config "$PYTHON_DIR"/bin/idle* +find "$PYTHON_DIR" -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true + +echo "Packaging standalone Python with diffusers..." +tar -czf "$TARBALL" -C "$PYTHON_DIR" . + +SIZE=$(du -h "$TARBALL" | cut -f1) +echo "Created: $TARBALL ($SIZE)" +echo "" +echo "This tarball is fully self-contained (includes Python 3.12 + all packages)." +echo "To use: extract to a directory and run bin/python3 -m diffusers_server.server"