From 7cc2425c08660820a1e5c9ff8eeb71551664ed37 Mon Sep 17 00:00:00 2001
From: "M. Mansour" <3020010+marazik@users.noreply.github.com>
Date: Wed, 3 Jun 2026 17:41:56 -0400
Subject: [PATCH] feature(litellm): integrate docker profile and config
 overrides

---
 api/config/embedder.litellm.json  |  27 ++++
 api/config/generator.litellm.json | 220 ++++++++++++++++++++++++++++++
 docker-compose-litellm.yml        |   3 +
 3 files changed, 250 insertions(+)
 create mode 100644 api/config/embedder.litellm.json
 create mode 100644 api/config/generator.litellm.json

diff --git a/api/config/embedder.litellm.json b/api/config/embedder.litellm.json
new file mode 100644
index 000000000..bf23bc8ed
--- /dev/null
+++ b/api/config/embedder.litellm.json
@@ -0,0 +1,27 @@
+{
+  "embedder_ollama": {
+    "client_class": "OllamaClient",
+    "model_kwargs": {
+      "model": "nomic-embed-text"
+    }
+  },
+  "embedder": {
+    "client_class": "LiteLLMClient",
+    "initialize_kwargs": {
+      "api_key": "${LITELLM_API_KEY}",
+      "base_url": "${LITELLM_BASE_URL}"
+    },
+    "batch_size": 10,
+    "model_kwargs": {
+      "model": "nomic-embed-text"
+    }
+  },
+  "retriever": {
+    "top_k": 20
+  },
+  "text_splitter": {
+    "split_by": "word",
+    "chunk_size": 350,
+    "chunk_overlap": 100
+  }
+}
diff --git a/api/config/generator.litellm.json b/api/config/generator.litellm.json
new file mode 100644
index 000000000..7cb15b687
--- /dev/null
+++ b/api/config/generator.litellm.json
@@ -0,0 +1,220 @@
+{
+  "default_provider": "litellm",
+  "providers": {
+    "dashscope": {
+      "default_model": "qwen-plus",
+      "supportsCustomModel": true,
+      "models": {
+        "qwen-plus": {
+          "temperature": 0.7,
+          "top_p": 0.8
+        },
+        "qwen-turbo": {
+          "temperature": 0.7,
+          "top_p": 0.8
+        },
+        "deepseek-r1": {
+          "temperature": 0.7,
+          "top_p": 0.8
+        }
+      }
+    },
+    "google": {
+      "default_model": "gemini-2.5-flash",
+      "supportsCustomModel": true,
+      "models": {
+        "gemini-2.5-flash": {
+          "temperature": 1.0,
+          "top_p": 0.8,
+          "top_k": 20
+        },
+        "gemini-2.5-flash-lite": {
+          "temperature": 1.0,
+          "top_p": 0.8,
+          "top_k": 20
+        },
+        "gemini-2.5-pro": {
+          "temperature": 1.0,
+          "top_p": 0.8,
+          "top_k": 20
+        }
+      }
+    },
+    "openai": {
+      "default_model": "gpt-5-nano",
+      "supportsCustomModel": true,
+      "models": {
+        "gpt-5": {
+          "temperature": 1.0
+        },
+        "gpt-5-nano": {
+          "temperature": 1.0
+        },
+        "gpt-5-mini": {
+          "temperature": 1.0
+        },
+        "gpt-4o": {
+          "temperature": 0.7,
+          "top_p": 0.8
+        },
+        "gpt-4.1": {
+          "temperature": 0.7,
+          "top_p": 0.8
+        },
+        "o1": {
+          "temperature": 0.7,
+          "top_p": 0.8
+        },
+        "o3": {
+          "temperature": 1.0
+        },
+        "o4-mini": {
+          "temperature": 1.0
+        }
+      }
+    },
+    "litellm": {
+      "default_model": "qwen3:1.7b",
+      "supportsCustomModel": true,
+      "models": {
+        "qwen3:1.7b": {
+            "temperature": 0.7,
+            "top_p": 0.8,
+            "num_ctx": 32000
+        },
+        "llama3:8b": {
+            "temperature": 0.7,
+            "top_p": 0.8,
+            "num_ctx": 8000
+        },
+        "qwen3:8b": {
+            "temperature": 0.7,
+            "top_p": 0.8,
+            "num_ctx": 32000
+        }
+      }
+    },
+    "openrouter": {
+      "default_model": "openai/gpt-5-nano",
+      "supportsCustomModel": true,
+      "models": {
+        "openai/gpt-5-nano": {
+          "temperature": 0.7,
+          "top_p": 0.8
+        },
+        "openai/gpt-4o": {
+          "temperature": 0.7,
+          "top_p": 0.8
+        },
+        "deepseek/deepseek-r1": {
+          "temperature": 0.7,
+          "top_p": 0.8
+        },
+        "openai/gpt-4.1": {
+          "temperature": 0.7,
+          "top_p": 0.8
+        },
+        "openai/o1": {
+          "temperature": 0.7,
+          "top_p": 0.8
+        },
+        "openai/o3": {
+          "temperature": 1.0
+        },
+        "openai/o4-mini": {
+          "temperature": 1.0
+        },
+        "anthropic/claude-3.7-sonnet": {
+          "temperature": 0.7,
+          "top_p": 0.8
+        },
+        "anthropic/claude-3.5-sonnet": {
+          "temperature": 0.7,
+          "top_p": 0.8
+        }
+      }
+    },
+    "ollama": {
+      "default_model": "qwen3:1.7b",
+      "supportsCustomModel": true,
+      "models": {
+        "qwen3:1.7b": {
+          "options": {
+            "temperature": 0.7,
+            "top_p": 0.8,
+            "num_ctx": 32000
+          }
+        },
+        "llama3:8b": {
+          "options": {
+            "temperature": 0.7,
+            "top_p": 0.8,
+            "num_ctx": 8000
+          }
+        },
+        "qwen3:8b": {
+          "options": {
+            "temperature": 0.7,
+            "top_p": 0.8,
+            "num_ctx": 32000
+          }
+        }
+      }
+    },
+    "bedrock": {
+      "client_class": "BedrockClient",
+      "default_model": "anthropic.claude-3-sonnet-20240229-v1:0",
+      "supportsCustomModel": true,
+      "models": {
+        "anthropic.claude-3-sonnet-20240229-v1:0": {
+          "temperature": 0.7,
+          "top_p": 0.8
+        },
+        "anthropic.claude-3-haiku-20240307-v1:0": {
+          "temperature": 0.7,
+          "top_p": 0.8
+        },
+        "anthropic.claude-3-opus-20240229-v1:0": {
+          "temperature": 0.7,
+          "top_p": 0.8
+        },
+        "amazon.titan-text-express-v1": {
+          "temperature": 0.7,
+          "top_p": 0.8
+        },
+        "cohere.command-r-v1:0": {
+          "temperature": 0.7,
+          "top_p": 0.8
+        },
+        "ai21.j2-ultra-v1": {
+          "temperature": 0.7,
+          "top_p": 0.8
+        }
+      }
+    },
+    "azure": {
+      "client_class": "AzureAIClient",
+      "default_model": "gpt-4o",
+      "supportsCustomModel": true,
+      "models": {
+        "gpt-4o": {
+          "temperature": 0.7,
+          "top_p": 0.8
+        },
+        "gpt-4": {
+          "temperature": 0.7,
+          "top_p": 0.8
+        },
+        "gpt-35-turbo": {
+          "temperature": 0.7,
+          "top_p": 0.8
+        },
+        "gpt-4-turbo": {
+          "temperature": 0.7,
+          "top_p": 0.8
+        }
+      }
+    }
+  }
+}
+
diff --git a/docker-compose-litellm.yml b/docker-compose-litellm.yml
index 6f90878a2..a082aff81 100644
--- a/docker-compose-litellm.yml
+++ b/docker-compose-litellm.yml
@@ -53,6 +53,9 @@ services:
     volumes:
       - ~/.adalflow:/root/.adalflow      # Persist repository and embedding data
       - ./api/logs:/app/api/logs          # Persist log files across container restarts
+      # LiteLLM-specific config overrides
+      - ./api/config/generator.litellm.json:/app/api/config/generator.json:ro
+      - ./api/config/embedder.litellm.json:/app/api/config/embedder.json:ro
     # Resource limits for docker-compose up (not Swarm mode)
     mem_limit: 6g
     mem_reservation: 2g