eliott/anything-llm
1
0
Fork 0
mirror of https://github.com/Mintplex-Labs/anything-llm synced 2026-04-25 17:15:37 +02:00
Code Issues Packages Projects Releases Wiki Activity

Lemonade integration (#5077)

Browse Source 
* lemonade integration

* lemonade embedder

* log

* load model

* readme updates

* update embedder privacy entry
This commit is contained in:
Timothy Carambat
2026-02-27 11:02:38 -08:00
committed by GitHub
parent fc29461718
commit a6ba5a4034
33 changed files with 1728 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
README.md
View File

@@ -109,6 +109,7 @@ AnythingLLM divides your documents into objects called `workspaces`. A Workspace
- [Docker Model Runner](https://docs.docker.com/ai/model-runner/)
- [PrivateModeAI (chat models)](https://privatemode.ai/)
- [SambaNova Cloud (chat models)](https://cloud.sambanova.ai/)
- [Lemonade by AMD](https://lemonade-server.ai)
**Embedder models:**

5
docker/.env.example
View File

@@ -239,6 +239,11 @@ GID='1000'
# EMBEDDING_MODEL_PREF='baai/bge-m3'
# OPENROUTER_API_KEY=''
# EMBEDDING_ENGINE='lemonade'
# EMBEDDING_BASE_PATH='http://127.0.0.1:8000'
# EMBEDDING_MODEL_PREF='Qwen3-embedder'
# EMBEDDING_MODEL_MAX_CHUNK_LENGTH=8192
###########################################
######## Vector Database Selection ########
###########################################

227
frontend/src/components/EmbeddingSelection/LemonadeOptions/index.jsx Normal file
View File

@@ -0,0 +1,227 @@
import React, { useEffect, useState } from "react";
import System from "@/models/system";
import { LEMONADE_COMMON_URLS } from "@/utils/constants";
import { CaretDown, CaretUp, Info, CircleNotch } from "@phosphor-icons/react";
import { Tooltip } from "react-tooltip";
import useProviderEndpointAutoDiscovery from "@/hooks/useProviderEndpointAutoDiscovery";
import { cleanBasePath } from "@/components/LLMSelection/LemonadeOptions";
export default function LemonadeEmbeddingOptions({ settings }) {
const {
autoDetecting: loading,
basePath,
basePathValue,
showAdvancedControls,
setShowAdvancedControls,
handleAutoDetectClick,
} = useProviderEndpointAutoDiscovery({
provider: "lemonade",
initialBasePath: settings?.EmbeddingBasePath,
ENDPOINTS: LEMONADE_COMMON_URLS,
});
const [maxChunkLength, setMaxChunkLength] = useState(
settings?.EmbeddingModelMaxChunkLength || 8192
);
const handleMaxChunkLengthChange = (e) => {
setMaxChunkLength(Number(e.target.value));
};
return (
<div className="w-full flex flex-col gap-y-7">
<div className="w-full flex items-start gap-[36px] mt-1.5">
<LemonadeModelSelection settings={settings} basePath={basePath.value} />
<div className="flex flex-col w-60">
<div
data-tooltip-place="top"
data-tooltip-id="max-embedding-chunk-length-tooltip"
className="flex gap-x-1 items-center mb-3"
>
<label className="text-white text-sm font-semibold block">
Max embedding chunk length
</label>
<Info
size={16}
className="text-theme-text-secondary cursor-pointer"
/>
<Tooltip id="max-embedding-chunk-length-tooltip">
Maximum length of text chunks, in characters, for embedding.
</Tooltip>
</div>
<input
type="number"
name="EmbeddingModelMaxChunkLength"
className="border-none bg-theme-settings-input-bg text-white placeholder:text-theme-settings-input-placeholder text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
placeholder="8192"
min={1}
value={maxChunkLength}
onChange={handleMaxChunkLengthChange}
onScroll={(e) => e.target.blur()}
required={true}
autoComplete="off"
/>
</div>
</div>
<div className="flex justify-start mt-4">
<button
onClick={(e) => {
e.preventDefault();
setShowAdvancedControls(!showAdvancedControls);
}}
className="border-none text-theme-text-primary hover:text-theme-text-secondary flex items-center text-sm"
>
{showAdvancedControls ? "Hide" : "Show"} Manual Endpoint Input
{showAdvancedControls ? (
<CaretUp size={14} className="ml-1" />
) : (
<CaretDown size={14} className="ml-1" />
)}
</button>
</div>
<div hidden={!showAdvancedControls}>
<div className="w-full flex items-start gap-4">
<div className="flex flex-col w-[300px]">
<div className="flex justify-between items-center mb-2">
<div className="flex items-center gap-1">
<label className="text-white text-sm font-semibold">
Lemonade Base URL
</label>
<Info
size={18}
className="text-theme-text-secondary cursor-pointer"
data-tooltip-id="lemonade-base-url"
data-tooltip-content="Enter the URL where Lemonade is running."
/>
<Tooltip
id="lemonade-base-url"
place="top"
delayShow={300}
className="tooltip !text-xs !opacity-100"
style={{
maxWidth: "250px",
whiteSpace: "normal",
wordWrap: "break-word",
}}
/>
</div>
{loading ? (
<CircleNotch
size={16}
className="text-theme-text-secondary animate-spin"
/>
) : (
<>
{!basePathValue.value && (
<button
onClick={handleAutoDetectClick}
className="border-none bg-primary-button text-xs font-medium px-2 py-1 rounded-lg hover:bg-secondary hover:text-white shadow-[0_4px_14px_rgba(0,0,0,0.25)]"
>
Auto-Detect
</button>
)}
</>
)}
</div>
<input
type="url"
name="EmbeddingBasePath"
className="border-none bg-theme-settings-input-bg text-white placeholder:text-theme-settings-input-placeholder text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
placeholder="http://localhost:8000/live"
value={cleanBasePath(basePathValue.value)}
required={true}
autoComplete="off"
spellCheck={false}
onChange={basePath.onChange}
onBlur={basePath.onBlur}
/>
</div>
</div>
</div>
</div>
);
}
function LemonadeModelSelection({ settings, basePath = null }) {
const [customModels, setCustomModels] = useState([]);
const [loading, setLoading] = useState(true);
useEffect(() => {
async function findCustomModels() {
if (!basePath) {
setCustomModels([]);
setLoading(false);
return;
}
setLoading(true);
try {
const { models } = await System.customModels(
"lemonade-embedder",
null,
basePath
);
setCustomModels(models || []);
} catch (error) {
console.error("Failed to fetch custom models:", error);
setCustomModels([]);
}
setLoading(false);
}
findCustomModels();
}, [basePath]);
if (loading || customModels.length == 0) {
return (
<div className="flex flex-col w-60">
<label className="text-white text-sm font-semibold block mb-2">
Lemonade Embedding Model
</label>
<select
name="EmbeddingModelPref"
disabled={true}
className="border-none bg-theme-settings-input-bg border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
>
<option disabled={true} selected={true}>
{!!basePath
? "--loading available models--"
: "Enter Lemonade URL first"}
</option>
</select>
<p className="text-xs leading-[18px] font-base text-white text-opacity-60 mt-2">
Select the Lemonade model for embeddings. Models will load after
entering a valid Lemonade URL.
</p>
</div>
);
}
return (
<div className="flex flex-col w-60">
<label className="text-white text-sm font-semibold block mb-3">
Lemonade Embedding Model
</label>
<select
name="EmbeddingModelPref"
required={true}
className="border-none bg-theme-settings-input-bg border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
>
{customModels.length > 0 && (
<optgroup label="Your loaded models">
{customModels.map((model) => {
return (
<option
key={model.id}
value={model.id}
selected={settings.EmbeddingModelPref === model.id}
>
{model.id}
</option>
);
})}
</optgroup>
)}
</select>
</div>
);
}

409
frontend/src/components/LLMSelection/LemonadeOptions/index.jsx Normal file
View File

@@ -0,0 +1,409 @@
import { useState, useEffect } from "react";
import System from "@/models/system";
import useProviderEndpointAutoDiscovery from "@/hooks/useProviderEndpointAutoDiscovery";
import { CircleNotch, Info } from "@phosphor-icons/react";
import strDistance from "js-levenshtein";
import { LLM_PREFERENCE_CHANGED_EVENT } from "@/pages/GeneralSettings/LLMPreference";
import { LEMONADE_COMMON_URLS } from "@/utils/constants";
import { Tooltip } from "react-tooltip";
import { Link } from "react-router-dom";
import ModelTable from "@/components/lib/ModelTable";
import ModelTableLayout from "@/components/lib/ModelTable/layout";
import ModelTableLoadingSkeleton from "@/components/lib/ModelTable/loading";
import showToast from "@/utils/toast";
import LemonadeUtils from "@/models/utils/lemonadeUtils";
export function cleanBasePath(basePath = "") {
try {
const url = new URL(basePath);
return url.origin;
} catch {
return basePath;
}
}
export default function LemonadeOptions({ settings }) {
const {
autoDetecting: loading,
basePath,
basePathValue,
handleAutoDetectClick,
} = useProviderEndpointAutoDiscovery({
provider: "lemonade",
initialBasePath: settings?.LemonadeLLMBasePath,
ENDPOINTS: LEMONADE_COMMON_URLS,
});
const [selectedModelId, setSelectedModelId] = useState(
settings?.LemonadeLLMModelPref
);
const [maxTokens, setMaxTokens] = useState(
settings?.LemonadeLLMModelTokenLimit || 4096
);
return (
<div className="w-full flex flex-col gap-y-7">
<div className="flex gap-[36px] mt-1.5 flex-wrap">
<div className="flex flex-col w-60">
<div className="flex items-center gap-1 mb-3">
<div className="flex justify-between items-center gap-x-2">
<label className="text-white text-sm font-semibold">
Base URL
</label>
{loading ? (
<CircleNotch className="w-4 h-4 text-theme-text-secondary animate-spin" />
) : (
<>
{!basePathValue.value && (
<button
onClick={handleAutoDetectClick}
className="border-none bg-primary-button text-xs font-medium px-2 py-1 rounded-lg hover:bg-secondary hover:text-white shadow-[0_4px_14px_rgba(0,0,0,0.25)]"
>
Auto-Detect
</button>
)}
</>
)}
</div>
<Tooltip
id="lemonade-base-url"
place="top"
delayShow={300}
delayHide={800}
clickable={true}
className="tooltip !text-xs !opacity-100 z-99"
style={{
maxWidth: "250px",
whiteSpace: "normal",
wordWrap: "break-word",
}}
>
Enter the URL where the Lemonade is running.
<br />
<br />
You <b>must</b> have enabled the Lemonade TCP support for this to
work.
<br />
<br />
<Link
to="https://lemonade-server.ai/docs"
target="_blank"
className="text-blue-500 hover:underline"
>
Learn more &rarr;
</Link>
</Tooltip>
<div
className="text-theme-text-secondary cursor-pointer hover:bg-theme-bg-primary flex items-center justify-center rounded-full"
data-tooltip-id="lemonade-base-url"
data-tooltip-place="top"
data-tooltip-delay-hide={800}
>
<Info size={18} className="text-theme-text-secondary" />
</div>
</div>
<input
type="url"
name="LemonadeLLMBasePath"
className="border-none bg-theme-settings-input-bg text-white placeholder:text-theme-settings-input-placeholder text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
placeholder="http://localhost:8000"
value={cleanBasePath(basePathValue.value)}
required={true}
autoComplete="off"
spellCheck={false}
onChange={basePath.onChange}
onBlur={basePath.onBlur}
/>
</div>
<div className="flex flex-col w-60">
<div className="flex items-center gap-1 mb-3">
<label className="text-white text-sm font-semibold block">
Model context window
</label>
<Tooltip
id="lemonade-model-context-window"
place="top"
delayShow={300}
delayHide={800}
clickable={true}
className="tooltip !text-xs !opacity-100 z-99"
style={{
maxWidth: "350px",
whiteSpace: "normal",
wordWrap: "break-word",
}}
>
The maximum number of tokens that can be used for a model context
window. This must be set to a value that is supported by the
model.
</Tooltip>
<div
className="text-theme-text-secondary cursor-pointer hover:bg-theme-bg-primary flex items-center justify-center rounded-full"
data-tooltip-id="lemonade-model-context-window"
data-tooltip-place="top"
data-tooltip-delay-hide={800}
>
<Info size={18} className="text-theme-text-secondary" />
</div>
</div>
<input
type="number"
name="LemonadeLLMModelTokenLimit"
className="border-none bg-theme-settings-input-bg text-white placeholder:text-theme-settings-input-placeholder text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
placeholder="4096"
min={1}
value={maxTokens}
onChange={(e) => setMaxTokens(Number(e.target.value))}
onScroll={(e) => e.target.blur()}
required={true}
autoComplete="off"
/>
</div>
<LemonadeModelSelection
selectedModelId={selectedModelId}
setSelectedModelId={setSelectedModelId}
basePath={basePathValue.value}
/>
</div>
</div>
);
}
function LemonadeModelSelection({
selectedModelId,
setSelectedModelId,
basePath = null,
}) {
const [customModels, setCustomModels] = useState([]);
const [filteredModels, setFilteredModels] = useState([]);
const [loading, setLoading] = useState(true);
const [searchQuery, setSearchQuery] = useState("");
async function fetchModels() {
if (!basePath) {
setCustomModels([]);
setFilteredModels([]);
setLoading(false);
setSearchQuery("");
return;
}
setLoading(true);
const { models } = await System.customModels("lemonade", null, basePath);
setCustomModels(models || []);
setFilteredModels(models || []);
setSearchQuery("");
setLoading(false);
}
useEffect(() => {
fetchModels();
}, [basePath]);
useEffect(() => {
if (!searchQuery || !customModels.length) {
setFilteredModels(customModels || []);
return;
}
const normalizedSearchQuery = searchQuery.toLowerCase().trim();
const filteredModels = new Map();
customModels.forEach((model) => {
const modelNameNormalized = model.name.toLowerCase();
const modelOrganizationNormalized = model.organization.toLowerCase();
if (modelNameNormalized.startsWith(normalizedSearchQuery))
filteredModels.set(model.id, model);
if (modelOrganizationNormalized.startsWith(normalizedSearchQuery))
filteredModels.set(model.id, model);
if (strDistance(modelNameNormalized, normalizedSearchQuery) <= 2)
filteredModels.set(model.id, model);
if (strDistance(modelOrganizationNormalized, normalizedSearchQuery) <= 2)
filteredModels.set(model.id, model);
});
setFilteredModels(Array.from(filteredModels.values()));
}, [searchQuery]);
async function uninstallModel(modelId) {
try {
if (
!window.confirm(
`Are you sure you want to uninstall this model? You will need to download it again to use it.`
)
)
return;
const { success, error } = await LemonadeUtils.deleteModel(
modelId,
basePath
);
if (!success)
throw new Error(
error || "An error occurred while uninstalling the model"
);
const updatedModels = customModels.map((model) =>
model.id === modelId ? { ...model, downloaded: false } : model
);
setCustomModels(updatedModels);
setFilteredModels(updatedModels);
setSearchQuery("");
} catch (e) {
console.error("Error uninstalling model:", e);
showToast(
e.message || "An error occurred while uninstalling the model",
"error",
{ clear: true }
);
} finally {
setLoading(false);
}
}
async function downloadModel(modelId, fileSize, progressCallback) {
try {
if (
!window.confirm(
`Are you sure you want to download this model? It is ${fileSize} in size and may take a while to download.`
)
)
return;
const { success, error } = await LemonadeUtils.downloadModel(
modelId,
basePath,
progressCallback
);
if (!success)
throw new Error(
error || "An error occurred while downloading the model"
);
progressCallback(100);
handleSetActiveModel(modelId);
const existingModels = [...customModels];
const newModel = existingModels.find((model) => model.id === modelId);
if (newModel) {
newModel.downloaded = true;
setCustomModels(existingModels);
setFilteredModels(existingModels);
setSearchQuery("");
}
} catch (e) {
console.error("Error downloading model:", e);
showToast(
e.message || "An error occurred while downloading the model",
"error",
{ clear: true }
);
} finally {
setLoading(false);
}
}
function groupModelsByAlias(models) {
const mapping = new Map();
mapping.set("installed", new Map());
mapping.set("not installed", new Map());
const groupedModels = models.reduce((acc, model) => {
acc[model.organization] = acc[model.organization] || [];
acc[model.organization].push(model);
return acc;
}, {});
Object.entries(groupedModels).forEach(([organization, models]) => {
const hasInstalled = models.some((model) => model.downloaded);
if (hasInstalled) {
const installedModels = models.filter((model) => model.downloaded);
mapping
.get("installed")
.set("Downloaded Models", [
...(mapping.get("installed").get("Downloaded Models") || []),
...installedModels,
]);
}
const tags = models.map((model) => ({
...model,
name: model.name.split(":")[1],
}));
mapping.get("not installed").set(organization, tags);
});
const orderedMap = new Map();
const installedMap = new Map();
mapping
.get("installed")
.entries()
.forEach(([organization, models]) =>
installedMap.set(organization, models)
);
mapping
.get("not installed")
.entries()
.forEach(([organization, models]) =>
orderedMap.set(organization, models)
);
// Sort the models by organization/creator name alphabetically but keep the installed models at the top
return Object.fromEntries(
Array.from(installedMap.entries())
.sort((a, b) => a[0].localeCompare(b[0]))
.concat(
Array.from(orderedMap.entries()).sort((a, b) =>
a[0].localeCompare(b[0])
)
)
);
}
function handleSetActiveModel(modelId) {
if (modelId === selectedModelId) return;
setSelectedModelId(modelId);
window.dispatchEvent(new Event(LLM_PREFERENCE_CHANGED_EVENT));
}
const groupedModels = groupModelsByAlias(filteredModels);
return (
<ModelTableLayout
fetchModels={fetchModels}
searchQuery={searchQuery}
setSearchQuery={setSearchQuery}
loading={loading}
>
<Tooltip
id="install-model-tooltip"
place="top"
className="tooltip !text-xs !opacity-100 z-99"
/>
<input
type="hidden"
name="LemonadeLLMModelPref"
id="LemonadeLLMModelPref"
value={selectedModelId}
/>
{loading ? (
<ModelTableLoadingSkeleton />
) : filteredModels.length === 0 ? (
<div className="flex flex-col w-full gap-y-2 mt-4">
<p className="text-theme-text-secondary text-sm">No models found!</p>
</div>
) : (
Object.entries(groupedModels).map(([alias, models]) => (
<ModelTable
key={alias}
alias={alias}
models={models}
setActiveModel={handleSetActiveModel}
downloadModel={downloadModel}
selectedModelId={selectedModelId}
uninstallModel={uninstallModel}
ui={{
showRuntime: false,
}}
/>
))
)}
</ModelTableLayout>
);
}

15
frontend/src/components/ProviderPrivacy/constants.js
View File

@@ -44,6 +44,7 @@ import GiteeAILogo from "@/media/llmprovider/giteeai.png";
import DockerModelRunnerLogo from "@/media/llmprovider/docker-model-runner.png";
import PrivateModeLogo from "@/media/llmprovider/privatemode.png";
import SambaNovaLogo from "@/media/llmprovider/sambanova.png";
import LemonadeLogo from "@/media/llmprovider/lemonade.png";
const LLM_PROVIDER_PRIVACY_MAP = {
openai: {
@@ -244,6 +245,13 @@ const LLM_PROVIDER_PRIVACY_MAP = {
policyUrl: "https://sambanova.ai/privacy-policy",
logo: SambaNovaLogo,
},
lemonade: {
name: "Lemonade",
description: [
"Your model and chats are only accessible on the machine running the Lemonade server.",
],
logo: LemonadeLogo,
},
};
const VECTOR_DB_PROVIDER_PRIVACY_MAP = {
@@ -387,6 +395,13 @@ const EMBEDDING_ENGINE_PROVIDER_PRIVACY_MAP = {
policyUrl: "https://policies.google.com/privacy",
logo: GeminiLogo,
},
lemonade: {
name: "Lemonade",
description: [
"Your document text is embedded privately on the machine running the Lemonade server.",
],
logo: LemonadeLogo,
},
};
export const PROVIDER_PRIVACY_MAP = {

9
frontend/src/hooks/useProviderEndpointAutoDiscovery.js
View File

@@ -1,6 +1,5 @@
import { useEffect, useState } from "react";
import System from "@/models/system";
import showToast from "@/utils/toast";
export default function useProviderEndpointAutoDiscovery({
provider = null,
@@ -48,20 +47,12 @@ export default function useProviderEndpointAutoDiscovery({
setBasePath(endpoint);
setBasePathValue(endpoint);
setLoading(false);
showToast("Provider endpoint discovered automatically.", "success", {
clear: true,
});
setShowAdvancedControls(false);
return;
}
setLoading(false);
setShowAdvancedControls(true);
showToast(
"Couldn't automatically discover the provider endpoint. Please enter it manually.",
"info",
{ clear: true }
);
}
function handleAutoDetectClick(e) {

BIN
frontend/src/media/llmprovider/lemonade.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 49 KiB

115
frontend/src/models/utils/lemonadeUtils.js Normal file
View File

@@ -0,0 +1,115 @@
import { API_BASE } from "@/utils/constants";
import { baseHeaders } from "@/utils/request";
import { safeJsonParse } from "@/utils/request";
const LemonadeUtils = {
/**
* Download a Lemonade model.
* @param {string} modelId - The ID of the model to download.
* @param {(percentage: number) => void} progressCallback - The callback to receive the progress percentage. If the model is already downloaded, it will be called once with 100.
* @returns {Promise<{success: boolean, error: string|null}>}
*/
downloadModel: async function (
modelId,
basePath = "",
progressCallback = () => {}
) {
// eslint-disable-next-line no-async-promise-executor
return new Promise(async (resolve) => {
try {
const response = await fetch(
`${API_BASE}/utils/lemonade/download-model`,
{
method: "POST",
headers: baseHeaders(),
body: JSON.stringify({ modelId, basePath }),
}
);
if (!response.ok)
throw new Error("Error downloading model: " + response.statusText);
const reader = response.body.getReader();
let done = false;
while (!done) {
const { value, done: readerDone } = await reader.read();
if (readerDone) {
done = true;
resolve({ success: true });
} else {
const chunk = new TextDecoder("utf-8").decode(value);
const lines = chunk.split("\n");
for (const line of lines) {
if (line.startsWith("data:")) {
const data = safeJsonParse(line.slice(5));
switch (data?.type) {
case "success":
done = true;
resolve({ success: true });
break;
case "error":
done = true;
resolve({
success: false,
error: data?.error || data?.message,
});
break;
case "progress":
progressCallback(data?.percentage);
break;
default:
break;
}
}
}
}
}
} catch (error) {
console.error("Error downloading model:", error);
resolve({
success: false,
error:
error?.message || "An error occurred while downloading the model",
});
}
});
},
/**
* Delete a Lemonade model from local storage.
* If the model is currently loaded, it will be unloaded first.
* @param {string} modelId - The ID of the model to delete.
* @param {string} basePath - The base path of the Lemonade server.
* @returns {Promise<{success: boolean, message?: string, error?: string}>}
*/
deleteModel: async function (modelId, basePath = "") {
try {
const response = await fetch(`${API_BASE}/utils/lemonade/delete-model`, {
method: "POST",
headers: baseHeaders(),
body: JSON.stringify({ modelId, basePath }),
});
const data = await response.json();
if (!response.ok || !data.success) {
return {
success: false,
error: data.error || "An error occurred while deleting the model",
};
}
return {
success: true,
message: data.message,
};
} catch (error) {
console.error("Error deleting model:", error);
return {
success: false,
error: error?.message || "An error occurred while deleting the model",
};
}
},
};
export default LemonadeUtils;

12
frontend/src/pages/GeneralSettings/EmbeddingPreference/index.jsx
View File

@@ -16,6 +16,7 @@ import LiteLLMLogo from "@/media/llmprovider/litellm.png";
import GenericOpenAiLogo from "@/media/llmprovider/generic-openai.png";
import MistralAiLogo from "@/media/llmprovider/mistral.jpeg";
import OpenRouterLogo from "@/media/llmprovider/openrouter.jpeg";
import LemonadeLogo from "@/media/llmprovider/lemonade.png";
import PreLoader from "@/components/Preloader";
import ChangeWarningModal from "@/components/ChangeWarning";
@@ -31,6 +32,8 @@ import VoyageAiOptions from "@/components/EmbeddingSelection/VoyageAiOptions";
import LiteLLMOptions from "@/components/EmbeddingSelection/LiteLLMOptions";
import GenericOpenAiEmbeddingOptions from "@/components/EmbeddingSelection/GenericOpenAiOptions";
import OpenRouterOptions from "@/components/EmbeddingSelection/OpenRouterOptions";
import MistralAiOptions from "@/components/EmbeddingSelection/MistralAiOptions";
import LemonadeOptions from "@/components/EmbeddingSelection/LemonadeOptions";
import EmbedderItem from "@/components/EmbeddingSelection/EmbedderItem";
import { CaretUpDown, MagnifyingGlass, X } from "@phosphor-icons/react";
@@ -38,7 +41,6 @@ import { useModal } from "@/hooks/useModal";
import ModalWrapper from "@/components/ModalWrapper";
import CTAButton from "@/components/lib/CTAButton";
import { useTranslation } from "react-i18next";
import MistralAiOptions from "@/components/EmbeddingSelection/MistralAiOptions";
const EMBEDDERS = [
{
@@ -92,6 +94,14 @@ const EMBEDDERS = [
description:
"Discover, download, and run thousands of cutting edge LLMs in a few clicks.",
},
{
name: "Lemonade",
value: "lemonade",
logo: LemonadeLogo,
options: (settings) => <LemonadeOptions settings={settings} />,
description:
"Run embedding models locally on your own machine using Lemonade.",
},
{
name: "OpenRouter",
value: "openrouter",

11
frontend/src/pages/GeneralSettings/LLMPreference/index.jsx
View File

@@ -40,6 +40,7 @@ import GiteeAILogo from "@/media/llmprovider/giteeai.png";
import DockerModelRunnerLogo from "@/media/llmprovider/docker-model-runner.png";
import PrivateModeLogo from "@/media/llmprovider/privatemode.png";
import SambaNovaLogo from "@/media/llmprovider/sambanova.png";
import LemonadeLogo from "@/media/llmprovider/lemonade.png";
import PreLoader from "@/components/Preloader";
import OpenAiOptions from "@/components/LLMSelection/OpenAiOptions";
@@ -77,6 +78,7 @@ import GiteeAIOptions from "@/components/LLMSelection/GiteeAIOptions/index.jsx";
import DockerModelRunnerOptions from "@/components/LLMSelection/DockerModelRunnerOptions";
import PrivateModeOptions from "@/components/LLMSelection/PrivateModeOptions";
import SambaNovaOptions from "@/components/LLMSelection/SambaNovaOptions";
import LemonadeOptions from "@/components/LLMSelection/LemonadeOptions";
import LLMItem from "@/components/LLMSelection/LLMItem";
import { CaretUpDown, MagnifyingGlass, X } from "@phosphor-icons/react";
@@ -179,6 +181,15 @@ export const AVAILABLE_LLM_PROVIDERS = [
"DockerModelRunnerModelTokenLimit",
],
},
{
name: "Lemonade",
value: "lemonade",
logo: LemonadeLogo,
options: (settings) => <LemonadeOptions settings={settings} />,
description:
"Run local LLMs, ASR, TTS, and more in a single unified AI runtime.",
requiredConfig: ["LemonadeLLMBasePath"],
},
{
name: "SambaNova",
value: "sambanova",

10
frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx
View File

@@ -34,6 +34,7 @@ import GiteeAILogo from "@/media/llmprovider/giteeai.png";
import DockerModelRunnerLogo from "@/media/llmprovider/docker-model-runner.png";
import PrivateModeLogo from "@/media/llmprovider/privatemode.png";
import SambaNovaLogo from "@/media/llmprovider/sambanova.png";
import LemonadeLogo from "@/media/llmprovider/lemonade.png";
import OpenAiOptions from "@/components/LLMSelection/OpenAiOptions";
import GenericOpenAiOptions from "@/components/LLMSelection/GenericOpenAiOptions";
@@ -69,6 +70,7 @@ import GiteeAiOptions from "@/components/LLMSelection/GiteeAIOptions";
import DockerModelRunnerOptions from "@/components/LLMSelection/DockerModelRunnerOptions";
import PrivateModeOptions from "@/components/LLMSelection/PrivateModeOptions";
import SambaNovaOptions from "@/components/LLMSelection/SambaNovaOptions";
import LemonadeOptions from "@/components/LLMSelection/LemonadeOptions";
import LLMItem from "@/components/LLMSelection/LLMItem";
import System from "@/models/system";
@@ -152,6 +154,14 @@ const LLMS = [
options: (settings) => <DockerModelRunnerOptions settings={settings} />,
description: "Run LLMs using Docker Model Runner.",
},
{
name: "Lemonade",
value: "lemonade",
logo: LemonadeLogo,
options: (settings) => <LemonadeOptions settings={settings} />,
description:
"Run local LLMs, ASR, TTS, and more in a single unified AI runtime.",
},
{
name: "Local AI",
value: "localai",

1
frontend/src/pages/WorkspaceSettings/AgentConfig/AgentLLMSelection/index.jsx
View File

@@ -40,6 +40,7 @@ const ENABLED_PROVIDERS = [
"docker-model-runner",
"privatemode",
"sambanova",
"lemonade",
// TODO: More agent support.
// "huggingface" // Can be done but already has issues with no-chat templated. Needs to be tested.
];

7
frontend/src/utils/constants.js
View File

@@ -62,6 +62,13 @@ export const DOCKER_MODEL_RUNNER_COMMON_URLS = [
"http://172.17.0.1:12434/engines/llama.cpp/v1",
];
export const LEMONADE_COMMON_URLS = [
"http://localhost:8000/live",
"http://127.0.0.1:8000/live",
"http://host.docker.internal:8000/live",
"http://172.17.0.1:8000/live",
];
export function fullApiUrl() {
if (API_BASE !== "/api") return API_BASE;
return `${window.location.origin}/api`;

1
locales/README.fa-IR.md
View File

@@ -109,6 +109,7 @@ AnythingLLM اسناد شما را به اشیایی به نام `workspaces` ت
- [Docker Model Runner](https://docs.docker.com/ai/model-runner/)
- [PrivateModeAI (chat models)](https://privatemode.ai/)
- [SambaNova Cloud (chat models)](https://cloud.sambanova.ai/)
- [Lemonade by AMD](https://lemonade-server.ai)
<div dir="rtl">

1
locales/README.ja-JP.md
View File

@@ -97,6 +97,7 @@ AnythingLLMは、ドキュメントを`ワークスペース`と呼ばれるオ
- [Docker Model Runner](https://docs.docker.com/ai/model-runner/)
- [PrivateModeAI (chat models)](https://privatemode.ai/)
- [SambaNova Cloud (chat models)](https://cloud.sambanova.ai/)
- [Lemonade by AMD](https://lemonade-server.ai)
**埋め込みモデル：**

1
locales/README.tr-TR.md
View File

@@ -104,6 +104,7 @@ AnythingLLM, belgelerinizi **"çalışma alanları" (workspaces)** adı verilen
- [Docker Model Runner](https://docs.docker.com/ai/model-runner/)
- [PrivateModeAI (chat models)](https://privatemode.ai/)
- [SambaNova Cloud (chat models)](https://cloud.sambanova.ai/)
- [Lemonade by AMD](https://lemonade-server.ai)
**Embedder modelleri:**

1
locales/README.zh-CN.md
View File

@@ -106,6 +106,7 @@ AnythingLLM将您的文档划分为称为`workspaces` (工作区)的对象。工
- [Docker Model Runner](https://docs.docker.com/ai/model-runner/)
- [PrivateModeAI (chat models)](https://privatemode.ai/)
- [SambaNova Cloud (chat models)](https://cloud.sambanova.ai/)
- [Lemonade by AMD](https://lemonade-server.ai)
**支持的嵌入模型：**

5
server/.env.example
View File

@@ -238,6 +238,11 @@ SIG_SALT='salt' # Please generate random string at least 32 chars long.
# EMBEDDING_MODEL_PREF='baai/bge-m3'
# OPENROUTER_API_KEY=''
# EMBEDDING_ENGINE='lemonade'
# EMBEDDING_BASE_PATH='http://127.0.0.1:8000'
# EMBEDDING_MODEL_PREF='Qwen3-embedder'
# EMBEDDING_MODEL_MAX_CHUNK_LENGTH=8192
###########################################
######## Vector Database Selection ########
###########################################

6
server/endpoints/utils.js
View File

@@ -26,6 +26,9 @@ function utilEndpoints(app) {
dockerModelRunnerUtilsEndpoints,
} = require("./utils/dockerModelRunnerUtils");
dockerModelRunnerUtilsEndpoints(app);
const { lemonadeUtilsEndpoints } = require("./utils/lemonadeUtilsEndpoints");
lemonadeUtilsEndpoints(app);
}
function getGitVersion() {
@@ -168,6 +171,9 @@ function getModelTag() {
case "sambanova":
model = process.env.SAMBANOVA_LLM_MODEL_PREF;
break;
case "lemonade":
model = process.env.LEMONADE_LLM_MODEL_PREF;
break;
default:
model = "--";
break;

18
server/endpoints/utils/dockerModelRunnerUtils.js
View File

@@ -4,23 +4,7 @@ const {
ROLES,
} = require("../../utils/middleware/multiUserProtected");
const { reqBody } = require("../../utils/http");
const { safeJsonParse } = require("../../utils/http");
/**
* Decode HTML entities from a string.
* The DMR response is encoded with HTML entities, so we need to decode them
* so we can parse the JSON and report the progress percentage.
* @param {string} str - The string to decode.
* @returns {string} The decoded string.
*/
function decodeHtmlEntities(str) {
return str
.replace(/&#34;/g, '"')
.replace(/&#39;/g, "'")
.replace(/&lt;/g, "<")
.replace(/&gt;/g, ">")
.replace(/&amp;/g, "&");
}
const { safeJsonParse, decodeHtmlEntities } = require("../../utils/http");
function dockerModelRunnerUtilsEndpoints(app) {
if (!app) return;

164
server/endpoints/utils/lemonadeUtilsEndpoints.js Normal file
View File

@@ -0,0 +1,164 @@
const { validatedRequest } = require("../../utils/middleware/validatedRequest");
const {
flexUserRoleValid,
ROLES,
} = require("../../utils/middleware/multiUserProtected");
const { reqBody } = require("../../utils/http");
const { safeJsonParse, decodeHtmlEntities } = require("../../utils/http");
const {
parseLemonadeServerEndpoint,
} = require("../../utils/AiProviders/lemonade");
function lemonadeUtilsEndpoints(app) {
if (!app) return;
app.post(
"/utils/lemonade/download-model",
[validatedRequest, flexUserRoleValid([ROLES.admin])],
async (request, response) => {
try {
const { modelId, basePath = "" } = reqBody(request);
const lemonadeUrl = new URL(
parseLemonadeServerEndpoint(
basePath ?? process.env.LEMONADE_LLM_BASE_PATH,
"base"
)
);
lemonadeUrl.pathname += "api/v1/pull";
response.writeHead(200, {
"Content-Type": "text/event-stream",
"Cache-Control": "no-cache",
Connection: "keep-alive",
});
const lemonadeResponse = await fetch(lemonadeUrl.toString(), {
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify({
model_name: String(modelId),
stream: true,
}),
});
if (!lemonadeResponse.ok)
throw new Error(
lemonadeResponse.statusText ||
"An error occurred while downloading the model"
);
const reader = lemonadeResponse.body.getReader();
let done = false;
let currentEvent = null;
while (!done) {
const { value, done: readerDone } = await reader.read();
if (readerDone) done = true;
const chunk = new TextDecoder("utf-8").decode(value);
const lines = chunk.split("\n");
for (const line of lines) {
if (!line.trim()) continue;
if (line.startsWith("event:")) {
currentEvent = line.replace("event:", "").trim();
continue;
}
if (line.startsWith("data:")) {
const jsonStr = line.replace("data:", "").trim();
const decodedLine = decodeHtmlEntities(jsonStr);
const data = safeJsonParse(decodedLine);
if (!data) continue;
if (currentEvent === "error") {
throw new Error(
data.message ||
"An error occurred while downloading the model"
);
} else if (currentEvent === "complete") {
response.write(
`data: ${JSON.stringify({ type: "success", percentage: 100, message: "Model downloaded successfully" })}\n\n`
);
done = true;
} else if (currentEvent === "progress") {
const percentage = data.percent ?? 0;
const message = data.file
? `Downloading ${data.file}`
: "Downloading model...";
response.write(
`data: ${JSON.stringify({ type: "progress", percentage, message })}\n\n`
);
}
currentEvent = null;
}
}
}
} catch (e) {
console.error(e);
response.write(
`data: ${JSON.stringify({ type: "error", message: e.message })}\n\n`
);
} finally {
response.end();
}
}
);
app.post(
"/utils/lemonade/delete-model",
[validatedRequest, flexUserRoleValid([ROLES.admin])],
async (request, response) => {
try {
const { modelId, basePath = "" } = reqBody(request);
if (!modelId) {
return response.status(400).json({
success: false,
error: "modelId is required",
});
}
const lemonadeUrl = new URL(
parseLemonadeServerEndpoint(
basePath ?? process.env.LEMONADE_LLM_BASE_PATH,
"base"
)
);
lemonadeUrl.pathname += "api/v1/delete";
const lemonadeResponse = await fetch(lemonadeUrl.toString(), {
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify({
model_name: String(modelId),
}),
});
const data = await lemonadeResponse.json();
if (!lemonadeResponse.ok || data.status === "error") {
return response.status(lemonadeResponse.status || 500).json({
success: false,
error: data.message || "An error occurred while deleting the model",
});
}
return response.status(200).json({
success: true,
message: data.message || `Deleted model: ${modelId}`,
});
} catch (e) {
console.error(e);
return response.status(500).json({
success: false,
error: e.message || "An error occurred while deleting the model",
});
}
}
);
}
module.exports = {
lemonadeUtilsEndpoints,
};

6
server/models/systemSettings.js
View File

@@ -688,6 +688,12 @@ const SystemSettings = {
// SambaNova Keys
SambaNovaLLMApiKey: !!process.env.SAMBANOVA_LLM_API_KEY,
SambaNovaLLMModelPref: process.env.SAMBANOVA_LLM_MODEL_PREF,
// Lemonade Keys
LemonadeLLMBasePath: process.env.LEMONADE_LLM_BASE_PATH,
LemonadeLLMModelPref: process.env.LEMONADE_LLM_MODEL_PREF,
LemonadeLLMModelTokenLimit:
process.env.LEMONADE_LLM_MODEL_TOKEN_LIMIT || 8192,
};
},

380
server/utils/AiProviders/lemonade/index.js Normal file
View File

@@ -0,0 +1,380 @@
const { NativeEmbedder } = require("../../EmbeddingEngines/native");
const {
handleDefaultStreamResponseV2,
formatChatHistory,
} = require("../../helpers/chat/responses");
const {
LLMPerformanceMonitor,
} = require("../../helpers/chat/LLMPerformanceMonitor");
const { OpenAI: OpenAIApi } = require("openai");
const { humanFileSize } = require("../../helpers");
class LemonadeLLM {
constructor(embedder = null, modelPreference = null) {
if (!process.env.LEMONADE_LLM_BASE_PATH)
throw new Error("No Lemonade API Base Path was set.");
if (!process.env.LEMONADE_LLM_MODEL_PREF && !modelPreference)
throw new Error("No Lemonade Model Pref was set.");
this.className = "LemonadeLLM";
this.lemonade = new OpenAIApi({
baseURL: parseLemonadeServerEndpoint(
process.env.LEMONADE_LLM_BASE_PATH,
"openai"
),
apiKey: null,
});
this.model = modelPreference || process.env.LEMONADE_LLM_MODEL_PREF;
this.embedder = embedder ?? new NativeEmbedder();
this.defaultTemp = 0.7;
// We can establish here since we cannot dynamically curl the context window limit from the API.
this.limits = {
history: this.promptWindowLimit() * 0.15,
system: this.promptWindowLimit() * 0.15,
user: this.promptWindowLimit() * 0.7,
};
this.#log(`initialized with model: ${this.model}`);
}
#log(text, ...args) {
console.log(`\x1b[32m[Lemonade]\x1b[0m ${text}`, ...args);
}
static slog(text, ...args) {
console.log(`\x1b[32m[Lemonade]\x1b[0m ${text}`, ...args);
}
async assertModelContextLimits() {
if (this.limits !== null) return;
this.limits = {
history: this.promptWindowLimit() * 0.15,
system: this.promptWindowLimit() * 0.15,
user: this.promptWindowLimit() * 0.7,
};
this.#log(
`${this.model} is using a max context window of ${this.promptWindowLimit()} tokens.`
);
}
#appendContext(contextTexts = []) {
if (!contextTexts || !contextTexts.length) return "";
return (
"\nContext:\n" +
contextTexts
.map((text, i) => {
return `[CONTEXT ${i}]:\n${text}\n[END CONTEXT ${i}]\n\n`;
})
.join("")
);
}
streamingEnabled() {
return "streamGetChatCompletion" in this;
}
/** Lemonade does not support curling the context window limit from the API, so we return the system defined limit. */
static promptWindowLimit(_) {
return Number(process.env.LEMONADE_LLM_MODEL_TOKEN_LIMIT) || 8192;
}
promptWindowLimit() {
return this.constructor.promptWindowLimit(this.model);
}
async isValidChatCompletionModel(_ = "") {
return true;
}
/**
* Generates appropriate content array for a message + attachments.
* @param {{userPrompt:string, attachments: import("../../helpers").Attachment[]}}
* @returns {string|object[]}
*/
#generateContent({ userPrompt, attachments = [] }) {
if (!attachments.length) {
return userPrompt;
}
const content = [{ type: "text", text: userPrompt }];
for (let attachment of attachments) {
content.push({
type: "image_url",
image_url: {
url: attachment.contentString,
detail: "auto",
},
});
}
return content.flat();
}
/**
* Construct the user prompt for this model.
* @param {{attachments: import("../../helpers").Attachment[]}} param0
* @returns
*/
constructPrompt({
systemPrompt = "",
contextTexts = [],
chatHistory = [],
userPrompt = "",
attachments = [],
}) {
const prompt = {
role: "system",
content: `${systemPrompt}${this.#appendContext(contextTexts)}`,
};
return [
prompt,
...formatChatHistory(chatHistory, this.#generateContent),
{
role: "user",
content: this.#generateContent({ userPrompt, attachments }),
},
];
}
async getChatCompletion(messages = null, { temperature = 0.7 }) {
await LemonadeLLM.loadModel(this.model);
const result = await LLMPerformanceMonitor.measureAsyncFunction(
this.lemonade.chat.completions.create({
model: this.model,
messages,
temperature,
})
);
if (
!result.output.hasOwnProperty("choices") ||
result.output.choices.length === 0
)
return null;
return {
textResponse: result.output.choices[0].message.content,
metrics: {
prompt_tokens: result.output.usage?.prompt_tokens || 0,
completion_tokens: result.output.usage?.completion_tokens || 0,
total_tokens: result.output.usage?.total_tokens || 0,
outputTps: result.output.usage?.completion_tokens / result.duration,
duration: result.duration,
model: this.model,
provider: this.className,
timestamp: new Date(),
},
};
}
async streamGetChatCompletion(messages = null, { temperature = 0.7 }) {
await LemonadeLLM.loadModel(this.model);
const measuredStreamRequest = await LLMPerformanceMonitor.measureStream({
func: this.lemonade.chat.completions.create({
model: this.model,
stream: true,
messages,
temperature,
}),
messages,
runPromptTokenCalculation: true,
modelTag: this.model,
provider: this.className,
});
return measuredStreamRequest;
}
handleStream(response, stream, responseProps) {
return handleDefaultStreamResponseV2(response, stream, responseProps);
}
/**
* Returns the capabilities of the model.
* Note: This is a heuristic approach to get the capabilities of the model based on the model metadata.
* It is not perfect, but works since every model metadata is different and may not have key values we rely on.
* There is no "capabilities" key in the metadata via any API endpoint - so we do this.
* @returns {Promise<{tools: 'unknown' | boolean, reasoning: 'unknown' | boolean, imageGeneration: 'unknown' | boolean, vision: 'unknown' | boolean}>}
*/
async getModelCapabilities() {
try {
const client = new OpenAIApi({
baseURL: parseLemonadeServerEndpoint(
process.env.LEMONADE_LLM_BASE_PATH,
"openai"
),
apiKey: null,
});
const { labels = [] } = await client.models.retrieve(this.model);
return {
tools: labels.includes("tool-calling"),
reasoning: labels.includes("reasoning"),
imageGeneration: "unknown",
vision: labels.includes("vision"),
};
} catch (error) {
console.error("Error getting model capabilities:", error);
return {
tools: "unknown",
reasoning: "unknown",
imageGeneration: "unknown",
vision: "unknown",
};
}
}
/**
* Utility function to load a model from the Lemonade server.
* Does not check if the model is already loaded or unloads any models.
* @param {*} model
*/
static async loadModel(model, basePath = process.env.LEMONADE_LLM_BASE_PATH) {
try {
const endpoint = new URL(parseLemonadeServerEndpoint(basePath, "openai"));
endpoint.pathname += "/load";
console.log(endpoint.toString());
LemonadeLLM.slog(
`Loading model ${model} with context size ${this.promptWindowLimit()}`
);
await fetch(endpoint.toString(), {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
model_name: String(model),
ctx_size: Number(this.promptWindowLimit()),
}),
})
.then((response) => {
if (!response.ok)
throw new Error(
`Failed to load model ${model}: ${response.statusText}`
);
return response.json();
})
.then((data) => {
if (data.status !== "success") throw new Error(data.message);
LemonadeLLM.slog(`Model ${model} loaded successfully`);
return true;
});
} catch (error) {
LemonadeLLM.slog(`Error loading model ${model}:`, error);
return false;
}
}
// Simple wrapper for dynamic embedder & normalize interface for all LLM implementations
async embedTextInput(textInput) {
return await this.embedder.embedTextInput(textInput);
}
async embedChunks(textChunks = []) {
return await this.embedder.embedChunks(textChunks);
}
async compressMessages(promptArgs = {}, rawHistory = []) {
await this.assertModelContextLimits();
const { messageArrayCompressor } = require("../../helpers/chat");
const messageArray = this.constructPrompt(promptArgs);
return await messageArrayCompressor(this, messageArray, rawHistory);
}
}
/**
* Extracts the model family/organization name from a model ID.
* For example:
* - "Qwen3-VL-8B-Instruct-GGUF" → "Qwen"
* - "SmolLM3-3B-GGUF" → "SmolLM"
* - "Llama-3.2-8B" → "Llama"
* - "DeepSeek-V3-GGUF" → "DeepSeek"
* @param {string} modelId - The model identifier
* @returns {string} The organization/family name
*/
function extractModelOrganization(modelId) {
const match = modelId.match(/^([A-Za-z]+)/);
return match ? match[1] : modelId;
}
/**
* Parse the base path of the Docker Model Runner endpoint and return the host and port.
* @param {string} basePath - The base path of the Lemonade server endpoint.
* @param {'base' | 'openai' | 'ollama'} to - The provider to parse the endpoint for (internal DMR or openai-compatible)
* @returns {string | null}
*/
function parseLemonadeServerEndpoint(basePath = null, to = "openai") {
if (!basePath) return null;
try {
const url = new URL(basePath);
if (to === "openai") url.pathname = "api/v1";
else if (to === "ollama") url.pathname = "api";
else if (to === "base") url.pathname = ""; // only used for /live
return url.toString();
} catch (e) {
return basePath;
}
}
/**
* This function will fetch the remote models from the Lemonade server as well
* as the local models installed on the system.
* @param {string} basePath - The base path of the Lemonade server endpoint.
* @param {'chat' | 'embedding' | 'reranking'} task - The task to fetch the models for.
*/
async function getAllLemonadeModels(basePath = null, task = "chat") {
const availableModels = {};
function isValidForTask(model) {
if (task === "reranking") return model.labels?.includes("reranking");
if (task === "embedding") return model.labels?.includes("embeddings");
if (task === "chat")
return !["embeddings", "reranking"].some((label) =>
model.labels?.includes(label)
);
return true;
}
try {
// Grab the locally installed models from the Lemonade server API
const lemonadeUrl = new URL(
parseLemonadeServerEndpoint(
basePath ?? process.env.LEMONADE_LLM_BASE_PATH,
"openai"
)
);
lemonadeUrl.pathname += "/models";
lemonadeUrl.searchParams.append("show_all", "true");
await fetch(lemonadeUrl.toString())
.then((res) => res.json())
.then(({ data }) => {
data?.forEach((model) => {
if (!isValidForTask(model)) return;
const organization = extractModelOrganization(model.id);
const modelData = {
id: model.id,
name: organization + ":" + model.id,
// Reports in GB, convert to bytes
size: model?.size
? humanFileSize(model.size * 1024 ** 3)
: "Unknown size",
downloaded: model?.downloaded ?? false,
organization,
};
if (!availableModels[organization])
availableModels[organization] = { tags: [] };
availableModels[organization].tags.push(modelData);
});
});
} catch (e) {
LemonadeLLM.slog(`Error getting Lemonade models`, e);
} finally {
return Object.values(availableModels).flatMap((m) => m.tags);
}
}
module.exports = {
LemonadeLLM,
parseLemonadeServerEndpoint,
getAllLemonadeModels,
};

60
server/utils/EmbeddingEngines/lemonade/index.js Normal file
View File

@@ -0,0 +1,60 @@
const { toChunks } = require("../../helpers");
const { parseLemonadeServerEndpoint } = require("../../AiProviders/lemonade");
class LemonadeEmbedder {
constructor() {
if (!process.env.EMBEDDING_BASE_PATH)
throw new Error("No Lemonade API Base Path was set.");
if (!process.env.EMBEDDING_MODEL_PREF)
throw new Error("No Embedding Model Pref was set.");
this.className = "LemonadeEmbedder";
const { OpenAI: OpenAIApi } = require("openai");
this.lemonade = new OpenAIApi({
baseURL: parseLemonadeServerEndpoint(
process.env.EMBEDDING_BASE_PATH,
"openai"
),
apiKey: null,
});
this.model = process.env.EMBEDDING_MODEL_PREF;
this.embeddingMaxChunkLength =
process.env.EMBEDDING_MODEL_MAX_CHUNK_LENGTH || 8_191;
}
log(text, ...args) {
console.log(`\x1b[36m[${this.className}]\x1b[0m ${text}`, ...args);
}
async embedTextInput(textInput) {
try {
this.log(`Embedding text input...`);
const response = await this.lemonade.embeddings.create({
model: this.model,
input: textInput,
});
return response?.data[0]?.embedding || [];
} catch (error) {
console.error("Failed to get embedding from Lemonade.", error.message);
return [];
}
}
async embedChunks(textChunks = []) {
try {
this.log(`Embedding ${textChunks.length} chunks of text...`);
const response = await this.lemonade.embeddings.create({
model: this.model,
input: textChunks,
});
return response?.data?.map((emb) => emb.embedding) || [];
} catch (error) {
console.error("Failed to get embeddings from Lemonade.", error.message);
return new Array(textChunks.length).fill([]);
}
}
}
module.exports = {
LemonadeEmbedder,
};

2
server/utils/agents/aibitat/index.js
View File

@@ -1037,6 +1037,8 @@ ${this.getHistory({ to: route.to })
return new Providers.PrivatemodeProvider({ model: config.model });
case "sambanova":
return new Providers.SambaNovaProvider({ model: config.model });
case "lemonade":
return new Providers.LemonadeProvider({ model: config.model });
default:
throw new Error(
`Unknown provider: ${config.provider}. Please use a valid provider.`

8
server/utils/agents/aibitat/providers/ai-provider.js
View File

@@ -342,6 +342,14 @@ class Provider {
apiKey: null,
...config,
});
case "lemonade":
return new ChatOpenAI({
configuration: {
baseURL: process.env.LEMONADE_LLM_BASE_PATH,
},
apiKey: null,
...config,
});
default:
throw new Error(`Unsupported provider ${provider} for this task.`);
}

2
server/utils/agents/aibitat/providers/index.js
View File

@@ -32,6 +32,7 @@ const CohereProvider = require("./cohere.js");
const DockerModelRunnerProvider = require("./dockerModelRunner.js");
const PrivatemodeProvider = require("./privatemode.js");
const SambaNovaProvider = require("./sambanova.js");
const LemonadeProvider = require("./lemonade.js");
module.exports = {
OpenAIProvider,
@@ -68,4 +69,5 @@ module.exports = {
DockerModelRunnerProvider,
PrivatemodeProvider,
SambaNovaProvider,
LemonadeProvider,
};

211
server/utils/agents/aibitat/providers/lemonade.js Normal file
View File

@@ -0,0 +1,211 @@
const OpenAI = require("openai");
const Provider = require("./ai-provider.js");
const InheritMultiple = require("./helpers/classes.js");
const UnTooled = require("./helpers/untooled.js");
const { tooledStream, tooledComplete } = require("./helpers/tooled.js");
const { RetryError } = require("../error.js");
const {
LemonadeLLM,
parseLemonadeServerEndpoint,
} = require("../../../AiProviders/lemonade/index.js");
/**
* The agent provider for the Lemonade.
*/
class LemonadeProvider extends InheritMultiple([Provider, UnTooled]) {
model;
/**
*
* @param {{model?: string}} config
*/
constructor(config = {}) {
super();
const model = config?.model || process.env.LEMONADE_LLM_MODEL_PREF || null;
const client = new OpenAI({
baseURL: parseLemonadeServerEndpoint(
process.env.LEMONADE_LLM_BASE_PATH,
"openai"
),
apiKey: null,
maxRetries: 3,
});
this._client = client;
this.model = model;
this.verbose = true;
this.preloaded = false;
this._supportsToolCalling = null;
}
get client() {
return this._client;
}
get supportsAgentStreaming() {
return true;
}
async preloadModel() {
if (this.preloaded) return;
await LemonadeLLM.loadModel(this.model);
this.preloaded = true;
}
/**
* Whether this provider supports native OpenAI-compatible tool calling.
* - Since Lemonade models vary in tool calling support, we check the ENV.
* - If the ENV is not set and the capabilities are not set, we default to false.
* - To enable tool calling for a model, set the ENV flag for `PROVIDER_SUPPORTS_NATIVE_TOOL_CALLING` to include `lemonade`.
* - or update the label in the Lemonade server to include `tool-calling`.
* @returns {boolean|Promise<boolean>}
*/
async supportsNativeToolCalling() {
if (this._supportsToolCalling !== null) return this._supportsToolCalling;
const lemonade = new LemonadeLLM(null, this.model);
// Labels can be missing for tool calling models, so we also check if ENV flag is set
const supportsToolCallingFlag =
process.env.PROVIDER_SUPPORTS_NATIVE_TOOL_CALLING?.includes("lemonade");
if (supportsToolCallingFlag) {
this.providerLog(
"Lemonade supports native tool calling is ENABLED via ENV."
);
this._supportsToolCalling = true;
return this._supportsToolCalling;
}
const capabilities = await lemonade.getModelCapabilities();
this._supportsToolCalling = capabilities.tools === true;
return this._supportsToolCalling;
}
async #handleFunctionCallChat({ messages = [] }) {
return await this.client.chat.completions
.create({
model: this.model,
messages,
})
.then((result) => {
if (!result.hasOwnProperty("choices"))
throw new Error("Lemonade chat: No results!");
if (result.choices.length === 0)
throw new Error("Lemonade chat: No results length!");
return result.choices[0].message.content;
})
.catch((_) => {
return null;
});
}
async #handleFunctionCallStream({ messages = [] }) {
return await this.client.chat.completions.create({
model: this.model,
stream: true,
messages,
});
}
/**
* Stream a chat completion with tool calling support.
* Uses native tool calling when supported, otherwise falls back to UnTooled.
*/
async stream(messages, functions = [], eventHandler = null) {
await this.preloadModel();
const useNative =
functions.length > 0 && (await this.supportsNativeToolCalling());
if (!useNative) {
return await UnTooled.prototype.stream.call(
this,
messages,
functions,
this.#handleFunctionCallStream.bind(this),
eventHandler
);
}
this.providerLog(
"LemonadeProvider.stream (tooled) - will process this chat completion."
);
try {
return await tooledStream(
this.client,
this.model,
messages,
functions,
eventHandler
);
} catch (error) {
console.error(error.message, error);
if (error instanceof OpenAI.AuthenticationError) throw error;
if (
error instanceof OpenAI.RateLimitError ||
error instanceof OpenAI.InternalServerError ||
error instanceof OpenAI.APIError
) {
throw new RetryError(error.message);
}
throw error;
}
}
/**
* Create a non-streaming completion with tool calling support.
* Uses native tool calling when supported, otherwise falls back to UnTooled.
*/
async complete(messages, functions = []) {
await this.preloadModel();
const useNative =
functions.length > 0 && (await this.supportsNativeToolCalling());
if (!useNative) {
return await UnTooled.prototype.complete.call(
this,
messages,
functions,
this.#handleFunctionCallChat.bind(this)
);
}
try {
const result = await tooledComplete(
this.client,
this.model,
messages,
functions,
this.getCost.bind(this)
);
if (result.retryWithError) {
return this.complete([...messages, result.retryWithError], functions);
}
return result;
} catch (error) {
if (error instanceof OpenAI.AuthenticationError) throw error;
if (
error instanceof OpenAI.RateLimitError ||
error instanceof OpenAI.InternalServerError ||
error instanceof OpenAI.APIError
) {
throw new RetryError(error.message);
}
throw error;
}
}
/**
* Get the cost of the completion.
*
* @param _usage The completion to get the cost for.
* @returns The cost of the completion.
* Stubbed since Lemonade has no cost basis.
*/
getCost(_usage) {
return 0;
}
}
module.exports = LemonadeProvider;

6
server/utils/agents/index.js
View File

@@ -233,6 +233,10 @@ class AgentHandler {
if (!process.env.SAMBANOVA_LLM_API_KEY)
throw new Error("SambaNova API key must be provided to use agents.");
break;
case "lemonade":
if (!process.env.LEMONADE_LLM_BASE_PATH)
throw new Error("Lemonade base path must be provided to use agents.");
break;
default:
throw new Error(
"No workspace agent provider set. Please set your agent provider in the workspace's settings"
@@ -319,6 +323,8 @@ class AgentHandler {
return process.env.PRIVATEMODE_LLM_MODEL_PREF ?? null;
case "sambanova":
return process.env.SAMBANOVA_LLM_MODEL_PREF ?? null;
case "lemonade":
return process.env.LEMONADE_LLM_MODEL_PREF ?? null;
default:
return null;
}

17
server/utils/helpers/customModels.js
View File

@@ -14,6 +14,7 @@ const { GeminiLLM } = require("../AiProviders/gemini");
const { fetchCometApiModels } = require("../AiProviders/cometapi");
const { parseFoundryBasePath } = require("../AiProviders/foundry");
const { getDockerModels } = require("../AiProviders/dockerModelRunner");
const { getAllLemonadeModels } = require("../AiProviders/lemonade");
const SUPPORT_CUSTOM_MODELS = [
"openai",
@@ -47,10 +48,12 @@ const SUPPORT_CUSTOM_MODELS = [
"docker-model-runner",
"privatemode",
"sambanova",
"lemonade",
// Embedding Engines
"native-embedder",
"cohere-embedder",
"openrouter-embedder",
"lemonade-embedder",
];
async function getCustomModels(provider = "", apiKey = null, basePath = null) {
@@ -126,6 +129,10 @@ async function getCustomModels(provider = "", apiKey = null, basePath = null) {
return await getPrivatemodeModels(basePath, "generate");
case "sambanova":
return await getSambaNovaModels(apiKey);
case "lemonade":
return await getLemonadeModels(basePath);
case "lemonade-embedder":
return await getLemonadeModels(basePath, "embedding");
default:
return { models: [], error: "Invalid provider for custom models" };
}
@@ -892,6 +899,16 @@ async function getDockerModelRunnerModels(basePath = null) {
}
}
async function getLemonadeModels(basePath = null, task = "chat") {
try {
const models = await getAllLemonadeModels(basePath, task);
return { models, error: null };
} catch (e) {
console.error(`Lemonade:getLemonadeModels`, e.message);
return { models: [], error: "Could not fetch Lemonade Models" };
}
}
/**
* Get Privatemode models
* @param {string} basePath - The base path of the Privatemode endpoint.

11
server/utils/helpers/index.js
View File

@@ -240,6 +240,9 @@ function getLLMProvider({ provider = null, model = null } = {}) {
case "sambanova":
const { SambaNovaLLM } = require("../AiProviders/sambanova");
return new SambaNovaLLM(embedder, model);
case "lemonade":
const { LemonadeLLM } = require("../AiProviders/lemonade");
return new LemonadeLLM(embedder, model);
default:
throw new Error(
`ENV: No valid LLM_PROVIDER value found in environment! Using ${process.env.LLM_PROVIDER}`
@@ -297,6 +300,9 @@ function getEmbeddingEngineSelection() {
case "openrouter":
const { OpenRouterEmbedder } = require("../EmbeddingEngines/openRouter");
return new OpenRouterEmbedder();
case "lemonade":
const { LemonadeEmbedder } = require("../EmbeddingEngines/lemonade");
return new LemonadeEmbedder();
default:
return new NativeEmbedder();
}
@@ -416,6 +422,9 @@ function getLLMProviderClass({ provider = null } = {}) {
case "sambanova":
const { SambaNovaLLM } = require("../AiProviders/sambanova");
return SambaNovaLLM;
case "lemonade":
const { LemonadeLLM } = require("../AiProviders/lemonade");
return LemonadeLLM;
default:
return null;
}
@@ -498,6 +507,8 @@ function getBaseLLMProviderModel({ provider = null } = {}) {
return process.env.PRIVATEMODE_LLM_MODEL_PREF;
case "sambanova":
return process.env.SAMBANOVA_LLM_MODEL_PREF;
case "lemonade":
return process.env.LEMONADE_LLM_MODEL_PREF;
default:
return null;
}

16
server/utils/helpers/updateENV.js
View File

@@ -827,6 +827,20 @@ const KEY_MAPPING = {
envKey: "SAMBANOVA_LLM_MODEL_PREF",
checks: [isNotEmpty],
},
// Lemonade Options
LemonadeLLMBasePath: {
envKey: "LEMONADE_LLM_BASE_PATH",
checks: [isValidURL],
},
LemonadeLLMModelPref: {
envKey: "LEMONADE_LLM_MODEL_PREF",
checks: [isNotEmpty],
},
LemonadeLLMModelTokenLimit: {
envKey: "LEMONADE_LLM_MODEL_TOKEN_LIMIT",
checks: [nonZero],
},
};
function isNotEmpty(input = "") {
@@ -943,6 +957,7 @@ function supportedLLM(input = "") {
"docker-model-runner",
"privatemode",
"sambanova",
"lemonade",
].includes(input);
return validSelection ? null : `${input} is not a valid LLM provider.`;
}
@@ -981,6 +996,7 @@ function supportedEmbeddingModel(input = "") {
"generic-openai",
"mistral",
"openrouter",
"lemonade",
];
return supported.includes(input)
? null

17
server/utils/http/index.js
View File

@@ -104,6 +104,22 @@ function toValidNumber(number = null, fallback = null) {
return Number(number);
}
/**
* Decode HTML entities from a string.
* The DMR response is encoded with HTML entities, so we need to decode them
* so we can parse the JSON and report the progress percentage.
* @param {string} str - The string to decode.
* @returns {string} The decoded string.
*/
function decodeHtmlEntities(str) {
return str
.replace(/&#34;/g, '"')
.replace(/&#39;/g, "'")
.replace(/&lt;/g, "<")
.replace(/&gt;/g, ">")
.replace(/&amp;/g, "&");
}
module.exports = {
reqBody,
multiUserMode,
@@ -115,4 +131,5 @@ module.exports = {
safeJsonParse,
isValidUrl,
toValidNumber,
decodeHtmlEntities,
};