feat(llm): LLM 配置轮询(故障转移),某个配置没额度/失效时自动切下一个
默认关闭。开启后「未指定模型」的 agent 在当前配置不可用(402 余额不足 /
401 key 失效 / 429 限流 / 5xx)时自动切到下一个配置;agent 绑定或任务 pin
了配置的默认不轮询(绑定即独占,可由 llm_pool_bind_fallback 打开兜底)。
链序 = 激活配置 + 其余按 priority DESC,排除 pool_exclude 的。只给
llm_profiles 加两列,不引入关联表,所以 agents/tasks/conversations 对
llm_profile_id 的四处引用一行未改。
核心约束:只在本次调用尚未 yield 出任何事件时切换成员。已经吐出
text_delta / tool_use_start 后中途断流一律原样上抛(交给 agentcore 的
resume),否则重发同一 request 会重复输出、污染会话历史。SDK 的 doStream
保证 4xx/5xx 在读 body 前返回,所以目标场景都落在这个安全窗口内。
context.Canceled / DeadlineExceeded 绝不触发切换——那是用户取消或任务
超时,误判会白烧备用 key 并污染 a6bd180 的终止原因诊断。400 也不切
(请求本身有问题,换模型结果一样)。
熔断:402/401/403/404 一次即熔断,429/5xx/网络错误连续 3 次;冷却
60s→5min→30min 退避,到期半开试探,成功清零。Registry 是进程级的,独立
于 provider 缓存,重建链路不丢状态,且全进程共享(A 任务撞见的 402,B 任务
立刻知道)。状态旁路落库,重启只恢复尚未到期的冷却窗口。
上下文窗口装不下当前请求的成员直接跳过(必然 400),不把主力模型的窗口
压到全链最小值。全员熔断时仍探链首,不让引擎停摆。
Pool 包在 llmrec 外面、每个成员自带 recorder,失败调用会以 status=error
+ 正确的 profile 名进 llm_records;切换/熔断走 log.Printf,自动出现在
系统日志页。
顺带修两个既有问题:
- DecomposeGoals 原本自己 NewProvider(),绕过限速器/录制/轮询,且每个
任务新建一个实例——改为由 server 传入共享 provider
- saveLLMConfig 会把 priority/pool_exclude 重置为零值
前端:LLM 页新增轮询卡片(总开关/兜底开关/实际链路顺序/健康徽章/冷却
倒计时/立即恢复),配置编辑区加优先级与「不参与轮询」。
只有一个配置时链长度为 1,poolChain 返回 nil 走裸 provider,行为与开关
打开前完全一致。设计见 docs/LLM轮询设计.md。
This commit is contained in:
Autumn-27 committed
1 parent
89c5e9b26f
commit
0c653a2135
19 files changed
+1750
-50
No files matched your search
+7
-1
@@ -14,7 +14,7 @@ import type {
|
||||
InterceptRule, InterceptPending, InterceptApprovalRow,
|
||||
TaskAssetView, SSProject, SSTask, ConvTokenSummary, CoverageGraphData, CoverageAssetRefs,
|
||||
WorkspaceListing, WorkspaceFile,
|
||||
CommandRecord, LLMRecordItem, LLMRecordDetail, LLMTask,
|
||||
CommandRecord, LLMRecordItem, LLMRecordDetail, LLMTask, LLMPoolStatus,
|
||||
} from "@/lib/types";
|
||||
|
||||
function getToken(): string | null {
|
||||
@@ -409,9 +409,15 @@ export const api = {
|
||||
rate_per_minute?: number;
|
||||
context_window_k?: number;
|
||||
reasoning_effort?: string; // ""|"off"|"low"|"medium"|"high"|"max"
|
||||
priority?: number; // 轮询顺位，越大越先用
|
||||
pool_exclude?: boolean; // true=不作为故障转移目标
|
||||
}) => post<{ id: number }>("/llm/profiles", p),
|
||||
deleteLLMProfile: (id: string) => del<{ deleted: number }>(`/llm/profiles/${id}`),
|
||||
activateLLMProfile: (id: string) => post<{ ok: boolean }>("/llm/profiles/active", { id: Number(id) }),
|
||||
// 轮询链的实际顺序 + 各配置的熔断状态。
|
||||
llmPool: () => get<LLMPoolStatus>("/llm/pool"),
|
||||
// 清除熔断，让下一次调用立刻重试该配置；不传 id = 全部清除。
|
||||
resetLLMPool: (id?: string) => post<LLMPoolStatus>("/llm/pool/reset", { id: id ? Number(id) : 0 }),
|
||||
fetchLLMModels: (provider: string, base_url: string, api_key: string, proxy = "", profile_id?: number) =>
|
||||
post<{ ok: boolean; error?: string; models?: string[] }>("/llm/models", {
|
||||
provider,
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
import type {
|
||||
Activity, Agent, AgentDetail, Asset, Audit, Company, Conversation,
|
||||
ConvTokenSummary, DailyTokenBucket, Edge, Finding, InterceptApprovalRow,
|
||||
InterceptPending, InterceptRule, LLMProfile, MCPServer, MCPTool, PromptVar,
|
||||
InterceptPending, InterceptRule, LLMPoolStatus, LLMProfile, MCPServer, MCPTool, PromptVar,
|
||||
PromptVersion, Settings, SkillItem, Stats, TaskNode, Task, TokenTotal,
|
||||
TokenUsage, Tool, TrafficResp, TrafficDetail, TrafficHost, LLMTask,
|
||||
} from "@/lib/types";
|
||||
@@ -595,8 +595,8 @@ export const audit: Audit = {
|
||||
|
||||
// ── LLM profiles ─────────────────────────────────────────────────────────────
|
||||
export const llmProfiles: LLMProfile[] = [
|
||||
{ id: "1", name: "Claude Opus 4.8", format: "anthropic", model: "claude-opus-4-8", api_key_hint: "…a3f2", rate_per_second: 0, rate_per_minute: 0, context_window_k: 1000, reasoning_effort: "high", is_default: true },
|
||||
{ id: "2", name: "DeepSeek V4", format: "openai", base_url: "https://api.deepseek.com", model: "deepseek-v4-flash", api_key_hint: "…9c11", rate_per_second: 0, rate_per_minute: 60, context_window_k: 128, reasoning_effort: "", is_default: false },
|
||||
{ id: "1", name: "Claude Opus 4.8", format: "anthropic", model: "claude-opus-4-8", api_key_hint: "…a3f2", rate_per_second: 0, rate_per_minute: 0, context_window_k: 1000, reasoning_effort: "high", is_default: true, priority: 0, pool_exclude: false },
|
||||
{ id: "2", name: "DeepSeek V4", format: "openai", base_url: "https://api.deepseek.com", model: "deepseek-v4-flash", api_key_hint: "…9c11", rate_per_second: 0, rate_per_minute: 60, context_window_k: 128, reasoning_effort: "", is_default: false, priority: 10, pool_exclude: false },
|
||||
];
|
||||
|
||||
export const llmConfig = {
|
||||
@@ -689,6 +689,19 @@ export const settings: Settings = {
|
||||
web_search_proxy: "",
|
||||
python_interpreter: "/usr/bin/python3",
|
||||
workers: 3,
|
||||
llm_pool_enabled: true,
|
||||
llm_pool_bind_fallback: false,
|
||||
};
|
||||
|
||||
// ── LLM 轮询（故障转移）──────────────────────────────────────────────────────
|
||||
// demo：激活配置正常，备用配置刚因余额不足熔断，正在冷却。
|
||||
export const llmPool: LLMPoolStatus = {
|
||||
enabled: true,
|
||||
bind_fallback: false,
|
||||
chain: [
|
||||
{ profile_id: "1", name: "Claude Opus 4.8", model: "claude-opus-4-8", format: "anthropic", priority: 0, active: true, excluded: false, state: "ok", fails: 0, trips: 0, cooldown_secs: 0 },
|
||||
{ profile_id: "2", name: "DeepSeek V4", model: "deepseek-v4-flash", format: "openai", priority: 10, active: false, excluded: false, state: "tripped", fails: 0, trips: 1, cooldown_secs: 42, last_error: "openai: status 402: insufficient balance", last_at: T("2026-07-26T00:10:00Z") },
|
||||
],
|
||||
};
|
||||
|
||||
// ── Intercept ────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -501,6 +501,11 @@ export interface Settings {
|
||||
// 任务并发上限:同时「运行中」的任务数上限。关闭=不限;开启后新建任务超限则排队,有空位自动启动。
|
||||
task_concurrency_enabled?: boolean; // 默认 false
|
||||
task_concurrency_limit?: number; // 开启后默认 5
|
||||
// LLM 轮询(故障转移)。默认关；开启后「未指定模型」的 agent 在当前配置不可用
|
||||
// （余额不足/key 失效/限流/服务异常）时自动切到下一个配置。
|
||||
llm_pool_enabled?: boolean; // 默认 false
|
||||
// 绑定了指定配置的 agent/任务失败时是否也回落到轮询链。默认 false = 绑定即独占。
|
||||
llm_pool_bind_fallback?: boolean;
|
||||
}
|
||||
|
||||
// ---- LLM config ----
|
||||
@@ -518,6 +523,37 @@ export interface LLMProfile {
|
||||
// 思考模式: ""=默认(不发送) | "off"=关闭 | "low"/"medium"/"high"/"max"=开启并设强度
|
||||
reasoning_effort?: string;
|
||||
is_default: boolean;
|
||||
// 轮询顺位：越大越先被选中。激活配置恒为链首，与本值无关。
|
||||
priority?: number;
|
||||
// true = 不作为故障转移目标（仍可被 agent/任务显式绑定使用）。
|
||||
pool_exclude?: boolean;
|
||||
}
|
||||
|
||||
// ---- LLM 轮询（故障转移）----
|
||||
// 一个配置在轮询链中的位置与健康状态。state:
|
||||
// ok 正常
|
||||
// degraded 有连续失败但未达熔断阈值
|
||||
// tripped 已熔断，冷却期内被跳过（cooldown_secs 为剩余秒数）
|
||||
export interface LLMPoolMember {
|
||||
profile_id: string;
|
||||
name: string;
|
||||
model: string;
|
||||
format: string;
|
||||
priority: number;
|
||||
active: boolean; // 是否为当前激活配置（恒为链首）
|
||||
excluded: boolean; // pool_exclude：不参与轮询
|
||||
state: "ok" | "degraded" | "tripped";
|
||||
fails: number;
|
||||
trips: number;
|
||||
cooldown_secs: number;
|
||||
last_error?: string;
|
||||
last_at?: string;
|
||||
}
|
||||
|
||||
export interface LLMPoolStatus {
|
||||
enabled: boolean;
|
||||
bind_fallback: boolean;
|
||||
chain: LLMPoolMember[];
|
||||
}
|
||||
|
||||
// ---- Agents ----
|
||||
|
||||
Reference in new issue
Block a user