feat(llm): LLM 配置轮询(故障转移),某个配置没额度/失效时自动切下一个
默认关闭。开启后「未指定模型」的 agent 在当前配置不可用(402 余额不足 /
401 key 失效 / 429 限流 / 5xx)时自动切到下一个配置;agent 绑定或任务 pin
了配置的默认不轮询(绑定即独占,可由 llm_pool_bind_fallback 打开兜底)。
链序 = 激活配置 + 其余按 priority DESC,排除 pool_exclude 的。只给
llm_profiles 加两列,不引入关联表,所以 agents/tasks/conversations 对
llm_profile_id 的四处引用一行未改。
核心约束:只在本次调用尚未 yield 出任何事件时切换成员。已经吐出
text_delta / tool_use_start 后中途断流一律原样上抛(交给 agentcore 的
resume),否则重发同一 request 会重复输出、污染会话历史。SDK 的 doStream
保证 4xx/5xx 在读 body 前返回,所以目标场景都落在这个安全窗口内。
context.Canceled / DeadlineExceeded 绝不触发切换——那是用户取消或任务
超时,误判会白烧备用 key 并污染 a6bd180 的终止原因诊断。400 也不切
(请求本身有问题,换模型结果一样)。
熔断:402/401/403/404 一次即熔断,429/5xx/网络错误连续 3 次;冷却
60s→5min→30min 退避,到期半开试探,成功清零。Registry 是进程级的,独立
于 provider 缓存,重建链路不丢状态,且全进程共享(A 任务撞见的 402,B 任务
立刻知道)。状态旁路落库,重启只恢复尚未到期的冷却窗口。
上下文窗口装不下当前请求的成员直接跳过(必然 400),不把主力模型的窗口
压到全链最小值。全员熔断时仍探链首,不让引擎停摆。
Pool 包在 llmrec 外面、每个成员自带 recorder,失败调用会以 status=error
+ 正确的 profile 名进 llm_records;切换/熔断走 log.Printf,自动出现在
系统日志页。
顺带修两个既有问题:
- DecomposeGoals 原本自己 NewProvider(),绕过限速器/录制/轮询,且每个
任务新建一个实例——改为由 server 传入共享 provider
- saveLLMConfig 会把 priority/pool_exclude 重置为零值
前端:LLM 页新增轮询卡片(总开关/兜底开关/实际链路顺序/健康徽章/冷却
倒计时/立即恢复),配置编辑区加优先级与「不参与轮询」。
只有一个配置时链长度为 1,poolChain 返回 nil 走裸 provider,行为与开关
打开前完全一致。设计见 docs/LLM轮询设计.md。
This commit is contained in:
Autumn-27 committed
1 parent
89c5e9b26f
commit
0c653a2135
19 files changed
+1750
-50
No files matched your search
+54
-13
@@ -27,10 +27,29 @@ type LLMProfile struct {
|
||||
// 开启思考并设强度. Mapped to the provider request in agent.Config.NewProvider.
|
||||
ReasoningEffort string `json:"reasoning_effort"`
|
||||
IsDefault bool `json:"is_default"`
|
||||
// Priority orders the failover chain: higher goes first. The ACTIVE profile
|
||||
// (IsDefault) always heads the chain regardless of this value.
|
||||
Priority int `json:"priority"`
|
||||
// PoolExclude=true keeps this profile out of the failover chain — it stays
|
||||
// usable when an agent/task binds it explicitly, it just never gets picked up
|
||||
// as a fallback target.
|
||||
PoolExclude bool `json:"pool_exclude"`
|
||||
}
|
||||
|
||||
// profileCols is the read column list (hint variant, no api key) shared by the
|
||||
// list query; profileColsKey is the same with api_key for the single-row loads.
|
||||
const profileCols = `id,name,format,COALESCE(base_url,''),COALESCE(proxy,''),model,COALESCE(api_key_hint,''),rate_per_second,rate_per_minute,context_window_k,COALESCE(reasoning_effort,''),is_default,priority,pool_exclude`
|
||||
const profileColsKey = `id,name,format,COALESCE(base_url,''),COALESCE(proxy,''),model,COALESCE(api_key,''),rate_per_second,rate_per_minute,context_window_k,COALESCE(reasoning_effort,''),is_default,priority,pool_exclude`
|
||||
|
||||
// scanProfile reads one row in the profileCols / profileColsKey column order. The
|
||||
// 7th column lands in APIKeyHint or APIKey depending on which list the caller used.
|
||||
func scanProfile(sc interface{ Scan(...any) error }, into *string, p *LLMProfile) error {
|
||||
return sc.Scan(&p.ID, &p.Name, &p.Format, &p.BaseURL, &p.Proxy, &p.Model, into,
|
||||
&p.RatePerSecond, &p.RatePerMinute, &p.ContextWindowK, &p.ReasoningEffort, &p.IsDefault, &p.Priority, &p.PoolExclude)
|
||||
}
|
||||
|
||||
func (d *DB) ListProfiles() ([]*LLMProfile, error) {
|
||||
rows, err := d.Query(`SELECT id,name,format,COALESCE(base_url,''),COALESCE(proxy,''),model,COALESCE(api_key_hint,''),rate_per_second,rate_per_minute,context_window_k,COALESCE(reasoning_effort,''),is_default FROM llm_profiles ORDER BY id`)
|
||||
rows, err := d.Query(`SELECT ` + profileCols + ` FROM llm_profiles ORDER BY id`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -38,7 +57,7 @@ func (d *DB) ListProfiles() ([]*LLMProfile, error) {
|
||||
var out []*LLMProfile
|
||||
for rows.Next() {
|
||||
var p LLMProfile
|
||||
if err := rows.Scan(&p.ID, &p.Name, &p.Format, &p.BaseURL, &p.Proxy, &p.Model, &p.APIKeyHint, &p.RatePerSecond, &p.RatePerMinute, &p.ContextWindowK, &p.ReasoningEffort, &p.IsDefault); err != nil {
|
||||
if err := scanProfile(rows, &p.APIKeyHint, &p); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, &p)
|
||||
@@ -49,8 +68,7 @@ func (d *DB) ListProfiles() ([]*LLMProfile, error) {
|
||||
// ActiveProfile returns the default (active) profile with its api key, or nil.
|
||||
func (d *DB) ActiveProfile() (*LLMProfile, error) {
|
||||
var p LLMProfile
|
||||
err := d.QueryRow(`SELECT id,name,format,COALESCE(base_url,''),COALESCE(proxy,''),model,COALESCE(api_key,''),rate_per_second,rate_per_minute,context_window_k,COALESCE(reasoning_effort,''),is_default FROM llm_profiles WHERE is_default LIMIT 1`).
|
||||
Scan(&p.ID, &p.Name, &p.Format, &p.BaseURL, &p.Proxy, &p.Model, &p.APIKey, &p.RatePerSecond, &p.RatePerMinute, &p.ContextWindowK, &p.ReasoningEffort, &p.IsDefault)
|
||||
err := scanProfile(d.QueryRow(`SELECT `+profileColsKey+` FROM llm_profiles WHERE is_default LIMIT 1`), &p.APIKey, &p)
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -61,14 +79,37 @@ func (d *DB) ActiveProfile() (*LLMProfile, error) {
|
||||
// Used to run a task on a specific (non-default) LLM profile.
|
||||
func (d *DB) ProfileByID(id int64) (*LLMProfile, error) {
|
||||
var p LLMProfile
|
||||
err := d.QueryRow(`SELECT id,name,format,COALESCE(base_url,''),COALESCE(proxy,''),model,COALESCE(api_key,''),rate_per_second,rate_per_minute,context_window_k,COALESCE(reasoning_effort,''),is_default FROM llm_profiles WHERE id=$1`, id).
|
||||
Scan(&p.ID, &p.Name, &p.Format, &p.BaseURL, &p.Proxy, &p.Model, &p.APIKey, &p.RatePerSecond, &p.RatePerMinute, &p.ContextWindowK, &p.ReasoningEffort, &p.IsDefault)
|
||||
err := scanProfile(d.QueryRow(`SELECT `+profileColsKey+` FROM llm_profiles WHERE id=$1`, id), &p.APIKey, &p)
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
return &p, err
|
||||
}
|
||||
|
||||
// PoolProfiles returns the failover chain in run order, api keys included: the
|
||||
// active profile first, then every other keyed profile that isn't excluded, by
|
||||
// priority DESC (id ASC to stay stable). Profiles without an api key can't serve
|
||||
// a request, so they never enter the chain. The ordering IS the policy — callers
|
||||
// walk the slice front to back.
|
||||
func (d *DB) PoolProfiles() ([]*LLMProfile, error) {
|
||||
rows, err := d.Query(`SELECT ` + profileColsKey + ` FROM llm_profiles
|
||||
WHERE COALESCE(api_key,'') <> '' AND (is_default OR NOT pool_exclude)
|
||||
ORDER BY is_default DESC, priority DESC, id ASC`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []*LLMProfile
|
||||
for rows.Next() {
|
||||
var p LLMProfile
|
||||
if err := scanProfile(rows, &p.APIKey, &p); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, &p)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// SaveProfile inserts (id==0) or updates a profile. Empty apiKey on update keeps existing.
|
||||
func (d *DB) SaveProfile(p *LLMProfile) (int64, error) {
|
||||
hint := p.APIKeyHint
|
||||
@@ -77,18 +118,18 @@ func (d *DB) SaveProfile(p *LLMProfile) (int64, error) {
|
||||
}
|
||||
if p.ID == 0 {
|
||||
var id int64
|
||||
err := d.QueryRow(`INSERT INTO llm_profiles(name,format,base_url,proxy,model,api_key,api_key_hint,rate_per_second,rate_per_minute,context_window_k,reasoning_effort)
|
||||
VALUES ($1,$2,NULLIF($3,''),NULLIF($4,''),$5,NULLIF($6,''),NULLIF($7,''),$8,$9,$10,$11) RETURNING id`,
|
||||
p.Name, p.Format, p.BaseURL, p.Proxy, p.Model, p.APIKey, hint, p.RatePerSecond, p.RatePerMinute, p.ContextWindowK, p.ReasoningEffort).Scan(&id)
|
||||
err := d.QueryRow(`INSERT INTO llm_profiles(name,format,base_url,proxy,model,api_key,api_key_hint,rate_per_second,rate_per_minute,context_window_k,reasoning_effort,priority,pool_exclude)
|
||||
VALUES ($1,$2,NULLIF($3,''),NULLIF($4,''),$5,NULLIF($6,''),NULLIF($7,''),$8,$9,$10,$11,$12,$13) RETURNING id`,
|
||||
p.Name, p.Format, p.BaseURL, p.Proxy, p.Model, p.APIKey, hint, p.RatePerSecond, p.RatePerMinute, p.ContextWindowK, p.ReasoningEffort, p.Priority, p.PoolExclude).Scan(&id)
|
||||
return id, err
|
||||
}
|
||||
if p.APIKey == "" {
|
||||
_, err := d.Exec(`UPDATE llm_profiles SET name=$1,format=$2,base_url=NULLIF($3,''),proxy=NULLIF($4,''),model=$5,rate_per_second=$6,rate_per_minute=$7,context_window_k=$8,reasoning_effort=$9 WHERE id=$10`,
|
||||
p.Name, p.Format, p.BaseURL, p.Proxy, p.Model, p.RatePerSecond, p.RatePerMinute, p.ContextWindowK, p.ReasoningEffort, p.ID)
|
||||
_, err := d.Exec(`UPDATE llm_profiles SET name=$1,format=$2,base_url=NULLIF($3,''),proxy=NULLIF($4,''),model=$5,rate_per_second=$6,rate_per_minute=$7,context_window_k=$8,reasoning_effort=$9,priority=$10,pool_exclude=$11 WHERE id=$12`,
|
||||
p.Name, p.Format, p.BaseURL, p.Proxy, p.Model, p.RatePerSecond, p.RatePerMinute, p.ContextWindowK, p.ReasoningEffort, p.Priority, p.PoolExclude, p.ID)
|
||||
return p.ID, err
|
||||
}
|
||||
_, err := d.Exec(`UPDATE llm_profiles SET name=$1,format=$2,base_url=NULLIF($3,''),proxy=NULLIF($4,''),model=$5,api_key=$6,api_key_hint=$7,rate_per_second=$8,rate_per_minute=$9,context_window_k=$10,reasoning_effort=$11 WHERE id=$12`,
|
||||
p.Name, p.Format, p.BaseURL, p.Proxy, p.Model, p.APIKey, hint, p.RatePerSecond, p.RatePerMinute, p.ContextWindowK, p.ReasoningEffort, p.ID)
|
||||
_, err := d.Exec(`UPDATE llm_profiles SET name=$1,format=$2,base_url=NULLIF($3,''),proxy=NULLIF($4,''),model=$5,api_key=$6,api_key_hint=$7,rate_per_second=$8,rate_per_minute=$9,context_window_k=$10,reasoning_effort=$11,priority=$12,pool_exclude=$13 WHERE id=$14`,
|
||||
p.Name, p.Format, p.BaseURL, p.Proxy, p.Model, p.APIKey, hint, p.RatePerSecond, p.RatePerMinute, p.ContextWindowK, p.ReasoningEffort, p.Priority, p.PoolExclude, p.ID)
|
||||
return p.ID, err
|
||||
}
|
||||
|
||||
|
||||
@@ -5,6 +5,54 @@ import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestPoolProfilesOrder pins the failover chain query: keyless profiles can't
|
||||
// serve a request and excluded ones aren't fallback targets, so neither belongs
|
||||
// in the chain; the rest come back by priority, highest first.
|
||||
// Deliberately does NOT touch is_default — flipping the active profile would be a
|
||||
// side effect on the shared dev database.
|
||||
func TestPoolProfilesOrder(t *testing.T) {
|
||||
d, err := Open(testDSN(t))
|
||||
if err != nil {
|
||||
t.Skipf("postgres unavailable (%v) — skipping", err)
|
||||
}
|
||||
defer d.Close()
|
||||
|
||||
mk := func(name string, priority int, exclude bool, key string) int64 {
|
||||
id, err := d.SaveProfile(&LLMProfile{
|
||||
Name: name, Format: "openai", Model: "m", APIKey: key,
|
||||
Priority: priority, PoolExclude: exclude,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { d.Exec(`DELETE FROM llm_profiles WHERE id=$1`, id) })
|
||||
return id
|
||||
}
|
||||
lo := mk("t-pool-lo", 1, false, "k1")
|
||||
hi := mk("t-pool-hi", 9, false, "k2")
|
||||
mk("t-pool-excluded", 99, true, "k3") // excluded despite the top priority
|
||||
mk("t-pool-nokey", 50, false, "") // no key → cannot serve anything
|
||||
|
||||
chain, err := d.PoolProfiles()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got []int64
|
||||
for _, p := range chain {
|
||||
switch p.Name {
|
||||
case "t-pool-lo", "t-pool-hi":
|
||||
got = append(got, p.ID)
|
||||
case "t-pool-excluded":
|
||||
t.Fatal("pool_exclude profile entered the failover chain")
|
||||
case "t-pool-nokey":
|
||||
t.Fatal("keyless profile entered the failover chain")
|
||||
}
|
||||
}
|
||||
if len(got) != 2 || got[0] != hi || got[1] != lo {
|
||||
t.Fatalf("chain order = %v, want [hi=%d lo=%d]", got, hi, lo)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigStores(t *testing.T) {
|
||||
d, err := Open(testDSN(t))
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
package db
|
||||
|
||||
import "time"
|
||||
|
||||
// LLM failover health: one row per profile tracking its circuit-breaker state.
|
||||
// The authoritative copy lives in memory (llmpool.Registry); this table only
|
||||
// survives a restart so a cooling-off window isn't silently reset by one.
|
||||
|
||||
// LLMHealth is one profile's circuit-breaker state.
|
||||
type LLMHealth struct {
|
||||
ProfileID int64 `json:"profile_id"`
|
||||
Fails int `json:"fails"` // consecutive failures; cleared on success
|
||||
Trips int `json:"trips"` // total trips, drives the backoff ladder
|
||||
OpenUntil *time.Time `json:"open_until"` // nil/past = closed (healthy)
|
||||
LastError string `json:"last_error"`
|
||||
LastAt time.Time `json:"last_at"`
|
||||
}
|
||||
|
||||
// LoadLLMHealth returns the profiles still in an UNEXPIRED cooling-off window.
|
||||
// Expired rows are deliberately skipped: after a restart a profile that has
|
||||
// finished cooling should be treated as healthy again and re-probed on its next
|
||||
// call, not resurrected as broken.
|
||||
func (d *DB) LoadLLMHealth() ([]LLMHealth, error) {
|
||||
rows, err := d.Query(`SELECT profile_id,fails,trips,open_until,COALESCE(last_error,''),last_at
|
||||
FROM llm_profile_health WHERE open_until IS NOT NULL AND open_until > now()`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []LLMHealth
|
||||
for rows.Next() {
|
||||
var h LLMHealth
|
||||
if err := rows.Scan(&h.ProfileID, &h.Fails, &h.Trips, &h.OpenUntil, &h.LastError, &h.LastAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, h)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// SaveLLMHealth upserts one profile's circuit-breaker state.
|
||||
func (d *DB) SaveLLMHealth(h LLMHealth) error {
|
||||
_, err := d.Exec(`
|
||||
INSERT INTO llm_profile_health(profile_id,fails,trips,open_until,last_error,last_at)
|
||||
VALUES ($1,$2,$3,$4,$5,now())
|
||||
ON CONFLICT (profile_id) DO UPDATE SET
|
||||
fails=EXCLUDED.fails, trips=EXCLUDED.trips, open_until=EXCLUDED.open_until,
|
||||
last_error=EXCLUDED.last_error, last_at=now()`,
|
||||
h.ProfileID, h.Fails, h.Trips, h.OpenUntil, h.LastError)
|
||||
return err
|
||||
}
|
||||
|
||||
// ClearLLMHealth drops one profile's state (manual "recover now" from the UI).
|
||||
func (d *DB) ClearLLMHealth(profileID int64) error {
|
||||
_, err := d.Exec(`DELETE FROM llm_profile_health WHERE profile_id=$1`, profileID)
|
||||
return err
|
||||
}
|
||||
@@ -214,6 +214,11 @@ CREATE TABLE IF NOT EXISTS llm_profiles (
|
||||
context_window_k INTEGER NOT NULL DEFAULT 0,
|
||||
reasoning_effort TEXT NOT NULL DEFAULT '',
|
||||
is_default BOOLEAN NOT NULL DEFAULT false,
|
||||
-- 轮询(故障转移)参数，见 docs/LLM轮询设计.md：
|
||||
-- priority 顺位，越大越先被选中；激活配置(is_default)永远排链首，与本值无关。
|
||||
-- pool_exclude true=不作为故障转移目标(仍可被 agent/任务显式绑定使用)。
|
||||
priority INTEGER NOT NULL DEFAULT 0,
|
||||
pool_exclude BOOLEAN NOT NULL DEFAULT false,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
@@ -221,6 +226,21 @@ CREATE UNIQUE INDEX IF NOT EXISTS uq_llm_one_default ON llm_profiles(is_default)
|
||||
DROP TRIGGER IF EXISTS trg_llm_upd ON llm_profiles;
|
||||
CREATE TRIGGER trg_llm_upd BEFORE UPDATE ON llm_profiles
|
||||
FOR EACH ROW EXECUTE FUNCTION set_updated_at();
|
||||
-- 轮询顺位/排除标记；补旧库。默认 0 / false = 全部配置都参与轮询。
|
||||
ALTER TABLE llm_profiles ADD COLUMN IF NOT EXISTS priority INTEGER NOT NULL DEFAULT 0;
|
||||
ALTER TABLE llm_profiles ADD COLUMN IF NOT EXISTS pool_exclude BOOLEAN NOT NULL DEFAULT false;
|
||||
|
||||
-- LLM 轮询熔断状态：某个配置连续失败(余额不足/key 失效/限流)后进入冷却，冷却期内
|
||||
-- 轮询直接跳过它。内存态为准，这里落库只为重启后不丢冷却窗口——加载时只取尚未
|
||||
-- 到期的行(open_until > now)，已到期的自然回到"正常"，等下一次调用半开试探。
|
||||
CREATE TABLE IF NOT EXISTS llm_profile_health (
|
||||
profile_id BIGINT PRIMARY KEY REFERENCES llm_profiles(id) ON DELETE CASCADE,
|
||||
fails INTEGER NOT NULL DEFAULT 0, -- 当前连续失败次数(成功即清零)
|
||||
trips INTEGER NOT NULL DEFAULT 0, -- 累计熔断次数,用于冷却时间指数退避
|
||||
open_until TIMESTAMPTZ, -- 冷却截止;NULL/过期 = 未熔断
|
||||
last_error TEXT NOT NULL DEFAULT '',
|
||||
last_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
-- =====================================================================
|
||||
-- D. 任务层
|
||||
|
||||
Reference in new issue
Block a user