feat(llm): add profile failover and pool health
This commit is contained in:
pphua committed
1 parent
d4b21f0bdb
commit
18926c129f
15 files changed
+2507
-761
No files matched your search
@@ -0,0 +1,43 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestEngineActiveLLMCallsConcurrent(t *testing.T) {
|
||||
e := NewEngine(nil)
|
||||
const taskID = "42"
|
||||
const calls = 64
|
||||
|
||||
var started sync.WaitGroup
|
||||
var release sync.WaitGroup
|
||||
var finished sync.WaitGroup
|
||||
started.Add(calls)
|
||||
finished.Add(calls)
|
||||
release.Add(1)
|
||||
for range calls {
|
||||
go func() {
|
||||
defer finished.Done()
|
||||
e.BeginLLMCall(taskID)
|
||||
started.Done()
|
||||
release.Wait()
|
||||
e.EndLLMCall(taskID)
|
||||
}()
|
||||
}
|
||||
started.Wait()
|
||||
if got := e.ActiveLLMCalls(taskID); got != calls {
|
||||
t.Fatalf("active calls = %d, want %d", got, calls)
|
||||
}
|
||||
release.Done()
|
||||
finished.Wait()
|
||||
if got := e.ActiveLLMCalls(taskID); got != 0 {
|
||||
t.Fatalf("active calls after completion = %d, want 0", got)
|
||||
}
|
||||
|
||||
// A defensive extra end must never expose a negative UI count.
|
||||
e.EndLLMCall(taskID)
|
||||
if got := e.ActiveLLMCalls(taskID); got != 0 {
|
||||
t.Fatalf("active calls after extra end = %d, want 0", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,239 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/Autumn-27/artex/agent"
|
||||
"github.com/Autumn-27/artex/db"
|
||||
"github.com/Autumn-27/artex/llmpool"
|
||||
"github.com/Autumn-27/norma/llm"
|
||||
)
|
||||
|
||||
// LLM 轮询(故障转移)的服务端接线。设计见 docs/LLM轮询设计.md：
|
||||
// - 全局激活配置这条路径(agent 未绑定、任务未 pin)才轮询;
|
||||
// - 绑定/pin 的路径默认独占该配置,失败即失败(可由 llm_pool_bind_fallback 打开兜底);
|
||||
// - 链序 = 激活配置 → 其余按 priority DESC,排除 pool_exclude 的;
|
||||
// - 熔断状态进程级共享(s.llmHealth),重建 pool 不清空。
|
||||
|
||||
// newLLMHealthRegistry builds the process-wide circuit-breaker registry, mirroring
|
||||
// state into PG so a cooling-off window survives a restart. Writes are async and
|
||||
// best-effort — the in-memory copy is authoritative.
|
||||
func newLLMHealthRegistry(pg *db.DB) *llmpool.Registry {
|
||||
if pg == nil {
|
||||
return llmpool.NewRegistry(nil, nil)
|
||||
}
|
||||
persist := func(id int64, st llmpool.State) {
|
||||
h := db.LLMHealth{ProfileID: id, Fails: st.Fails, Trips: st.Trips, LastError: st.LastError}
|
||||
if !st.OpenUntil.IsZero() {
|
||||
t := st.OpenUntil
|
||||
h.OpenUntil = &t
|
||||
}
|
||||
go func() {
|
||||
if err := pg.SaveLLMHealth(h); err != nil {
|
||||
log.Printf("[llmpool] 熔断状态落库失败: %v", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
forget := func(id int64) {
|
||||
go func() { _ = pg.ClearLLMHealth(id) }()
|
||||
}
|
||||
reg := llmpool.NewRegistry(persist, forget)
|
||||
// Restore only windows that haven't expired (LoadLLMHealth filters), so a
|
||||
// profile that finished cooling while we were down comes back healthy.
|
||||
if rows, err := pg.LoadLLMHealth(); err == nil {
|
||||
for _, h := range rows {
|
||||
st := llmpool.State{Fails: h.Fails, Trips: h.Trips, LastError: h.LastError, LastAt: h.LastAt}
|
||||
if h.OpenUntil != nil {
|
||||
st.OpenUntil = *h.OpenUntil
|
||||
}
|
||||
reg.Restore(h.ProfileID, st)
|
||||
log.Printf("[llmpool] 恢复熔断状态: 配置 #%d 冷却至 %s", h.ProfileID, st.OpenUntil.Format(time.RFC3339))
|
||||
}
|
||||
}
|
||||
return reg
|
||||
}
|
||||
|
||||
// poolMember builds one chain member from a profile, reusing the per-profile
|
||||
// provider cache so every agent pointing at the same profile shares one provider
|
||||
// (hence one rate limiter). Returns nil when the profile can't be built.
|
||||
func (s *Server) poolMember(p *db.LLMProfile, rank int) *llmpool.Member {
|
||||
prov, cfg, ok := s.providerForProfile(p.ID)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return &llmpool.Member{
|
||||
ID: p.ID, Name: p.Name, Model: p.Model, Format: p.Format,
|
||||
Priority: p.Priority, Active: p.IsDefault, Rank: rank,
|
||||
WindowTokens: cfg.CompactionWindow(), Prov: prov,
|
||||
}
|
||||
}
|
||||
|
||||
// poolChain reads the failover chain from the DB. headID/headProv/headCfg describe
|
||||
// the member that must lead the chain — the globally active profile on the normal
|
||||
// path, or an explicitly bound one when bound-profile fallback is on. The head is
|
||||
// passed in (rather than resolved through the cache) so applyLLM's already-built
|
||||
// provider is reused instead of duplicated.
|
||||
//
|
||||
// Returns nil when failover is off or the chain has fewer than two usable members
|
||||
// — callers then use the bare provider, which is byte-for-byte the old behavior.
|
||||
func (s *Server) poolChain(headID int64, headProv llm.Provider, headCfg agent.Config) *llmpool.Pool {
|
||||
if s.m == nil || s.m.pg == nil || !s.m.LLMPoolEnabled() {
|
||||
return nil
|
||||
}
|
||||
profs, err := s.m.pg.PoolProfiles()
|
||||
if err != nil {
|
||||
log.Printf("[llmpool] 读取轮询链失败: %v", err)
|
||||
return nil
|
||||
}
|
||||
var head *db.LLMProfile
|
||||
for _, p := range profs {
|
||||
if p.ID == headID {
|
||||
head = p
|
||||
break
|
||||
}
|
||||
}
|
||||
if head == nil { // head excluded from the chain (bound + pool_exclude) — still leads
|
||||
if p, err := s.m.pg.ProfileByID(headID); err == nil && p != nil {
|
||||
head = p
|
||||
} else {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
members := []*llmpool.Member{{
|
||||
ID: head.ID, Name: head.Name, Model: head.Model, Format: head.Format,
|
||||
Priority: head.Priority, Active: head.IsDefault, Rank: llmpool.RankActive,
|
||||
WindowTokens: headCfg.CompactionWindow(), Prov: headProv,
|
||||
}}
|
||||
for _, p := range profs {
|
||||
if p.ID == headID {
|
||||
continue
|
||||
}
|
||||
// The globally active profile outranks the others when it isn't the head
|
||||
// (bound-fallback chains), so it stays the first fallback tried.
|
||||
rank := p.Priority
|
||||
if p.IsDefault {
|
||||
rank = llmpool.RankActive - 1
|
||||
}
|
||||
if m := s.poolMember(p, rank); m != nil {
|
||||
members = append(members, m)
|
||||
}
|
||||
}
|
||||
if len(members) < 2 {
|
||||
return nil // nothing to fail over to
|
||||
}
|
||||
return llmpool.New(members, s.llmHealth)
|
||||
}
|
||||
|
||||
// poolForActive wraps the globally-active provider in the failover chain. Returns
|
||||
// prov unchanged when failover is off or there's nothing to fall back to.
|
||||
func (s *Server) poolForActive(activeID int64, prov llm.Provider, cfg agent.Config) llm.Provider {
|
||||
pool := s.poolChain(activeID, prov, cfg)
|
||||
if pool == nil {
|
||||
return prov
|
||||
}
|
||||
names := make([]string, 0, len(pool.Members()))
|
||||
for _, m := range pool.Members() {
|
||||
names = append(names, m.Name+"/"+m.Model)
|
||||
}
|
||||
log.Printf("[llmpool] LLM 轮询已启用，链路(%d): %v", len(names), names)
|
||||
return pool
|
||||
}
|
||||
|
||||
// poolForBinding wraps a BOUND profile's provider so it falls back to the chain.
|
||||
// Only active when both the failover master switch and the bound-fallback switch
|
||||
// are on; otherwise a bound profile stays exclusive (fails hard), which is the
|
||||
// default and the documented precedence.
|
||||
func (s *Server) poolForBinding(id int64, prov llm.Provider, cfg agent.Config) llm.Provider {
|
||||
if s.m == nil || !s.m.LLMPoolEnabled() || !s.m.LLMPoolBindFallback() {
|
||||
return prov
|
||||
}
|
||||
if pool := s.poolChain(id, prov, cfg); pool != nil {
|
||||
return pool
|
||||
}
|
||||
return prov
|
||||
}
|
||||
|
||||
// LLMPoolMemberStatus is one chain entry as shown in the UI.
|
||||
type LLMPoolMemberStatus struct {
|
||||
ProfileID string `json:"profile_id"`
|
||||
Name string `json:"name"`
|
||||
Model string `json:"model"`
|
||||
Format string `json:"format"`
|
||||
Priority int `json:"priority"`
|
||||
Active bool `json:"active"` // is the globally-activated profile
|
||||
Excluded bool `json:"excluded"` // pool_exclude — not a failover target
|
||||
// Health: state is "ok" | "degraded" (failing but not tripped) | "tripped".
|
||||
State string `json:"state"`
|
||||
Fails int `json:"fails"`
|
||||
Trips int `json:"trips"`
|
||||
CooldownSecs int `json:"cooldown_secs"` // remaining cooling-off seconds; 0 = none
|
||||
LastError string `json:"last_error,omitempty"`
|
||||
LastAt string `json:"last_at,omitempty"`
|
||||
}
|
||||
|
||||
// llmPoolStatus reports the whole picture for the LLM page: whether failover is
|
||||
// on, the resolved chain order, and each profile's breaker state. Excluded
|
||||
// profiles are listed too (flagged), so the user can see why one isn't in line.
|
||||
func (s *Server) llmPoolStatus() map[string]any {
|
||||
out := map[string]any{
|
||||
"enabled": false,
|
||||
"bind_fallback": false,
|
||||
"chain": []LLMPoolMemberStatus{},
|
||||
}
|
||||
if s.m == nil || s.m.pg == nil {
|
||||
return out
|
||||
}
|
||||
out["enabled"] = s.m.LLMPoolEnabled()
|
||||
out["bind_fallback"] = s.m.LLMPoolBindFallback()
|
||||
|
||||
all, err := s.m.pg.ListProfiles()
|
||||
if err != nil {
|
||||
return out
|
||||
}
|
||||
health := s.llmHealth.Snapshot()
|
||||
now := time.Now()
|
||||
// Chain order: active first, then priority DESC / id ASC — the same ordering
|
||||
// PoolProfiles applies, recomputed here so excluded ones can be shown in place.
|
||||
chain := make([]LLMPoolMemberStatus, 0, len(all))
|
||||
for _, p := range all {
|
||||
st := health[p.ID]
|
||||
m := LLMPoolMemberStatus{
|
||||
ProfileID: i64s(p.ID), Name: p.Name, Model: p.Model, Format: p.Format,
|
||||
Priority: p.Priority, Active: p.IsDefault, Excluded: p.PoolExclude,
|
||||
State: "ok", Fails: st.Fails, Trips: st.Trips,
|
||||
LastError: st.LastError,
|
||||
}
|
||||
if st.Open() {
|
||||
m.State = "tripped"
|
||||
m.CooldownSecs = int(st.OpenUntil.Sub(now).Seconds()) + 1
|
||||
} else if st.Fails > 0 {
|
||||
m.State = "degraded"
|
||||
}
|
||||
if !st.LastAt.IsZero() {
|
||||
m.LastAt = st.LastAt.Format(time.RFC3339)
|
||||
}
|
||||
chain = append(chain, m)
|
||||
}
|
||||
sortPoolStatus(chain)
|
||||
out["chain"] = chain
|
||||
return out
|
||||
}
|
||||
|
||||
// sortPoolStatus orders the status rows exactly like the live chain: active first,
|
||||
// then priority DESC, then id ASC (the incoming slice is already id-ordered, so a
|
||||
// stable insertion sort on the first two keys is enough).
|
||||
func sortPoolStatus(in []LLMPoolMemberStatus) {
|
||||
for i := 1; i < len(in); i++ {
|
||||
for j := i; j > 0 && poolLess(in[j], in[j-1]); j-- {
|
||||
in[j], in[j-1] = in[j-1], in[j]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func poolLess(a, b LLMPoolMemberStatus) bool {
|
||||
if a.Active != b.Active {
|
||||
return a.Active
|
||||
}
|
||||
return a.Priority > b.Priority
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package server
|
||||
|
||||
import "testing"
|
||||
|
||||
// The status list drives the "轮询顺序" strip in the UI, so it has to match the
|
||||
// order PoolProfiles actually runs: active first, then priority DESC, then id ASC
|
||||
// (input arrives id-ordered, so equal priorities must keep their relative order).
|
||||
func TestSortPoolStatusMatchesChainOrder(t *testing.T) {
|
||||
in := []LLMPoolMemberStatus{
|
||||
{ProfileID: "1", Name: "low", Priority: 0},
|
||||
{ProfileID: "2", Name: "active", Priority: 0, Active: true},
|
||||
{ProfileID: "3", Name: "high", Priority: 10},
|
||||
{ProfileID: "4", Name: "mid-a", Priority: 5},
|
||||
{ProfileID: "5", Name: "mid-b", Priority: 5},
|
||||
}
|
||||
sortPoolStatus(in)
|
||||
|
||||
want := []string{"active", "high", "mid-a", "mid-b", "low"}
|
||||
for i, w := range want {
|
||||
if in[i].Name != w {
|
||||
got := make([]string, len(in))
|
||||
for j, m := range in {
|
||||
got[j] = m.Name
|
||||
}
|
||||
t.Fatalf("order = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The active profile heads the chain no matter how low its own priority is —
|
||||
// that's the documented precedence, and the UI must not imply otherwise.
|
||||
func TestActiveProfileHeadsStatusList(t *testing.T) {
|
||||
in := []LLMPoolMemberStatus{
|
||||
{ProfileID: "1", Name: "loud", Priority: 999},
|
||||
{ProfileID: "2", Name: "active", Priority: -5, Active: true},
|
||||
}
|
||||
sortPoolStatus(in)
|
||||
if in[0].Name != "active" {
|
||||
t.Fatalf("head = %q, want the active profile regardless of priority", in[0].Name)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user