feat(llm): 每个 LLM 配置支持流式/非流式切换(默认流式)
依赖 norma v0.3.0 的 Provider.Complete。为每个 llm_profile 增加 streaming 开关:开(默认)走流式 SSE;关走真·非流式(stream:false、一次性返回完整 JSON),可绕开部分网关糟糕的 SSE 实现(空帧、思考字段丢帧),代价是失去 运行中的实时进度与实时 token 计数。 - agent.Config.Stream → agentcore.Options.NonStreaming = !Stream;worker / planner / mainagent / chat / goals 经 SetNonStreaming resolver 注入,任务 路由按当前激活 profile 动态取值 - llmpool.Pool / llmrec.Recorder / taskLLMRuntime 三个 Provider 包装均实现 Complete(分别带故障转移 / 录制 / 同 provider 重试与额度 failover),使 输出、token 统计、操作统计、LLM 录制四条链路在非流式下保持兼容 - db: llm_profiles 新增 streaming 列 + ALTER 补旧库(默认 true,旧配置无感) - 系统 → LLM 配置表单增加「流式输出」开关
This commit is contained in:
1 parent
19a7206d12
commit
587ba422b4
22 files changed
+409
-75
No files matched your search
@@ -122,6 +122,44 @@ func (p *Pool) Stream(ctx context.Context, req llm.CompletionRequest) iter.Seq2[
|
||||
}
|
||||
}
|
||||
|
||||
// Complete implements llm.Provider with failover for non-streaming calls. A
|
||||
// non-streaming request is atomic — it never delivers partial output — so every
|
||||
// failover-eligible failure lands in the safe window and the next member can be
|
||||
// tried without risk of duplicated output. Mirrors Stream's health-tripping and
|
||||
// chain-exhaustion behavior.
|
||||
func (p *Pool) Complete(ctx context.Context, req llm.CompletionRequest) (llm.Message, string, llm.Usage, error) {
|
||||
order := p.order(req)
|
||||
var lastErr error
|
||||
for i, m := range order {
|
||||
msg, sr, usage, err := m.Prov.Complete(ctx, req)
|
||||
if err == nil {
|
||||
p.health.Pass(m.ID)
|
||||
return msg, sr, usage, nil
|
||||
}
|
||||
if !shouldFailover(ctx, err) {
|
||||
// Non-failover error (e.g. ctx cancel, deterministic 4xx): surface as-is
|
||||
// without tripping health, matching Stream's non-failover path.
|
||||
p.health.Pass(m.ID)
|
||||
return llm.Message{}, "", llm.Usage{}, err
|
||||
}
|
||||
lastErr = err
|
||||
hard := isHardFailure(err)
|
||||
if p.health.Trip(m.ID, trimErr(err), hard) {
|
||||
log.Printf("[llmpool] 配置 %q(%s) 已熔断:%s", m.Name, m.Model, trimErr(err))
|
||||
}
|
||||
if i+1 < len(order) {
|
||||
n := order[i+1]
|
||||
log.Printf("[llmpool] LLM 故障转移:%q(%s) → %q(%s),原因:%s",
|
||||
m.Name, m.Model, n.Name, n.Model, trimErr(err))
|
||||
}
|
||||
}
|
||||
if lastErr == nil {
|
||||
lastErr = ErrExhausted
|
||||
}
|
||||
log.Printf("[llmpool] 轮询链已耗尽(%d 个配置全部失败),最后错误:%s", len(order), trimErr(lastErr))
|
||||
return llm.Message{}, "", llm.Usage{}, fmt.Errorf("%w:%v", ErrExhausted, lastErr)
|
||||
}
|
||||
|
||||
// order picks the members to try, in order: skip those in a cooling-off window
|
||||
// and those whose context window can't hold this request, then rotate within each
|
||||
// equal-priority group so same-priority profiles share the load. Never returns an
|
||||
|
||||
@@ -55,6 +55,17 @@ func (f *fakeProv) Stream(ctx context.Context, req llm.CompletionRequest) iter.S
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fakeProv) Complete(ctx context.Context, req llm.CompletionRequest) (llm.Message, string, llm.Usage, error) {
|
||||
acc := llm.NewAccumulator()
|
||||
for ev, err := range f.Stream(ctx, req) {
|
||||
if err != nil {
|
||||
return llm.Message{}, "", llm.Usage{}, err
|
||||
}
|
||||
acc.Add(ev)
|
||||
}
|
||||
return acc.Message(), acc.StopReason, acc.Usage, nil
|
||||
}
|
||||
|
||||
// ok builds a provider that always succeeds with one text delta.
|
||||
func okProv(name, text string) *fakeProv {
|
||||
return &fakeProv{name: name, events: [][]llm.StreamEvent{{{Type: llm.SETextDelta, Text: text}}}}
|
||||
|
||||
Reference in new issue
Block a user