feat(llm): 五层重试的次数与间隔可配

一次模型调用的失败由内到外经过五层重试:建连(SDK)、空响应(SDK)、同 provider
安全窗口、轮询熔断、意图重跑。此前五层的次数与间隔全是写死的常量,遇到脾气各异
的中转站只能改代码。现在都能在 LLM 页配置,不配就是原来的默认值。

配置语义五层统一(db.RetryRule):次数 0=内置默认 / -1=关掉这层 / >0=该值;
间隔 0=原本的指数退避 / >0=改用固定间隔。0 被"未配置"占用,故关闭用 -1。
写库前夹紧到 次数 [-1,20]、间隔 [0,1h],DB 另有 CHECK 兜底。

粒度按层的性质分:前三层跟着端点走,可在每个模型配置里逐字段覆盖全局默认(只钉
间隔的仍继承全局次数);熔断与意图重跑是进程级语义,只有全局一份。

生效路径:①②是 provider 构建参数,保存后重建缓存;③随每次 hooks.current()
解析,故障转移换配置时重试节奏跟着换;④在失败路径上每次都要读,由 applyRetryPolicy
推给 llmpool.Registry;⑤每条意图收尾读一次。全部热生效,无需重启。

连接测试刻意不带这些参数——它有 30s 硬超时,叠上用户配的重试只会把本来能用的
端点测成超时失败。

norma 升 v0.3.5(llm.Config 新增 RetryInterval / EmptyResponseRetries /
EmptyResponseInterval)。旧库升级后行为逐字节不变:新列默认 0,settings 键不
存在即全默认。设计见 docs/LLM重试设计.md。
This commit is contained in:
Autumn-27 committed 2026-09-09 13:35:04 -04:00
1 parent 377ccc7e5a
commit a50bf02428
21 files changed
+1216 -146

No files matched your search

+5
View File
@@ -591,6 +591,10 @@ type LLMProfileDTO struct {
// 自定义会话头名:非空时每次请求带该 HTTP 头,头值=当前会话/意图的 session id。
// ''=不发送。用于按 session-id 头做提示缓存/粘性路由的网关。
SessionHeaderKey string `json:"session_header_key"`
// 本配置对重试的覆盖(建连/空响应/同 provider 安全窗口)。每项 attempts:
// 0=继承全局策略 | -1=关闭该层重试 | >0=次数;interval_ms: 0=用默认指数退避 |
// >0=改用该固定毫秒间隔。全 0 = 完全跟随全局,即历史行为。
Retry db.RetryOverride `json:"retry"`
}
func llmProfileDTO(p *db.LLMProfile) LLMProfileDTO {
@@ -614,6 +618,7 @@ func llmProfileDTO(p *db.LLMProfile) LLMProfileDTO {
MaxTokens: p.MaxTokens,
MaxTokensField: p.MaxTokensField,
SessionHeaderKey: p.SessionHeaderKey,
Retry: p.Retry,
}
}
+4 -3
View File
@@ -972,12 +972,13 @@ func (e *Engine) runIntent(ctx context.Context, t *Task, name string, worker *ag
// model_error 收场 → 额外重跑几次(退避后再试)。仅在意图仍属本 work、任务
// 未暂停/未终止/未取消【且未进入收尾】时重试;否则让位给对应分支处理(收尾期不
// 再重试,避免退避挤占其他 worker 的优雅收尾窗口)。
for attempt := 1; attempt <= modelErrorRetries &&
maxRetries, retryBackoff := e.modelErrorRetryPolicy()
for attempt := 1; attempt <= maxRetries &&
retryableWorkerModelError(reason, err) &&
workCtx.Err() == nil && ectx.Err() == nil && !e.IsPaused(t.ID) && !e.isSettling(t.ID); attempt++ {
log.Printf("[worker %s] task %s 意图 #%d model_error 收场,%v 后重试 (%d/%d)",
name, t.ID, intent.ID, modelErrorRetryBackoff, attempt, modelErrorRetries)
if sleepCtx(workCtx, modelErrorRetryBackoff) {
name, t.ID, intent.ID, retryBackoff, attempt, maxRetries)
if sleepCtx(workCtx, retryBackoff) {
break // 退避期间被取消(终止/暂停)→ 交给下方分支处理
}
e.BeginLLMCall(t.ID)
+80
View File
@@ -0,0 +1,80 @@
package server
import (
"time"
"github.com/Autumn-27/artex/agent"
"github.com/Autumn-27/artex/db"
)
// 重试策略的服务端解析,见 docs/LLM重试设计.md。五层里:
// - 建连 / 空响应 / 同 provider 安全窗口 是「跟着端点走」的,每个 LLM 配置可以覆盖
// 全局默认(profile 的某项留空就继承全局,全局也没配就用内置默认);
// - 熔断 / 意图重跑 是进程级的,只有全局一份。
//
// 全局策略读一次 DB 一行 settings,调用点都在低频路径(构建 provider、work 收尾、
// 保存配置),不值得再加一层缓存;熔断参数是例外——它在失败路径上每次都要读,所以
// 由 applyRetryPolicy 推给 Registry 保存。
// retryPolicy reads the global policy; a nil DB yields the zero policy (all
// layers on their built-in defaults).
func (s *Server) retryPolicy() db.LLMRetryPolicy {
if s.m == nil || s.m.pg == nil {
return db.LLMRetryPolicy{}
}
return s.m.pg.LLMRetryPolicy()
}
// resolveRetry layers one profile's override on top of the global policy and
// converts the result into the form agent.Config carries. Rules combine field by
// field, so a profile that only pins an interval still inherits the global count.
func resolveRetry(o db.RetryOverride, pol db.LLMRetryPolicy) agent.RetryConfig {
connect := o.Connect.Or(pol.Connect)
empty := o.Empty.Or(pol.Empty)
stream := o.Stream.Or(pol.Stream)
return agent.RetryConfig{
// 次数在这里保持「0=默认 / 负=关闭」的原始语义:SDK 的 MaxRetries /
// EmptyResponseRetries 与之完全同构,交给它自己解析即可。
ConnectAttempts: connect.Attempts, ConnectInterval: connect.Interval(),
EmptyAttempts: empty.Attempts, EmptyInterval: empty.Interval(),
StreamAttempts: stream.Attempts, StreamInterval: stream.Interval(),
}
}
// applyProfileRetry fills cfg.Retry for a profile read from the DB.
func (s *Server) applyProfileRetry(cfg *agent.Config, p *db.LLMProfile) {
if p == nil {
return
}
cfg.Retry = resolveRetry(p.Retry, s.retryPolicy())
}
// 熔断(轮询冷却)的默认值,与 llmpool 内置的一致 —— 这里只在「用户配了值」时才覆盖。
// 意图重跑的默认值见 engine.go 的 modelErrorRetries / modelErrorRetryBackoff。
// applyRetryPolicy pushes the process-wide layers of the policy into the objects
// that consume them on a hot path: the circuit-breaker registry. Called at
// startup and whenever the policy is saved.
func (s *Server) applyRetryPolicy() {
pol := s.retryPolicy()
if s.llmHealth != nil {
s.llmHealth.SetPolicy(pol.Breaker.Attempts, pol.Breaker.Interval())
}
}
// modelErrorRetryPolicy resolves the intent-level replay knobs (layer ⑤): how
// many times a model_error work is re-run and how long to back off between runs.
func (e *Engine) modelErrorRetryPolicy() (retries int, backoff time.Duration) {
retries, backoff = modelErrorRetries, modelErrorRetryBackoff
if e == nil || e.m == nil || e.m.pg == nil {
return retries, backoff
}
rule := e.m.pg.LLMRetryPolicy().Intent
if rule.Attempts != 0 {
retries = max(rule.Attempts, 0)
}
if d := rule.Interval(); d > 0 {
backoff = d
}
return retries, backoff
}
+97
View File
@@ -0,0 +1,97 @@
package server
import (
"testing"
"time"
"github.com/Autumn-27/artex/agent"
"github.com/Autumn-27/artex/db"
)
// 没有任何配置时,解析结果必须是全零 —— 也就是 SDK 与 task_llm 各自的内置默认,
// 与「重试可配」这个特性上线之前逐字节一致。
func TestResolveRetryUnconfigured(t *testing.T) {
got := resolveRetry(db.RetryOverride{}, db.LLMRetryPolicy{})
if got != (agent.RetryConfig{}) {
t.Fatalf("resolveRetry=%+v, want zero", got)
}
retries, backoff := sameProviderRetryPolicy(got)
if retries != sameProviderStreamRetries {
t.Fatalf("retries=%d, want %d", retries, sameProviderStreamRetries)
}
if d := backoff(1); d != time.Second {
t.Fatalf("backoff(1)=%v, want 1s (the default ladder)", d)
}
}
// The global policy applies to a profile that overrides nothing.
func TestResolveRetryFromGlobal(t *testing.T) {
pol := db.LLMRetryPolicy{
Connect: db.RetryRule{Attempts: 5, IntervalMS: 2000},
Empty: db.RetryRule{Attempts: -1},
Stream: db.RetryRule{Attempts: 4, IntervalMS: 1500},
}
got := resolveRetry(db.RetryOverride{}, pol)
want := agent.RetryConfig{
ConnectAttempts: 5, ConnectInterval: 2 * time.Second,
EmptyAttempts: -1,
StreamAttempts: 4, StreamInterval: 1500 * time.Millisecond,
}
if got != want {
t.Fatalf("resolveRetry=%+v, want %+v", got, want)
}
retries, backoff := sameProviderRetryPolicy(got)
if retries != 4 {
t.Fatalf("retries=%d, want 4", retries)
}
for _, attempt := range []int{0, 3} {
if d := backoff(attempt); d != 1500*time.Millisecond {
t.Fatalf("backoff(%d)=%v, want a fixed 1.5s", attempt, d)
}
}
}
// A profile override wins field by field: the pinned count replaces the global
// one, while the interval it left alone still comes from the global policy.
func TestResolveRetryProfileOverridesPerField(t *testing.T) {
pol := db.LLMRetryPolicy{
Connect: db.RetryRule{Attempts: 5, IntervalMS: 2000},
Stream: db.RetryRule{Attempts: 4, IntervalMS: 1500},
}
over := db.RetryOverride{
Connect: db.RetryRule{Attempts: 1},
Stream: db.RetryRule{IntervalMS: 250},
}
got := resolveRetry(over, pol)
if got.ConnectAttempts != 1 || got.ConnectInterval != 2*time.Second {
t.Fatalf("connect=%d/%v, want 1 attempt inheriting the 2s interval", got.ConnectAttempts, got.ConnectInterval)
}
if got.StreamAttempts != 4 || got.StreamInterval != 250*time.Millisecond {
t.Fatalf("stream=%d/%v, want the global 4 attempts with a pinned 250ms", got.StreamAttempts, got.StreamInterval)
}
}
// -1 turns the same-provider layer off entirely (0 retries), rather than being
// mistaken for "unset".
func TestSameProviderRetryDisabled(t *testing.T) {
retries, _ := sameProviderRetryPolicy(agent.RetryConfig{StreamAttempts: -1})
if retries != 0 {
t.Fatalf("retries=%d, want 0 (disabled)", retries)
}
}
// Values beyond the sane range are clamped on the way into the DB, so a hostile
// or fat-fingered payload can't park a worker for a day.
func TestRetryRuleClamp(t *testing.T) {
pol := db.LLMRetryPolicy{
Connect: db.RetryRule{Attempts: -50, IntervalMS: -1},
Stream: db.RetryRule{Attempts: 9999, IntervalMS: 99_999_999},
}
got := resolveRetry(db.RetryOverride{}, pol.Clamped())
if got.ConnectAttempts != -1 || got.ConnectInterval != 0 {
t.Fatalf("connect=%d/%v, want -1 attempts and no interval", got.ConnectAttempts, got.ConnectInterval)
}
if got.StreamAttempts != 20 || got.StreamInterval != time.Hour {
t.Fatalf("stream=%d/%v, want the 20-attempt / 1h caps", got.StreamAttempts, got.StreamInterval)
}
}
+13
View File
@@ -157,6 +157,9 @@ func New(ctx context.Context, m *Manager, skillDir string, dataDir string, keyDi
profAgents: map[int64]*profBundle{}, profChatAgents: map[int64]*agent.ChatAgent{},
provByProfile: map[int64]*provEntry{}, llmHealth: newLLMHealthRegistry(m.pg),
taskAgents: map[string]*taskAgentBundle{}, archiveWake: make(chan struct{}, 1)}
// 熔断阈值/冷却是失败路径上的热参数,启动时把全局重试策略推给 Registry 一次;
// 之后每次保存策略再推一次(saveLLMRetryPolicy)。
s.applyRetryPolicy()
// Every task uses a stable task router. An empty explicit chain is resolved by
// that router through Agent bindings and then the global provider, so adding a
// first chain to a running task takes effect on its very next LLM call.
@@ -314,6 +317,7 @@ func (s *Server) loadLLMConfig() (agent.Config, bool) {
cfg.Stream = p.Streaming
cfg.MaxTokens, cfg.MaxTokensField = p.MaxTokens, p.MaxTokensField
cfg.SessionHeaderKey = p.SessionHeaderKey
s.applyProfileRetry(&cfg, p)
if cfg.APIKey == "" {
return cfg, false
}
@@ -338,12 +342,14 @@ func (s *Server) saveLLMConfig(cfg agent.Config) error {
var maxTokens int
var maxTokensField string
var sessionHeaderKey string
var retry db.RetryOverride
if profs, _ := s.m.pg.ListProfiles(); profs != nil {
for _, p := range profs {
if p.Name == "default" {
id, priority, poolExclude, streaming = p.ID, p.Priority, p.PoolExclude, p.Streaming
maxTokens, maxTokensField = p.MaxTokens, p.MaxTokensField
sessionHeaderKey = p.SessionHeaderKey
retry = p.Retry
break
}
}
@@ -354,6 +360,7 @@ func (s *Server) saveLLMConfig(cfg agent.Config) error {
ContextWindowK: cfg.ContextWindowK, ThinkingType: cfg.ThinkingType, ReasoningEffort: cfg.ReasoningEffort, IsDefault: true,
Priority: priority, PoolExclude: poolExclude, Streaming: streaming,
MaxTokens: maxTokens, MaxTokensField: maxTokensField, SessionHeaderKey: sessionHeaderKey,
Retry: retry,
})
if err != nil {
return err
@@ -516,6 +523,7 @@ func (s *Server) loadProfileConfig(id int64) (agent.Config, bool) {
cfg.Stream = p.Streaming
cfg.MaxTokens, cfg.MaxTokensField = p.MaxTokens, p.MaxTokensField
cfg.SessionHeaderKey = p.SessionHeaderKey
s.applyProfileRetry(&cfg, p)
if cfg.APIKey == "" {
return cfg, false
}
@@ -925,6 +933,8 @@ func (s *Server) Handler() http.Handler {
mux.HandleFunc("POST /api/llm/profiles", s.pgSaveProfile)
mux.HandleFunc("DELETE /api/llm/profiles/{id}", s.pgDeleteProfile)
mux.HandleFunc("POST /api/llm/profiles/active", s.pgActivateProfile)
mux.HandleFunc("GET /api/llm/retry-policy", s.pgGetLLMRetryPolicy)
mux.HandleFunc("POST /api/llm/retry-policy", s.pgSaveLLMRetryPolicy)
mux.HandleFunc("GET /api/llm/pool", s.pgLLMPoolStatus)
mux.HandleFunc("POST /api/llm/pool/reset", s.pgLLMPoolReset)
mux.HandleFunc("POST /api/llm/models", s.pgListModels)
@@ -1411,6 +1421,9 @@ func (s *Server) testLLM(w http.ResponseWriter, r *http.Request) {
writeJSON(w, 200, map[string]any{"ok": false, "error": "未提供 API Key"})
return
}
// 重试参数【不】带进连接测试:测试有 30s 硬超时,把配置的重试次数/长间隔叠上去
// 只会让一个本来能用的端点测成"超时失败"。测试看的是"这个端点通不通",重试节奏
// 是跑起来之后的事。
lat, reply, err := agent.TestConnection(r.Context(), cfg)
if err != nil {
writeJSON(w, 200, map[string]any{"ok": false, "error": err.Error()})
+33
View File
@@ -1830,6 +1830,39 @@ func (s *Server) pgSaveProfile(w http.ResponseWriter, r *http.Request) {
writeJSON(w, 200, map[string]any{"id": id})
}
// pgGetLLMRetryPolicy 返回全局重试策略(五层各自的次数+间隔)。未配置过 → 全零,
// 前端把零显示成「默认」。
func (s *Server) pgGetLLMRetryPolicy(w http.ResponseWriter, r *http.Request) {
pg := s.pg(w)
if pg == nil {
return
}
writeJSON(w, 200, pg.LLMRetryPolicy())
}
// pgSaveLLMRetryPolicy 保存全局重试策略。三个「跟着端点走」的层(建连/空响应/同
// provider 安全窗口)是 provider 的构建参数或调用参数,改完必须让缓存里的 provider
// 重建;熔断参数则直接推给进程级 Registry。
func (s *Server) pgSaveLLMRetryPolicy(w http.ResponseWriter, r *http.Request) {
pg := s.pg(w)
if pg == nil {
return
}
var pol db.LLMRetryPolicy
if err := decode(r, &pol); err != nil {
writeErr(w, 400, err.Error())
return
}
if err := pg.SetLLMRetryPolicy(pol); err != nil {
writeErr(w, 500, err.Error())
return
}
s.applyRetryPolicy()
s.invalidateProfileAgents()
s.reapplyActiveProfile()
writeJSON(w, 200, pg.LLMRetryPolicy())
}
func (s *Server) pgDeleteProfile(w http.ResponseWriter, r *http.Request) {
pg := s.pg(w)
if pg == nil {
+50 -28
View File
@@ -91,6 +91,9 @@ type taskLLMSelection struct {
profileID int64
revision int64
provider llm.Provider
// retry 是这次选中的配置解析出来的重试参数(profile 覆盖 → 全局策略 → 内置默认)。
// 同 provider 安全窗口重试按它走,所以换了 profile 就换一套重试节奏。
retry agent.RetryConfig
}
type taskLLMStreamHooks struct {
@@ -105,38 +108,43 @@ type taskLLMStreamHooks struct {
// 构建失败时才降级到任务链,任务链为空时再降级到全局配置。
// 返回的 profile id 只有走任务链时才非零 —— streamTaskLLM 以此判断额度错误是否
// 应该推进任务的故障转移状态(绑定/全局路径不改任务链状态,沿用既有语义)。
func (r *taskLLMRuntime) current() (*Task, int64, int64, llm.Provider, error) {
func (r *taskLLMRuntime) current() (taskLLMSelection, error) {
taskNum, err := parseTaskID(r.taskID)
if err != nil {
return nil, 0, 0, nil, err
return taskLLMSelection{}, err
}
pt, err := r.s.m.pg.GetTask(taskNum)
if err != nil {
return nil, 0, 0, nil, err
return taskLLMSelection{}, err
}
if pt == nil {
return nil, 0, 0, nil, fmt.Errorf("task %s not found", r.taskID)
return taskLLMSelection{}, fmt.Errorf("task %s not found", r.taskID)
}
r.s.syncTaskLLMState(pt)
t, _ := r.s.m.Task(r.taskID)
if prov, _, ok := r.s.agentBindingProvider(r.agentKey); ok {
return t, 0, pt.LLMChainRevision, prov, nil
sel := taskLLMSelection{task: t, revision: pt.LLMChainRevision}
if prov, cfg, ok := r.s.agentBindingProvider(r.agentKey); ok {
sel.provider, sel.retry = prov, cfg.Retry
return sel, nil
}
if len(pt.LLMProfileIDs) > 0 {
if pt.ActiveLLMProfileID == nil {
return t, 0, pt.LLMChainRevision, nil, &taskLLMError{taskID: r.taskID, chainExhausted: true, cause: errors.New("all selected profiles are quota exhausted")}
return sel, &taskLLMError{taskID: r.taskID, chainExhausted: true, cause: errors.New("all selected profiles are quota exhausted")}
}
prov, _, ok := r.s.providerForProfile(*pt.ActiveLLMProfileID)
sel.profileID = *pt.ActiveLLMProfileID
prov, cfg, ok := r.s.providerForProfile(sel.profileID)
if !ok {
return t, *pt.ActiveLLMProfileID, pt.LLMChainRevision, nil, fmt.Errorf("LLM profile #%d is missing or invalid", *pt.ActiveLLMProfileID)
return sel, fmt.Errorf("LLM profile #%d is missing or invalid", sel.profileID)
}
return t, *pt.ActiveLLMProfileID, pt.LLMChainRevision, prov, nil
sel.provider, sel.retry = prov, cfg.Retry
return sel, nil
}
prov, _, ok := r.s.globalProvider()
prov, cfg, ok := r.s.globalProvider()
if !ok {
return t, 0, pt.LLMChainRevision, nil, fmt.Errorf("task %s has no available fallback LLM provider", r.taskID)
return sel, fmt.Errorf("task %s has no available fallback LLM provider", r.taskID)
}
return t, 0, pt.LLMChainRevision, prov, nil
sel.provider, sel.retry = prov, cfg.Retry
return sel, nil
}
// activeCfg resolves the task's currently-active LLM config, mirroring current()'s
@@ -194,10 +202,7 @@ func parseTaskID(id string) (int64, error) {
// exhausted, and how to emit a failover transition.
func (r *taskLLMRuntime) streamHooks() taskLLMStreamHooks {
return taskLLMStreamHooks{
current: func() (taskLLMSelection, error) {
task, profileID, revision, provider, err := r.current()
return taskLLMSelection{task: task, profileID: profileID, revision: revision, provider: provider}, err
},
current: r.current,
exhaust: func(selection taskLLMSelection, cause error) (db.TaskLLMTransition, error) {
taskNum, _ := parseTaskID(r.taskID)
transition, err := r.s.m.pg.MarkTaskLLMProfileQuotaExhaustedAtRevision(taskNum, selection.profileID, selection.revision, cause.Error())
@@ -244,13 +249,14 @@ func completeTaskLLM(ctx context.Context, taskID string, req llm.CompletionReque
)
// 同 provider 安全窗口重试:非流式调用要么整体成功、要么整体失败,没有
// 中途已交付输出的问题,所以任何瞬时失败都可原样重试。
retries, backoffOf := sameProviderRetryPolicy(selection.retry)
for attempt := 0; ; attempt++ {
msg, sr, usage, callErr = selection.provider.Complete(ctx, req)
if callErr != nil && ctx.Err() == nil &&
attempt < sameProviderStreamRetries && isRetryableStreamError(callErr) {
backoff := sameProviderRetryBackoff(attempt)
attempt < retries && isRetryableStreamError(callErr) {
backoff := backoffOf(attempt)
log.Printf("[task-llm] task %s 非流式调用失败,%v 后同 provider 重试 (%d/%d): %v",
taskID, backoff, attempt+1, sameProviderStreamRetries, callErr)
taskID, backoff, attempt+1, retries, callErr)
if sleepCtx(ctx, backoff) {
break // 退避期间 ctx 取消 → 停止重试
}
@@ -290,6 +296,7 @@ func streamTaskLLM(ctx context.Context, taskID string, req llm.CompletionRequest
committed := false
var pending []llm.StreamEvent
var streamErr error
retries, backoffOf := sameProviderRetryPolicy(selection.retry)
// 同 provider 安全窗口重试:committed 之前(还没向调用方交付任何输出)
// 的瞬时失败可以原样重放,不会重复模型输出或工具执行。committed 之后、
// ctx 取消、或确定性/额度错误则跳出,交给下方原有的透传/故障转移逻辑。
@@ -320,10 +327,10 @@ func streamTaskLLM(ctx context.Context, taskID string, req llm.CompletionRequest
}
}
if streamErr != nil && !committed && ctx.Err() == nil &&
attempt < sameProviderStreamRetries && isRetryableStreamError(streamErr) {
backoff := sameProviderRetryBackoff(attempt)
attempt < retries && isRetryableStreamError(streamErr) {
backoff := backoffOf(attempt)
log.Printf("[task-llm] task %s 提交前流失败,%v 后同 provider 重试 (%d/%d): %v",
taskID, backoff, attempt+1, sameProviderStreamRetries, streamErr)
taskID, backoff, attempt+1, retries, streamErr)
if sleepCtx(ctx, backoff) {
break // 退避期间 ctx 取消 → 停止重试
}
@@ -388,20 +395,35 @@ func streamEventCommitsOutput(event llm.StreamEvent) bool {
}
}
// 提交前安全窗口内、对同一 provider 的重试次数。SDK 的 doStream 只重试建连阶段
// (拿到 200 之前);流一旦开始,中途断流 / overloaded / 流内 429 等瞬时故障会直接
// 提交前安全窗口内、对同一 provider 的【默认】重试次数。SDK 的 doStream 只重试建连
// 阶段(拿到 200 之前);流一旦开始,中途断流 / overloaded / 流内 429 等瞬时故障会直接
// 冒泡成 model_error,零重试。只要一个 token 都还没交给调用方(!committed),重放
// 完全相同的请求就不会重复模型输出或工具副作用,因此这里补一层同 provider 退避重试,
// 把这类抖动挡在意图整体重跑之前。
// 把这类抖动挡在意图整体重跑之前。可被 LLM 配置的重试覆盖/全局重试策略改写。
const sameProviderStreamRetries = 2
// sameProviderRetryBackoff 是第 attempt 次重试前的退避(0.5s、1s…,上限 4s),
// 与 SDK backoffSleep 同风格但封顶更小,避免拖住 worker 的收尾/取消响应。
// sameProviderRetryBackoff 是第 attempt 次重试前的【默认】退避(0.5s、1s…,上限 4s),
// 与 SDK 的指数梯度同风格但封顶更小,避免拖住 worker 的收尾/取消响应。
// 以变量形式暴露,便于测试将退避置零。
var sameProviderRetryBackoff = func(attempt int) time.Duration {
return min(500*time.Millisecond*(1<<attempt), 4*time.Second)
}
// sameProviderRetryPolicy 解析这次调用用哪套同 provider 重试参数:配置里设了次数就
// 用配置的(负数 = 关掉这层重试),设了间隔就把指数退避换成固定间隔,两者都没设时
// 与改可配之前逐字节一致。
func sameProviderRetryPolicy(r agent.RetryConfig) (retries int, backoff func(int) time.Duration) {
retries, backoff = sameProviderStreamRetries, sameProviderRetryBackoff
if r.StreamAttempts != 0 {
retries = max(r.StreamAttempts, 0)
}
if r.StreamInterval > 0 {
d := r.StreamInterval
backoff = func(int) time.Duration { return d }
}
return retries, backoff
}
// isRetryableStreamError 判断「提交前的流失败」是否值得在同一 provider 上重放。
// 瞬时的传输中断 / 供应商过载 / 限流会自行恢复,可安全重试;而以下三类不重试:
// - 额度耗尽:交给 profile 故障转移处理,别在这里白烧重试