fix(llm): 提交前安全窗口内对同一 provider 重试,减少 model_error

SDK 的 doStream 只重试建连阶段(拿到 200 之前);流一旦开始,中途断流 /
overloaded / 流内 429 等瞬时故障会直接冒泡成 model_error,零重试。表现为
worker 常出现「无总结,终态 model_error」,甚至只跑了一两个工具就中断。

在 streamTaskLLM(所有任务 agent 的统一出口)里补一层同 provider 退避重试:
- 仅在 committed 之前(还没向调用方交付任何输出)触发,重放相同请求不会
  重复模型输出或工具副作用,绝对安全
- 最多 2 次,退避 0.5s→1s(封顶 4s),ctx 取消立即让位
- 只重试瞬时故障(传输 reset/EOF/timeout、408/429/5xx、流内 overloaded);
  额度耗尽交给 profile 故障转移、上下文过长交给 harness reactive 压缩、
  4xx 确定性拒绝一律不重试
- 给不上时原样落到既有透传/故障转移逻辑,行为不变

新增 3 个回归测试(重试到成功 / 重试耗尽如实上报 / 4xx 不重试),退避以
变量暴露便于测试置零。go test -race 全绿。
This commit is contained in:
Autumn-27 committed 2026-08-23 13:02:54 -04:00
1 parent d8f34cc79d
commit 9d8e0ba988
2 files changed
+203 -19

No files matched your search

+124 -1
View File
@@ -36,6 +36,42 @@ func (p *scriptedLLMProvider) Stream(context.Context, llm.CompletionRequest) ite
}
}
// withZeroRetryBackoff sets the same-provider retry backoff to zero for the
// duration of a (serial) test and returns a restore func for defer.
func withZeroRetryBackoff() func() {
prev := sameProviderRetryBackoff
sameProviderRetryBackoff = func(int) time.Duration { return 0 }
return func() { sameProviderRetryBackoff = prev }
}
// flakyThenOKProvider fails its first failCount stream attempts pre-commit
// (emitting only a non-committing SEMessageStart before the error, mirroring a
// gateway that returns 200 then drops), then serves okEvents.
type flakyThenOKProvider struct {
failCount int
failErr error
okEvents []llm.StreamEvent
calls int
}
func (p *flakyThenOKProvider) Stream(context.Context, llm.CompletionRequest) iter.Seq2[llm.StreamEvent, error] {
return func(yield func(llm.StreamEvent, error) bool) {
p.calls++
if p.calls <= p.failCount {
if !yield(llm.StreamEvent{Type: llm.SEMessageStart}, nil) {
return
}
yield(llm.StreamEvent{}, p.failErr)
return
}
for _, ev := range p.okEvents {
if !yield(ev, nil) {
return
}
}
}
}
func collectTaskLLMStream(seq iter.Seq2[llm.StreamEvent, error]) ([]llm.StreamEvent, error) {
var events []llm.StreamEvent
var streamErr error
@@ -265,7 +301,9 @@ func TestTaskLLMStreamRetriesStalePreStreamFailureAgainstReplacementChain(t *tes
}
func TestTaskLLMStreamDoesNotSwitchForOrdinaryErrors(t *testing.T) {
t.Parallel()
// Not parallel: overrides the package-level backoff so the retryable cases
// don't sleep. Serial tests never overlap the parallel batch, so this is safe.
defer withZeroRetryBackoff()()
tests := []string{
"openai: status 429: rate limit exceeded",
"openai: status 401: invalid api key",
@@ -379,6 +417,91 @@ func TestTaskLLMRuntimeErrorSkipsWorkerReplay(t *testing.T) {
}
}
// A transient pre-commit stream failure (200-then-drop / overloaded / 5xx) is
// retried on the SAME provider, without switching profiles, and succeeds once the
// blip clears — instead of surfacing as a model_error.
func TestTaskLLMStreamRetriesTransientPreCommitOnSameProvider(t *testing.T) {
defer withZeroRetryBackoff()()
provider := &flakyThenOKProvider{
failCount: 2, // fail twice, succeed on the 3rd attempt (initial + 2 retries)
failErr: errors.New("anthropic: overloaded_error"),
okEvents: []llm.StreamEvent{{Type: llm.SEMessageStart}, {Type: llm.SETextDelta, Text: "ok"}},
}
exhausted := 0
hooks := taskLLMStreamHooks{
current: func() (taskLLMSelection, error) {
return taskLLMSelection{profileID: 11, provider: provider}, nil
},
exhaust: func(taskLLMSelection, error) (db.TaskLLMTransition, error) {
exhausted++
return db.TaskLLMTransition{}, nil
},
}
events, err := collectTaskLLMStream(streamTaskLLM(context.Background(), "7", llm.CompletionRequest{}, hooks))
if err != nil {
t.Fatalf("transient pre-commit failure should have been retried to success, got %v", err)
}
if provider.calls != 3 {
t.Fatalf("provider calls=%d, want 3 (1 initial + 2 retries)", provider.calls)
}
if exhausted != 0 {
t.Fatalf("same-provider retry must not change failover state (exhausted=%d)", exhausted)
}
if len(events) == 0 || events[len(events)-1].Text != "ok" {
t.Fatalf("recovered stream missing its committed output: %+v", events)
}
}
// Once retries are exhausted, the transient error is surfaced (becoming a
// model_error upstream) after exactly sameProviderStreamRetries+1 attempts.
func TestTaskLLMStreamSurfacesTransientAfterRetriesExhausted(t *testing.T) {
defer withZeroRetryBackoff()()
provider := &flakyThenOKProvider{
failCount: 99, // never recovers
failErr: errors.New("dial tcp: connection reset by peer"),
}
hooks := taskLLMStreamHooks{
current: func() (taskLLMSelection, error) {
return taskLLMSelection{profileID: 11, provider: provider}, nil
},
exhaust: func(taskLLMSelection, error) (db.TaskLLMTransition, error) {
return db.TaskLLMTransition{}, nil
},
}
_, err := collectTaskLLMStream(streamTaskLLM(context.Background(), "7", llm.CompletionRequest{}, hooks))
if err == nil {
t.Fatal("persistent transient failure must still surface an error")
}
if provider.calls != sameProviderStreamRetries+1 {
t.Fatalf("provider calls=%d, want %d", provider.calls, sameProviderStreamRetries+1)
}
}
// A deterministic 4xx rejection is NOT retried on the same provider — replaying
// the identical request everywhere fails the same way.
func TestTaskLLMStreamDoesNotRetryDeterministicRejection(t *testing.T) {
defer withZeroRetryBackoff()()
provider := &flakyThenOKProvider{
failCount: 99,
failErr: errors.New("anthropic: status 400: messages.1: invalid request"),
}
hooks := taskLLMStreamHooks{
current: func() (taskLLMSelection, error) {
return taskLLMSelection{profileID: 11, provider: provider}, nil
},
exhaust: func(taskLLMSelection, error) (db.TaskLLMTransition, error) {
return db.TaskLLMTransition{}, nil
},
}
_, err := collectTaskLLMStream(streamTaskLLM(context.Background(), "7", llm.CompletionRequest{}, hooks))
if err == nil {
t.Fatal("expected the deterministic rejection to surface")
}
if provider.calls != 1 {
t.Fatalf("deterministic 4xx must not be retried: provider calls=%d, want 1", provider.calls)
}
}
func TestEngineReadyForTaskOverride(t *testing.T) {
t.Parallel()
e := &Engine{}