fix(llm): 提交前安全窗口内对同一 provider 重试,减少 model_error
SDK 的 doStream 只重试建连阶段(拿到 200 之前);流一旦开始,中途断流 / overloaded / 流内 429 等瞬时故障会直接冒泡成 model_error,零重试。表现为 worker 常出现「无总结,终态 model_error」,甚至只跑了一两个工具就中断。 在 streamTaskLLM(所有任务 agent 的统一出口)里补一层同 provider 退避重试: - 仅在 committed 之前(还没向调用方交付任何输出)触发,重放相同请求不会 重复模型输出或工具副作用,绝对安全 - 最多 2 次,退避 0.5s→1s(封顶 4s),ctx 取消立即让位 - 只重试瞬时故障(传输 reset/EOF/timeout、408/429/5xx、流内 overloaded); 额度耗尽交给 profile 故障转移、上下文过长交给 harness reactive 压缩、 4xx 确定性拒绝一律不重试 - 给不上时原样落到既有透传/故障转移逻辑,行为不变 新增 3 个回归测试(重试到成功 / 重试耗尽如实上报 / 4xx 不重试),退避以 变量暴露便于测试置零。go test -race 全绿。
This commit is contained in:
1 parent
d8f34cc79d
commit
9d8e0ba988
2 files changed
+203
-19
No files matched your search
+124
-1
@@ -36,6 +36,42 @@ func (p *scriptedLLMProvider) Stream(context.Context, llm.CompletionRequest) ite
|
||||
}
|
||||
}
|
||||
|
||||
// withZeroRetryBackoff sets the same-provider retry backoff to zero for the
|
||||
// duration of a (serial) test and returns a restore func for defer.
|
||||
func withZeroRetryBackoff() func() {
|
||||
prev := sameProviderRetryBackoff
|
||||
sameProviderRetryBackoff = func(int) time.Duration { return 0 }
|
||||
return func() { sameProviderRetryBackoff = prev }
|
||||
}
|
||||
|
||||
// flakyThenOKProvider fails its first failCount stream attempts pre-commit
|
||||
// (emitting only a non-committing SEMessageStart before the error, mirroring a
|
||||
// gateway that returns 200 then drops), then serves okEvents.
|
||||
type flakyThenOKProvider struct {
|
||||
failCount int
|
||||
failErr error
|
||||
okEvents []llm.StreamEvent
|
||||
calls int
|
||||
}
|
||||
|
||||
func (p *flakyThenOKProvider) Stream(context.Context, llm.CompletionRequest) iter.Seq2[llm.StreamEvent, error] {
|
||||
return func(yield func(llm.StreamEvent, error) bool) {
|
||||
p.calls++
|
||||
if p.calls <= p.failCount {
|
||||
if !yield(llm.StreamEvent{Type: llm.SEMessageStart}, nil) {
|
||||
return
|
||||
}
|
||||
yield(llm.StreamEvent{}, p.failErr)
|
||||
return
|
||||
}
|
||||
for _, ev := range p.okEvents {
|
||||
if !yield(ev, nil) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func collectTaskLLMStream(seq iter.Seq2[llm.StreamEvent, error]) ([]llm.StreamEvent, error) {
|
||||
var events []llm.StreamEvent
|
||||
var streamErr error
|
||||
@@ -265,7 +301,9 @@ func TestTaskLLMStreamRetriesStalePreStreamFailureAgainstReplacementChain(t *tes
|
||||
}
|
||||
|
||||
func TestTaskLLMStreamDoesNotSwitchForOrdinaryErrors(t *testing.T) {
|
||||
t.Parallel()
|
||||
// Not parallel: overrides the package-level backoff so the retryable cases
|
||||
// don't sleep. Serial tests never overlap the parallel batch, so this is safe.
|
||||
defer withZeroRetryBackoff()()
|
||||
tests := []string{
|
||||
"openai: status 429: rate limit exceeded",
|
||||
"openai: status 401: invalid api key",
|
||||
@@ -379,6 +417,91 @@ func TestTaskLLMRuntimeErrorSkipsWorkerReplay(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A transient pre-commit stream failure (200-then-drop / overloaded / 5xx) is
|
||||
// retried on the SAME provider, without switching profiles, and succeeds once the
|
||||
// blip clears — instead of surfacing as a model_error.
|
||||
func TestTaskLLMStreamRetriesTransientPreCommitOnSameProvider(t *testing.T) {
|
||||
defer withZeroRetryBackoff()()
|
||||
provider := &flakyThenOKProvider{
|
||||
failCount: 2, // fail twice, succeed on the 3rd attempt (initial + 2 retries)
|
||||
failErr: errors.New("anthropic: overloaded_error"),
|
||||
okEvents: []llm.StreamEvent{{Type: llm.SEMessageStart}, {Type: llm.SETextDelta, Text: "ok"}},
|
||||
}
|
||||
exhausted := 0
|
||||
hooks := taskLLMStreamHooks{
|
||||
current: func() (taskLLMSelection, error) {
|
||||
return taskLLMSelection{profileID: 11, provider: provider}, nil
|
||||
},
|
||||
exhaust: func(taskLLMSelection, error) (db.TaskLLMTransition, error) {
|
||||
exhausted++
|
||||
return db.TaskLLMTransition{}, nil
|
||||
},
|
||||
}
|
||||
events, err := collectTaskLLMStream(streamTaskLLM(context.Background(), "7", llm.CompletionRequest{}, hooks))
|
||||
if err != nil {
|
||||
t.Fatalf("transient pre-commit failure should have been retried to success, got %v", err)
|
||||
}
|
||||
if provider.calls != 3 {
|
||||
t.Fatalf("provider calls=%d, want 3 (1 initial + 2 retries)", provider.calls)
|
||||
}
|
||||
if exhausted != 0 {
|
||||
t.Fatalf("same-provider retry must not change failover state (exhausted=%d)", exhausted)
|
||||
}
|
||||
if len(events) == 0 || events[len(events)-1].Text != "ok" {
|
||||
t.Fatalf("recovered stream missing its committed output: %+v", events)
|
||||
}
|
||||
}
|
||||
|
||||
// Once retries are exhausted, the transient error is surfaced (becoming a
|
||||
// model_error upstream) after exactly sameProviderStreamRetries+1 attempts.
|
||||
func TestTaskLLMStreamSurfacesTransientAfterRetriesExhausted(t *testing.T) {
|
||||
defer withZeroRetryBackoff()()
|
||||
provider := &flakyThenOKProvider{
|
||||
failCount: 99, // never recovers
|
||||
failErr: errors.New("dial tcp: connection reset by peer"),
|
||||
}
|
||||
hooks := taskLLMStreamHooks{
|
||||
current: func() (taskLLMSelection, error) {
|
||||
return taskLLMSelection{profileID: 11, provider: provider}, nil
|
||||
},
|
||||
exhaust: func(taskLLMSelection, error) (db.TaskLLMTransition, error) {
|
||||
return db.TaskLLMTransition{}, nil
|
||||
},
|
||||
}
|
||||
_, err := collectTaskLLMStream(streamTaskLLM(context.Background(), "7", llm.CompletionRequest{}, hooks))
|
||||
if err == nil {
|
||||
t.Fatal("persistent transient failure must still surface an error")
|
||||
}
|
||||
if provider.calls != sameProviderStreamRetries+1 {
|
||||
t.Fatalf("provider calls=%d, want %d", provider.calls, sameProviderStreamRetries+1)
|
||||
}
|
||||
}
|
||||
|
||||
// A deterministic 4xx rejection is NOT retried on the same provider — replaying
|
||||
// the identical request everywhere fails the same way.
|
||||
func TestTaskLLMStreamDoesNotRetryDeterministicRejection(t *testing.T) {
|
||||
defer withZeroRetryBackoff()()
|
||||
provider := &flakyThenOKProvider{
|
||||
failCount: 99,
|
||||
failErr: errors.New("anthropic: status 400: messages.1: invalid request"),
|
||||
}
|
||||
hooks := taskLLMStreamHooks{
|
||||
current: func() (taskLLMSelection, error) {
|
||||
return taskLLMSelection{profileID: 11, provider: provider}, nil
|
||||
},
|
||||
exhaust: func(taskLLMSelection, error) (db.TaskLLMTransition, error) {
|
||||
return db.TaskLLMTransition{}, nil
|
||||
},
|
||||
}
|
||||
_, err := collectTaskLLMStream(streamTaskLLM(context.Background(), "7", llm.CompletionRequest{}, hooks))
|
||||
if err == nil {
|
||||
t.Fatal("expected the deterministic rejection to surface")
|
||||
}
|
||||
if provider.calls != 1 {
|
||||
t.Fatalf("deterministic 4xx must not be retried: provider calls=%d, want 1", provider.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEngineReadyForTaskOverride(t *testing.T) {
|
||||
t.Parallel()
|
||||
e := &Engine{}
|
||||
|
||||
Reference in new issue
Block a user