feat(llm): add profile failover and pool health

agent/provider.go

@@ -8,8 +8,14 @@
 package agent
 
 import (
+	"bytes"
 	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
 	"os"
+	"regexp"
 	"strings"
 	"time"
 
@@ -36,9 +42,14 @@ type Config struct {
 	// ContextWindowK is the model's context window in K tokens (user-configured),
 	// used to size compaction thresholds. 0 = default; see CompactionWindow.
 	ContextWindowK int
-	// ReasoningEffort selects the thinking mode: "" = 默认(不发送思考参数);
-	// "off" = 关闭(thinking.type=disabled); "low"/"medium"/"high"/"max" = 开启并设强度.
-	// NewProvider derives the provider-specific request fields from it.
+	// ThinkingType 独立控制思考「开关」字段(thinking.type):
+	//   "" = 不发送(默认,兼容不支持该字段的模型); "disabled" = 显式关闭;
+	//   "enabled" = 开启. 与 ReasoningEffort 完全解耦——有些接口没有 thinking 字段、
+	//   只靠强度参数就能激活思考,故两者可各自单独设置.
+	ThinkingType string
+	// ReasoningEffort 独立控制思考「强度」字段:
+	//   "" = 不发送(默认); "low"/"medium"/"high"/"xhigh"/"max" = 对应强度.
+	//   OpenAI 映射为顶层 reasoning_effort;Anthropic 映射为 output_config.effort.
 	ReasoningEffort string
 }
 
@@ -168,32 +179,27 @@ func (c Config) Provider() string {
 // limiter lives on the single provider instance — so planner + all workers +
 // main agent (which share this provider) are bounded by one shared rate limit.
 func (c Config) NewProvider() (llm.Provider, error) {
+	client, err := quotaAwareHTTPClient(c.Proxy)
+	if err != nil {
+		return nil, err
+	}
 	lc := llm.Config{
-		Format:  c.Format,
-		BaseURL: c.BaseURL,
-		APIKey:  c.APIKey,
-		Model:   c.Model,
-		Proxy:   c.Proxy,
-	}
-	// Derive the provider thinking params from the single UI selector:
-	//   "" → omit both (provider default); "off" → disable; else → enable + effort.
-	switch c.ReasoningEffort {
-	case "":
-		// leave both empty → omitted from the request
-	case "off":
-		lc.ThinkingType = "disabled"
-	default:
-		lc.ThinkingType = "enabled"
-		lc.ReasoningEffort = c.ReasoningEffort
+		Format:     c.Format,
+		BaseURL:    c.BaseURL,
+		APIKey:     c.APIKey,
+		Model:      c.Model,
+		HTTPClient: client,
 	}
+	// 思考开关与强度两个字段各自透传(空 = 该字段不发送)。二者解耦:
+	// 可只发 thinking.type、只发 effort、都发、或都不发。
+	lc.ThinkingType = c.ThinkingType
+	lc.ReasoningEffort = c.ReasoningEffort
 	if c.RatePerSecond > 0 || c.RatePerMinute > 0 {
 		lc.RateLimit = &llm.RateLimit{PerSecond: c.RatePerSecond, PerMinute: c.RatePerMinute}
 	}
 	return llm.NewProvider(lc)
 }
 
-<<<<<<< Updated upstream
-=======
 // IsQuotaExhaustedMessage deliberately recognizes only explicit balance,
 // billing, credit, or quota-exhaustion signals. Generic 429/rate-limit text,
 // authentication failures, network errors, and server failures are excluded.
@@ -281,7 +287,6 @@ func quotaAwareHTTPClient(proxy string) (*http.Client, error) {
 	return &http.Client{Transport: quotaAwareTransport{base: transport}}, nil
 }
 
->>>>>>> Stashed changes
 // TestConnection makes a minimal real completion to verify the provider/model/
 // endpoint/key actually work. Returns the round-trip latency.
 func TestConnection(ctx context.Context, c Config) (time.Duration, error) {

agent/provider_quota_test.go

@@ -0,0 +1,131 @@
+package agent
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync/atomic"
+	"testing"
+
+	"github.com/Autumn-27/norma/llm"
+)
+
+type roundTripperFunc func(*http.Request) (*http.Response, error)
+
+func (f roundTripperFunc) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) }
+
+func TestIsQuotaExhaustedMessage(t *testing.T) {
+	t.Parallel()
+	positive := []string{
+		`{"error":{"code":"insufficient_quota"}}`,
+		`RESOURCE_EXHAUSTED`,
+		`You exceeded your current quota, please check your plan and billing details.`,
+		`billing_not_active`,
+		`credit balance is too low`,
+		`账户余额不足，请充值`,
+	}
+	for _, message := range positive {
+		if !IsQuotaExhaustedMessage(message) {
+			t.Errorf("expected quota classification for %q", message)
+		}
+	}
+
+	negative := []string{
+		`status 429: rate limit exceeded`,
+		`RESOURCE_EXHAUSTED: rate limit exceeded`,
+		`too many requests per minute`,
+		`quota exceeded for quota metric GenerateRequestsPerMinutePerProjectPerBaseModel`,
+		`RESOURCE_EXHAUSTED: TPM quota exceeded`,
+		`tokens per minute quota exceeded`,
+		`rate_limit_exceeded: requests per second`,
+		`status 401: invalid api key`,
+		`status 401: insufficient_quota`,
+		`HTTP 403: billing_hard_limit_reached`,
+		`status 500: internal server error`,
+		`status=503: insufficient_quota`,
+		`context length exceeded`,
+	}
+	for _, message := range negative {
+		if IsQuotaExhaustedMessage(message) {
+			t.Errorf("unexpected quota classification for %q", message)
+		}
+	}
+}
+
+func TestQuotaAwareTransportOnlyNormalizesExplicitQuota429(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name       string
+		status     int
+		body       string
+		wantStatus int
+	}{
+		{name: "quota", status: http.StatusTooManyRequests, body: `{"code":"insufficient_quota"}`, wantStatus: http.StatusPaymentRequired},
+		{name: "ordinary rate limit", status: http.StatusTooManyRequests, body: `{"message":"rate limit exceeded"}`, wantStatus: http.StatusTooManyRequests},
+		{name: "server error", status: http.StatusInternalServerError, body: `insufficient_quota`, wantStatus: http.StatusInternalServerError},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			transport := quotaAwareTransport{base: roundTripperFunc(func(*http.Request) (*http.Response, error) {
+				return &http.Response{
+					StatusCode: tt.status,
+					Status:     http.StatusText(tt.status),
+					Body:       io.NopCloser(strings.NewReader(tt.body)),
+					Header:     make(http.Header),
+				}, nil
+			})}
+			req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, "https://example.invalid", nil)
+			if err != nil {
+				t.Fatal(err)
+			}
+			resp, err := transport.RoundTrip(req)
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer resp.Body.Close()
+			if resp.StatusCode != tt.wantStatus {
+				t.Fatalf("status=%d, want %d", resp.StatusCode, tt.wantStatus)
+			}
+			gotBody, err := io.ReadAll(resp.Body)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if string(gotBody) != tt.body {
+				t.Fatalf("body=%q, want %q", gotBody, tt.body)
+			}
+		})
+	}
+}
+
+func TestProviderDoesNotRetryExplicitQuota429(t *testing.T) {
+	t.Parallel()
+	var requests atomic.Int32
+	upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		requests.Add(1)
+		w.Header().Set("content-type", "application/json")
+		w.WriteHeader(http.StatusTooManyRequests)
+		_, _ = io.WriteString(w, `{"error":{"code":"insufficient_quota","message":"You exceeded your current quota"}}`)
+	}))
+	defer upstream.Close()
+
+	provider, err := ConfigFrom("openai", "test-model", upstream.URL, "test-key", "").NewProvider()
+	if err != nil {
+		t.Fatal(err)
+	}
+	var streamErr error
+	for _, err := range provider.Stream(context.Background(), llm.CompletionRequest{
+		Messages: []llm.Message{llm.UserText("ping")},
+	}) {
+		if err != nil {
+			streamErr = err
+		}
+	}
+	if streamErr == nil || !IsQuotaExhaustedMessage(streamErr.Error()) {
+		t.Fatalf("expected explicit quota error, got %v", streamErr)
+	}
+	if got := requests.Load(); got != 1 {
+		t.Fatalf("explicit quota request retried %d times, want exactly one request", got)
+	}
+}