feat(llmrec): 录制 LLM 请求/响应 HTTP 原文
归一化视图(request_body/response_body)会丢弃工具 schema、tool_use 块, 且看不到原始 SSE 帧,排查 provider 侧问题时信息不足。新增在 HTTP transport 层捕获真实 wire body 的能力: - llmrec: 新增 Capture,Recorder 挂到 ctx,quotaAwareTransport 填充。 请求取 GetBody 副本(不消费原 body),响应用 tee 保持流式;norma 内部重试 的多次尝试逐次保留(旧代码会读掉丢弃),单次时为字节级原文。 - db: llm_records 增加 raw_request/raw_response 两列,含旧库 ALTER 迁移。 - web: 录制页详情面板增加「原文」视图切换,及请求/响应内容复制按钮 (兼容非安全上下文的 execCommand 回退)。 - 测试: 单元(capture)、端到端(逐字节比对捕获=服务器实际收到)、 迁移(真实 PG 事务回滚)、全链路(Recorder→provider→PG 落库)。
This commit is contained in:
1 parent
4d9580ebfa
commit
19a7206d12
12 files changed
+884
-17
No files matched your search
+36
-1
@@ -19,6 +19,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Autumn-27/artex/llmrec"
|
||||
"github.com/Autumn-27/norma/agentcore"
|
||||
"github.com/Autumn-27/norma/compaction"
|
||||
"github.com/Autumn-27/norma/llm"
|
||||
@@ -247,10 +248,25 @@ func IsQuotaExhaustedMessage(message string) bool {
|
||||
type quotaAwareTransport struct{ base http.RoundTripper }
|
||||
|
||||
func (t quotaAwareTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
// When LLM recording is on, the Recorder puts a Capture on the context so the
|
||||
// raw wire bodies can be persisted. This is the only layer that still sees
|
||||
// them: norma builds the request body internally and decodes the SSE response
|
||||
// before either reaches the recorder.
|
||||
capt := llmrec.CaptureFrom(req.Context())
|
||||
capt.SetRequest(requestBodySnapshot(req))
|
||||
|
||||
resp, err := t.base.RoundTrip(req)
|
||||
if err != nil || resp == nil || resp.StatusCode != http.StatusTooManyRequests {
|
||||
if err != nil || resp == nil {
|
||||
return resp, err
|
||||
}
|
||||
// Tee rather than read: a 200 is an SSE stream that must keep streaming. The
|
||||
// 429 branch below reads through this wrapper, so its body lands in the
|
||||
// capture before being replaced.
|
||||
resp.Body = capt.TeeResponse(resp.StatusCode, resp.Body)
|
||||
|
||||
if resp.StatusCode != http.StatusTooManyRequests {
|
||||
return resp, nil
|
||||
}
|
||||
body, readErr := io.ReadAll(resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
resp.Body = io.NopCloser(bytes.NewReader(body))
|
||||
@@ -265,6 +281,25 @@ func (t quotaAwareTransport) RoundTrip(req *http.Request) (*http.Response, error
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// requestBodySnapshot copies an outgoing request body without consuming it.
|
||||
// norma builds every model request from a *bytes.Reader, so net/http populates
|
||||
// GetBody and the copy has no effect on what gets sent.
|
||||
func requestBodySnapshot(req *http.Request) string {
|
||||
if req.GetBody == nil {
|
||||
return ""
|
||||
}
|
||||
rc, err := req.GetBody()
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
defer rc.Close()
|
||||
b, err := io.ReadAll(rc)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func quotaAwareHTTPClient(proxy string) (*http.Client, error) {
|
||||
transport := http.DefaultTransport.(*http.Transport).Clone()
|
||||
proxy = strings.TrimSpace(proxy)
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Autumn-27/artex/llmrec"
|
||||
"github.com/Autumn-27/norma/llm"
|
||||
)
|
||||
|
||||
// End-to-end through a real norma provider: the Capture rides the context into
|
||||
// norma, survives its internal request building, and comes back holding the
|
||||
// exact bytes buildBody() put on the wire. This is the load-bearing assumption
|
||||
// of the whole feature — norma must propagate the caller's context down to
|
||||
// http.NewRequestWithContext.
|
||||
func TestCapturePropagatesThroughNormaProvider(t *testing.T) {
|
||||
sse := strings.Join([]string{
|
||||
`event: message_start`,
|
||||
`data: {"type":"message_start","message":{"id":"msg_1","usage":{"input_tokens":11,"output_tokens":1}}}`,
|
||||
``,
|
||||
`event: content_block_delta`,
|
||||
`data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"hello"}}`,
|
||||
``,
|
||||
`event: message_delta`,
|
||||
`data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":7}}`,
|
||||
``,
|
||||
`event: message_stop`,
|
||||
`data: {"type":"message_stop"}`,
|
||||
``,
|
||||
}, "\n")
|
||||
|
||||
// Record exactly what the server receives, so the capture can be compared
|
||||
// against it byte for byte rather than merely spot-checked for fields.
|
||||
var gotPath, serverSaw string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
b, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
t.Errorf("server read body: %v", err)
|
||||
}
|
||||
serverSaw = string(b)
|
||||
w.Header().Set("content-type", "text/event-stream")
|
||||
_, _ = io.WriteString(w, sse)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
cfg := Config{
|
||||
Format: llm.FormatAnthropic,
|
||||
BaseURL: srv.URL,
|
||||
APIKey: "test-key",
|
||||
Model: "claude-test",
|
||||
}
|
||||
prov, err := cfg.NewProvider()
|
||||
if err != nil {
|
||||
t.Fatalf("NewProvider: %v", err)
|
||||
}
|
||||
|
||||
ctx, capt := llmrec.NewCapture(context.Background())
|
||||
req := llm.CompletionRequest{
|
||||
System: []string{"you are a scanner"},
|
||||
Messages: []llm.Message{{Role: "user", Content: []llm.ContentBlock{{Type: "text", Text: "go"}}}},
|
||||
Tools: []llm.ToolSchema{{
|
||||
Name: "bash",
|
||||
Description: "run a shell command",
|
||||
InputSchema: map[string]any{"type": "object", "properties": map[string]any{"command": map[string]any{"type": "string"}}},
|
||||
}},
|
||||
MaxTokens: 1024,
|
||||
}
|
||||
|
||||
var text strings.Builder
|
||||
for ev, err := range prov.Stream(ctx, req) {
|
||||
if err != nil {
|
||||
t.Fatalf("stream: %v", err)
|
||||
}
|
||||
if ev.Type == llm.SETextDelta {
|
||||
text.WriteString(ev.Text)
|
||||
}
|
||||
}
|
||||
if text.String() != "hello" {
|
||||
t.Fatalf("stream text=%q, capture interfered with delivery", text.String())
|
||||
}
|
||||
if gotPath != "/v1/messages" {
|
||||
t.Fatalf("path=%q", gotPath)
|
||||
}
|
||||
|
||||
// The raw request must be what norma actually sent, not the recorder's
|
||||
// re-serialization — which is exactly why it carries fields the normalized
|
||||
// view drops.
|
||||
raw := capt.RawRequest()
|
||||
if raw == "" {
|
||||
t.Fatal("no raw request captured — context did not reach the transport")
|
||||
}
|
||||
// The load-bearing claim: what we stored equals, byte for byte, what the
|
||||
// server received — not merely "has the right fields".
|
||||
if raw != serverSaw {
|
||||
t.Fatalf("captured request != what the server received:\n got: %q\nsaw: %q", raw, serverSaw)
|
||||
}
|
||||
var body map[string]any
|
||||
if err := json.Unmarshal([]byte(raw), &body); err != nil {
|
||||
t.Fatalf("raw request is not valid JSON: %v\n%s", err, raw)
|
||||
}
|
||||
if body["model"] != "claude-test" {
|
||||
t.Errorf("model=%v want claude-test (absent from CompletionRequest)", body["model"])
|
||||
}
|
||||
if body["stream"] != true {
|
||||
t.Errorf("stream=%v want true", body["stream"])
|
||||
}
|
||||
// The full tool schema is the headline gain: the normalized view keeps names only.
|
||||
tools, _ := body["tools"].([]any)
|
||||
if len(tools) != 1 {
|
||||
t.Fatalf("tools=%v", body["tools"])
|
||||
}
|
||||
tool, _ := tools[0].(map[string]any)
|
||||
if tool["description"] != "run a shell command" {
|
||||
t.Errorf("tool description missing: %v", tool)
|
||||
}
|
||||
if tool["input_schema"] == nil {
|
||||
t.Errorf("tool input_schema missing: %v", tool)
|
||||
}
|
||||
|
||||
// And the response is the untouched SSE frames, including the events the
|
||||
// recorder never turns into stored output.
|
||||
if capt.RawResponse() != sse {
|
||||
t.Errorf("RawResponse mismatch:\n got: %q\nwant: %q", capt.RawResponse(), sse)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Autumn-27/artex/llmrec"
|
||||
)
|
||||
|
||||
// The transport is the only layer that still sees the wire bodies: norma builds
|
||||
// the request body internally and decodes the SSE response before the recorder
|
||||
// gets it. This checks the round trip preserves both directions untouched.
|
||||
func TestRoundTripCapturesRawBodies(t *testing.T) {
|
||||
const sse = "event: message_start\ndata: {\"type\":\"message_start\"}\n\nevent: message_stop\ndata: {}\n\n"
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("content-type", "text/event-stream")
|
||||
_, _ = io.WriteString(w, sse)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
client, err := quotaAwareHTTPClient("")
|
||||
if err != nil {
|
||||
t.Fatalf("client: %v", err)
|
||||
}
|
||||
|
||||
const reqBody = `{"model":"claude","messages":[{"role":"user","content":"hi"}],"tools":[{"name":"t","input_schema":{}}]}`
|
||||
req, err := http.NewRequest(http.MethodPost, srv.URL, strings.NewReader(reqBody))
|
||||
if err != nil {
|
||||
t.Fatalf("request: %v", err)
|
||||
}
|
||||
ctx, capt := llmrec.NewCapture(req.Context())
|
||||
req = req.WithContext(ctx)
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("do: %v", err)
|
||||
}
|
||||
got, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("read: %v", err)
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
|
||||
if string(got) != sse {
|
||||
t.Fatal("capture altered the response delivered to norma")
|
||||
}
|
||||
if capt.RawRequest() != reqBody {
|
||||
t.Fatalf("RawRequest()=%q want %q", capt.RawRequest(), reqBody)
|
||||
}
|
||||
if capt.RawResponse() != sse {
|
||||
t.Fatalf("RawResponse()=%q want the raw SSE frames", capt.RawResponse())
|
||||
}
|
||||
}
|
||||
|
||||
// A 429 body is read and replaced in-place by the quota check. Capturing must
|
||||
// still see it, and the replacement body must remain readable downstream.
|
||||
func TestRoundTripCaptures429BodyAlongsideQuotaRewrite(t *testing.T) {
|
||||
const body = `{"error":{"message":"insufficient_quota"}}`
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusTooManyRequests)
|
||||
_, _ = io.WriteString(w, body)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
client, err := quotaAwareHTTPClient("")
|
||||
if err != nil {
|
||||
t.Fatalf("client: %v", err)
|
||||
}
|
||||
req, err := http.NewRequest(http.MethodPost, srv.URL, strings.NewReader("{}"))
|
||||
if err != nil {
|
||||
t.Fatalf("request: %v", err)
|
||||
}
|
||||
ctx, capt := llmrec.NewCapture(req.Context())
|
||||
req = req.WithContext(ctx)
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("do: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Quota exhaustion is normalized to 402 so the router fails over.
|
||||
if resp.StatusCode != http.StatusPaymentRequired {
|
||||
t.Fatalf("status=%d want 402", resp.StatusCode)
|
||||
}
|
||||
if capt.RawResponse() != body {
|
||||
t.Fatalf("RawResponse()=%q want %q", capt.RawResponse(), body)
|
||||
}
|
||||
rest, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("read replaced body: %v", err)
|
||||
}
|
||||
if string(rest) != body {
|
||||
t.Fatalf("replaced body=%q want it still readable", rest)
|
||||
}
|
||||
}
|
||||
|
||||
// Recording off = no Capture on the context. The transport must behave exactly
|
||||
// as before, including the quota rewrite.
|
||||
func TestRoundTripWithoutCaptureIsUnchanged(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = io.WriteString(w, "ok")
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
client, err := quotaAwareHTTPClient("")
|
||||
if err != nil {
|
||||
t.Fatalf("client: %v", err)
|
||||
}
|
||||
resp, err := client.Post(srv.URL, "application/json", strings.NewReader("{}"))
|
||||
if err != nil {
|
||||
t.Fatalf("post: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
got, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("read: %v", err)
|
||||
}
|
||||
if string(got) != "ok" {
|
||||
t.Fatalf("body=%q", got)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user