diff --git a/agent/finding_workflow.go b/agent/finding_workflow.go index 3bc8e7e..7b86c7d 100644 --- a/agent/finding_workflow.go +++ b/agent/finding_workflow.go @@ -19,8 +19,6 @@ func findingTrafficBindingEnabled() bool { return FindingTrafficBindingEnabled != nil && FindingTrafficBindingEnabled() } -const findingTrafficDisabled = "Agent 自动绑定流量已关闭；请在系统设置开启后重试，或使用页面人工绑定。未登记本次携带流量的漏洞。无流量上报可省略 traffic_refs / evidence_hint_id。" - // Applied after ToolResolve: user descriptions and prompts remain intact, while // all actual reporters (including Planner and custom chat agents) see the same // API contract. Disabled/unbound tools are never reintroduced here. diff --git a/agent/tools.go b/agent/tools.go index 8ae47c7..421b8ea 100644 --- a/agent/tools.go +++ b/agent/tools.go @@ -1393,8 +1393,13 @@ func (t *ToolSet) addFinding() actool.CoreTool { if t.ts == nil { return actool.Errorf("report_finding 需要任务上下文；平台对话请通过 add_task_hint 向对应任务交接漏洞，并在提示中携带已有的 traffic_refs，由任务 Agent 登记。已登记漏洞可用 bind_finding_traffic 补绑。"), nil } - if !findingTrafficBindingEnabled() && (len(a.TrafficRefs) > 0 || pid(a.EvidenceHintID) > 0) { - return actool.Errorf(findingTrafficDisabled), nil + // Auto-binding off: ignore the evidence params instead of rejecting the call. + // stripTrafficParameters already removes them from the advertised schema, but + // models routinely emit fields anyway — failing here would discard a confirmed + // finding over a stray parameter. The success path below reports evidence_status + // "not_bound" with the "已关闭，可在页面人工关联" note, which is what the caller needs. + if !findingTrafficBindingEnabled() { + a.TrafficRefs, a.EvidenceHintID = nil, nil } if len(a.EvidenceHintID) > 0 && pid(a.EvidenceHintID) <= 0 { return actool.Errorf("evidence_hint_id 必须为有效的提示节点 ID；无交接提示时省略"), nil diff --git a/db/finding_retests.go b/db/finding_retests.go index 3d9d312..05d562d 100644 --- a/db/finding_retests.go +++ b/db/finding_retests.go @@ -157,6 +157,18 @@ func (d *DB) FindingRetestForConversation(ctx context.Context, conversationID in return r, err } +// FailPendingRetestForConversation seals a conversation's unfinished retest when +// the runner could not even load it — the retest ID is unknown on that path, so +// the conversation ID is the only handle. Without it a transient read error +// leaves the row 'pending' forever: the findings list keeps showing 复测中 and +// every later 发起复测 is deduped against a run that is not happening, with only +// a process restart (RecoverFindingRetests) able to clear it. +func (d *DB) FailPendingRetestForConversation(conversationID int64, reason string) error { + _, err := d.Exec(`UPDATE finding_retests SET status='failed', error=$2, finished_at=now() + WHERE conversation_id=$1 AND status IN ('pending','running')`, conversationID, reason) + return err +} + func (d *DB) StartFindingRetest(ctx context.Context, id int64) (bool, error) { res, err := d.ExecContext(ctx, `UPDATE finding_retests SET status='running', started_at=now() WHERE id=$1 AND status='pending'`, id) if err != nil { diff --git a/db/finding_traffic.go b/db/finding_traffic.go index f898303..fc74462 100644 --- a/db/finding_traffic.go +++ b/db/finding_traffic.go @@ -109,7 +109,24 @@ type PreparedTrafficEvidence struct { Snapshot TrafficEvidenceSnapshot } +// Normalize makes the snapshot's text columns safe for PostgreSQL. URL and the +// head blocks come straight off the wire, so a target answering with a non-UTF-8 +// header (a GBK `Content-Disposition: filename=…`, a NUL byte) would otherwise +// abort the INSERT and roll back the whole finding — losing a confirmed finding +// over a malformed response header. Applied before hashing so the ID always +// matches the bytes that actually land in the table. +func (s TrafficEvidenceSnapshot) Normalize() TrafficEvidenceSnapshot { + s.SourceTrafficID = utf8Clean(s.SourceTrafficID) + s.URL = utf8Clean(s.URL) + s.Method = utf8Clean(s.Method) + s.ContentType = utf8Clean(s.ContentType) + s.ReqHead = utf8Clean(s.ReqHead) + s.RespHead = utf8Clean(s.RespHead) + return s +} + func TrafficSnapshotID(snapshot TrafficEvidenceSnapshot) string { + snapshot = snapshot.Normalize() snapshot.ID = "" snapshot.CreatedAt = time.Time{} raw, _ := json.Marshal(snapshot) @@ -169,6 +186,10 @@ func InsertEvidenceSnapshotTx(tx *sql.Tx, s TrafficEvidenceSnapshot) error { if s.ID != TrafficSnapshotID(s) { return errors.New("证据快照元数据哈希不匹配") } + // The ID was computed over the normalized form; store those same bytes. + id := s.ID + s = s.Normalize() + s.ID = id if s.CreatedAt.IsZero() { s.CreatedAt = time.Now().UTC() } diff --git a/evidence/store.go b/evidence/store.go index dd9aa06..b785176 100644 --- a/evidence/store.go +++ b/evidence/store.go @@ -169,6 +169,9 @@ func (s *Store) prepare(ctx context.Context, refs []db.TrafficRef) ([]db.Prepare } v := db.TrafficEvidenceSnapshot{SourceTrafficID: e.ID, CapturedAt: e.TS, URL: e.URL, Method: e.Method, Status: e.Status, ContentType: e.ContentType, ReqHead: e.ReqHead, RespHead: e.RespHead, ReqHash: rh, RespHash: ph, ReqLen: e.ReqLen, RespLen: e.RespLen} + // Raw wire bytes: normalize once here so the ID, the stored row and every + // downstream consumer (archive, API responses) all see the same text. + v = v.Normalize() v.ID = db.TrafficSnapshotID(v) out = append(out, db.PreparedTrafficEvidence{Ref: byID[e.ID], Snapshot: v}) return nil @@ -224,8 +227,23 @@ func (s *Store) WithBinding(ctx context.Context, findingID, bindingID int64, fn }) } -// OpenBody must be called while holding WithEvidenceTx/WithBinding, and the -// returned file closed before that callback returns. +// Binding resolves one binding's metadata under the evidence lock and releases +// the lock before returning. Callers that then stream a body to a client must +// use this instead of WithBinding: verifyFile+io.Copy is O(body size), so a +// large download (or a slow client) holding WithEvidenceTx would block every +// evidence write process-wide. Reading the blob afterwards is safe — blobs are +// content-addressed and GC only reaps unreferenced files after a 24h grace +// period, and an already-open fd survives an unlink regardless. +func (s *Store) Binding(ctx context.Context, findingID, bindingID int64) (db.FindingTrafficBinding, error) { + var out db.FindingTrafficBinding + err := s.WithBinding(ctx, findingID, bindingID, func(b db.FindingTrafficBinding) error { + out = b + return nil + }) + return out, err +} + +// OpenBody may be called without holding the evidence lock; see Binding. func (s *Store) OpenBody(snapshot db.TrafficEvidenceSnapshot, side string) (*os.File, int64, error) { hash, length := snapshot.ReqHash, snapshot.ReqLen if side == "response" { diff --git a/intercept/trace.go b/intercept/trace.go index ea5caae..b469228 100644 --- a/intercept/trace.go +++ b/intercept/trace.go @@ -166,6 +166,15 @@ func auditFor(ctx context.Context, dec Decision, input []byte, status string) *d if a.Correlation != "exact" { a.ExecutionStatus = "unknown" } + // A straight rule/model allow is a routine event, not an approval a human + // reviewed: with the fallback judge on, a pentest task emits thousands of + // them. Keeping the full snapshot would write 24×8KiB of context plus a + // 32KiB prompt per row into intercept_pending — hundreds of MB, which the + // task archive (SELECT *) then carries along. Decision metadata and the + // execution result still land; only the bulky replay snapshot is dropped. + // The ask path builds its audit in HandleAsk and is unaffected. + a.UserMessage, a.UserTruncated = "", false + a.Context, a.ContextTruncated = nil, false } if status == "denied" { a.EffectiveAction, a.ExecutionStatus = "deny", "not_executed" diff --git a/intercept/trace_test.go b/intercept/trace_test.go index b736164..d52b198 100644 --- a/intercept/trace_test.go +++ b/intercept/trace_test.go @@ -15,7 +15,9 @@ func TestTraceExactCorrelationAndSnapshot(t *testing.T) { trace.Start("call-a", "Write", []byte(`{"path":"a","n":12345678901234567890}`)) trace.Append(db.InterceptContextEntry{Kind: "text", Text: "later context"}) callCtx := WithCall(ctx, "Write", []byte(`{ "n":12345678901234567890, "path":"a" }`)) - a := auditFor(callCtx, Decision{Action: "allow"}, nil, "allowed") + // "pending" (the ask path) keeps the full snapshot; see TestAuditAllowDropsSnapshot + // for why a routine allow does not. + a := auditFor(callCtx, Decision{Action: "ask"}, nil, "pending") if a.Correlation != "exact" || a.ToolUseID != "call-a" || len(a.Context) != 1 || a.Context[0].Text != "prior message" { t.Fatalf("wrong snapshot: %+v", a) } @@ -32,6 +34,33 @@ func TestTraceExactCorrelationAndSnapshot(t *testing.T) { } } +// A rule/model allow is logged for auditability but must not carry the replay +// snapshot: with the fallback judge on those rows are emitted per tool call, and +// keeping 24×8KiB of context plus a 32KiB prompt each would put hundreds of MB +// into intercept_pending (and from there into the task archive). Decision +// metadata and correlation must survive so the execution result still binds. +func TestAuditAllowDropsSnapshot(t *testing.T) { + ctx, trace := WithTrace(context.Background(), "a long user prompt", []db.InterceptContextEntry{{Kind: "user", Text: "prior message"}}) + input := []byte(`{"path":"a"}`) + trace.Start("call-a", "Write", input) + a := auditFor(WithCall(ctx, "Write", input), Decision{Action: "allow", RuleName: "auto"}, input, "allowed") + if a.UserMessage != "" || a.UserTruncated || a.Context != nil || a.ContextTruncated { + t.Fatalf("allow kept the replay snapshot: %+v", a) + } + if a.Correlation != "exact" || a.ToolUseID != "call-a" || a.EffectiveAction != "allow" || + a.ExecutionStatus != "awaiting_result" || a.RuleName != "auto" || a.InputDigest == "" { + t.Fatalf("allow lost decision metadata: %+v", a) + } + + // A denial is rare and worth the full context, so it keeps its snapshot. + denied := []byte(`{"path":"b"}`) + trace.Start("call-b", "Write", denied) + d := auditFor(WithCall(ctx, "Write", denied), Decision{Action: "deny"}, denied, "denied") + if len(d.Context) == 0 || d.Context[0].Text != "prior message" || d.UserMessage == "" { + t.Fatalf("deny lost the snapshot: %+v", d) + } +} + func TestTraceIdenticalParallelCallsAreNeverGuessed(t *testing.T) { ctx, trace := WithTrace(context.Background(), "test", nil) input := []byte(`{"command":"pwd"}`) diff --git a/server/conversations.go b/server/conversations.go index 04b48de..dbc78be 100644 --- a/server/conversations.go +++ b/server/conversations.go @@ -496,7 +496,12 @@ func (s *Server) runConversationTurn(ctx context.Context, cancel context.CancelC // Read without the run cancellation so an immediate stop still seals pending. r, err := s.m.pg.FindingRetestForConversation(context.Background(), c.ID) if err != nil { + // The sealing defer below needs r.ID, which we do not have here. Seal by + // conversation instead, otherwise the row stays 'pending' forever. log.Printf("[conv %d] load retest: %v", c.ID, err) + if err := s.m.pg.FailPendingRetestForConversation(c.ID, "复测状态读取失败，请重新发起"); err != nil { + log.Printf("[conv %d] seal retest: %v", c.ID, err) + } return } if r != nil && r.Status == "pending" { diff --git a/server/finding_traffic.go b/server/finding_traffic.go index 92eea8d..103159e 100644 --- a/server/finding_traffic.go +++ b/server/finding_traffic.go @@ -324,43 +324,45 @@ func (s *Server) getFindingTrafficBody(w http.ResponseWriter, r *http.Request) { } side := r.URL.Query().Get("side") store := s.evidenceStore() - started := false - err = store.WithBinding(r.Context(), id, bid, func(b db.FindingTrafficBinding) error { - if r.URL.Query().Get("download") == "1" { - f, length, err := store.OpenBody(b.Snapshot, side) - if err != nil { - return err - } - defer f.Close() - w.Header().Set("Content-Type", "application/octet-stream") - w.Header().Set("X-Content-Type-Options", "nosniff") - w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"evidence-%d-%s.bin\"", bid, side)) - w.Header().Set("Content-Length", strconv.FormatInt(length, 10)) - started = true - _, err = io.Copy(w, f) - return err - } - offset, length := int64(0), int64(8192) - for name, dst := range map[string]*int64{"offset": &offset, "length": &length} { - if raw := r.URL.Query().Get(name); raw != "" { - v, err := strconv.ParseInt(raw, 10, 64) - if err != nil { - return err - } - *dst = v - } - } - preview, err := readEvidencePreview(store, b.Snapshot, side, offset, length) - if err != nil { - return err - } - started = true - writeJSON(w, 200, preview) - return nil - }) - if err != nil && !started { + // Resolve the binding under the evidence lock, then read the blob without it: + // both paths below are O(body size) and would otherwise stall every evidence + // write for as long as the client takes to receive the data. + b, err := store.Binding(r.Context(), id, bid) + if err != nil { evidenceError(w, err) + return } + if r.URL.Query().Get("download") == "1" { + f, length, err := store.OpenBody(b.Snapshot, side) + if err != nil { + evidenceError(w, err) + return + } + defer f.Close() + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("X-Content-Type-Options", "nosniff") + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"evidence-%d-%s.bin\"", bid, side)) + w.Header().Set("Content-Length", strconv.FormatInt(length, 10)) + _, _ = io.Copy(w, f) + return + } + offset, length := int64(0), int64(8192) + for name, dst := range map[string]*int64{"offset": &offset, "length": &length} { + if raw := r.URL.Query().Get(name); raw != "" { + v, err := strconv.ParseInt(raw, 10, 64) + if err != nil { + evidenceError(w, err) + return + } + *dst = v + } + } + preview, err := readEvidencePreview(store, b.Snapshot, side, offset, length) + if err != nil { + evidenceError(w, err) + return + } + writeJSON(w, 200, preview) } func (s *Server) toolGetFindingTraffic() actool.CoreTool { diff --git a/server/orchestration.go b/server/orchestration.go index a470d48..ee5aa95 100644 --- a/server/orchestration.go +++ b/server/orchestration.go @@ -463,12 +463,13 @@ func (s *Server) seedOrchestrationTools() { s.seedWorkerReadToolsUnbind() // list_facts/node_detail/list_companies/跨 work 检索从 worker 默认解绑(一次性) s.seedAutoReportFindingBinding() s.unbindGoalMetDefault() - s.reseedGoalsPrompt() // goals 提示词加入「抽操作约束」步 → 旧库追加一版新默认(一次性) - s.reseedMainAgentPrompt() // mainagent 提示词加入「目标达成后 add_intent 反问是否建目标」(一次性) - s.reseedPlannerPrompt() // planner 提示词:重写「0 意图」正当理由 + 加量化验收核对(一次性) - s.reseedWorkerPrompt() // worker 提示词:加否定结论证据门槛(一次性) - s.seedReporterAgent() // 预置「报告撰写」agent + 工具绑定 + finding 触发器(一次性) - s.seedFindingTrafficTools() // 增加可选证据参数及只读证据工具，保留用户配置 + s.reseedGoalsPrompt() // goals 提示词加入「抽操作约束」步 → 旧库追加一版新默认(一次性) + s.reseedMainAgentPrompt() // mainagent 提示词加入「目标达成后 add_intent 反问是否建目标」(一次性) + s.reseedPlannerPrompt() // planner 提示词:重写「0 意图」正当理由 + 加量化验收核对(一次性) + s.reseedWorkerPrompt() // worker 提示词:加否定结论证据门槛(一次性) + s.seedReporterAgent() // 预置「报告撰写」agent + 工具绑定 + finding 触发器(一次性) + s.upgradeReporterTriggerMessage() // 老库补迁移:让 reporter 回传 evidence_version(一次性) + s.seedFindingTrafficTools() // 增加可选证据参数及只读证据工具，保留用户配置 s.seedFindingWorkflowTools() // 注：pentest 的默认工具绑定无需迁移——BuiltinToolSeeds 在全新初始化时就把 // list_assets/insert_assets/report_finding/list_findings/list_companies 连同 @@ -651,6 +652,50 @@ func (s *Server) reseedWorkerPrompt() { log.Printf("[prompts] worker 提示词已追加新默认版本(查上下文段收敛为 list_assets/list_findings,去掉 list_facts/node_detail/asset_neighbors,一次性)") } +// reporterToolCallMessage 必须无条件要求先读一次 get_finding_traffic 再写报告。 +// 该工具是只读的、「不依赖捕获开关」,自动绑定关不关都能读到人工绑定的证据。若这里 +// 写成「启用自动绑定才读」,默认关闭配置下 reporter 就不会传 evidence_version, +// SetFindingReportVersionByNodeID 便按 legacy 语义写 -1,漏洞详情与 Markdown 导出 +// 从此常驻「证据已变更，报告待更新」,而 UI 上没有任何入口能把它清掉。 +const reporterToolCallMessage = "上面刚有一个漏洞被 report_finding 登记。请读取返回 JSON 的 finding_id（独立漏洞记录 ID）与 finding_node_id（探索节点 ID），" + + "先用 get_finding_traffic(finding_id) 读取当前证据清单及其 version（空清单是正常情况，照常写报告）；" + + "若运行指引启用自动绑定，在读取前先核实并关联本次漏洞的流量。节点详情使用 finding_node_id。" + + "最后调用 update_finding_report(finding_id=finding_node_id, report, evidence_version=实际读取版本) 保存，" + + "evidence_version 必须传，否则报告会被永久标记为待更新。不要混用两种编号。" + +// 旧版触发消息(0.3.8 及更早)。只有仍与它逐字相同的记录才会被迁移覆盖，用户改过的保持原样。 +const reporterToolCallMessageV1 = "上面刚有一个漏洞被 report_finding 登记。请从触发上下文里取出 finding_id" + + "（工具返回 \"finding recorded: \" 里的数字）与任务 id，按你的职责撰写该漏洞的详细报告，" + + "最后调用 update_finding_report(finding_id, report) 保存。" + +// upgradeReporterTriggerMessage 把老库里仍是默认文案的 reporter 触发消息刷成新版本。 +// seedReporterAgent 受 reporter_agent_seed_v1 守卫且只在新建 agent 时写触发器，所以 +// 升级上来的库拿不到新文案 —— 工具 schema 由 seedFindingTrafficTools 补齐了 +// evidence_version，但没有任何东西告诉 reporter 去用它。一次性，且只覆盖未被改动的文案。 +func (s *Server) upgradeReporterTriggerMessage() { + const flag = "reporter_trigger_evidence_version_v1" + if v, _, _ := s.m.pg.GetSetting(flag); v == "true" { + return + } + defer func() { _ = s.m.pg.SetSetting(flag, "true") }() // 只尝试一次 + triggers, err := s.m.pg.ListTriggersFor("reporter") + if err != nil { + log.Printf("[reporter] 读取触发器失败: %v", err) + return + } + for _, t := range triggers { + if !t.OnToolCall || t.ToolCallMessage != reporterToolCallMessageV1 { + continue // 用户改过或不是 finding 触发器，不动。 + } + t.ToolCallMessage = reporterToolCallMessage + if err := s.m.pg.UpdateTrigger(t); err != nil { + log.Printf("[reporter] 升级触发消息失败: %v", err) + return + } + log.Printf("[reporter] 触发消息已升级为读取并回传 evidence_version") + } +} + // seedReporterAgent 预置一个「报告撰写」自定义 agent(builtin=false，可在 UI 编辑/删除)： // 绑定 update_finding_report + 任务查询工具，并挂一个「report_finding 被调用即触发」的 // 触发器 —— 每登记一个漏洞就唤起它写详细报告。一次性(settings flag 守卫)：用户删掉后不再重建。 @@ -691,13 +736,11 @@ func (s *Server) seedReporterAgent() { // 触发器：report_finding 被调用即触发（工具返回 "finding recorded: " 带上 finding_id， // 任务 id 也在触发消息里）。 if _, err := s.m.pg.CreateTrigger(&db.AgentTrigger{ - AgentKey: "reporter", - Enabled: true, - OnToolCall: true, - ToolNames: []string{"report_finding"}, - ToolCallMessage: "上面刚有一个漏洞被 report_finding 登记。请读取返回 JSON 的 finding_id（独立漏洞记录 ID）与 finding_node_id（探索节点 ID），" + - "若运行指引启用自动绑定，先核实并关联本次漏洞的流量，再用 get_finding_traffic(finding_id) 读取最新证据；节点详情使用 finding_node_id。" + - "最后调用 update_finding_report(finding_id=finding_node_id, report, evidence_version=实际读取版本) 保存。不要混用两种编号。", + AgentKey: "reporter", + Enabled: true, + OnToolCall: true, + ToolNames: []string{"report_finding"}, + ToolCallMessage: reporterToolCallMessage, }); err != nil { log.Printf("[reporter] 创建触发器失败: %v", err) }