Merge origin/main into codex/btw-side-question

合入 main 的 norma v0.3.7、会话分组与流量证据更新,保留 /btw 快照和输入布局。同步自动放行、关闭流量绑定的测试断言,并等待归档测试夹具的旁路后台退出。
This commit is contained in:
ji ruo committed 2026-09-11 00:54:51 +08:00
commit 2930c4b915
94 files changed
+9737 -1241

No files matched your search

+3
View File
@@ -31,6 +31,9 @@ func utf8Clean(s string) string {
// Node is a typed reasoning node (= old task_nodes). kind ∈ goal|intent|finding|hint.
type Node struct {
FindingID int64 `json:"finding_id,omitempty"` // populated by finding-aware reads; never inferred from ID
FindingNodeID int64 `json:"finding_node_id,omitempty"`
TrafficCount int `json:"traffic_count,omitempty"`
ID int64 `json:"id"`
Kind string `json:"kind"`
Payload json.RawMessage `json:"payload"`
+250
View File
@@ -0,0 +1,250 @@
package db
import (
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
)
const FindingRetestAgentKey = "retester"
var ErrRetestNotRunning = errors.New("本次复测已结束或尚未开始,请从漏洞详情发起新的复测")
// FindingRetest is an immutable historical test once its conversation turn ends.
// Snapshot is only loaded for the agent, never sent with the history list.
type FindingRetest struct {
ID int64 `json:"id"`
FindingID int64 `json:"finding_id"`
ConversationID *int64 `json:"conversation_id"`
Status string `json:"status"`
Verdict string `json:"verdict"`
Notes string `json:"notes"`
Snapshot json.RawMessage `json:"snapshot,omitempty"`
Summary string `json:"summary"`
Evidence string `json:"evidence"`
Error string `json:"error"`
CreatedAt time.Time `json:"created_at"`
StartedAt *time.Time `json:"started_at"`
FinishedAt *time.Time `json:"finished_at"`
}
const retestCols = `id, finding_id, conversation_id, status, verdict, notes, summary, evidence, error, created_at, started_at, finished_at`
// ActiveFindingRetest is the small status payload polled by the findings list.
// Finding IDs use the same string representation as the findings API.
type ActiveFindingRetest struct {
ID int64 `json:"id"`
FindingID int64 `json:"finding_id,string"`
ConversationID int64 `json:"conversation_id"`
Status string `json:"status"`
}
func (d *DB) ListActiveFindingRetests(ctx context.Context) ([]ActiveFindingRetest, error) {
rows, err := d.QueryContext(ctx, `SELECT id, finding_id, conversation_id, status FROM finding_retests
WHERE status IN ('pending','running') AND conversation_id IS NOT NULL ORDER BY id`)
if err != nil {
return nil, err
}
defer rows.Close()
items := []ActiveFindingRetest{}
for rows.Next() {
var item ActiveFindingRetest
if err := rows.Scan(&item.ID, &item.FindingID, &item.ConversationID, &item.Status); err != nil {
return nil, err
}
items = append(items, item)
}
return items, rows.Err()
}
func scanRetest(row interface{ Scan(...any) error }) (*FindingRetest, error) {
r := &FindingRetest{}
err := row.Scan(&r.ID, &r.FindingID, &r.ConversationID, &r.Status, &r.Verdict, &r.Notes,
&r.Summary, &r.Evidence, &r.Error, &r.CreatedAt, &r.StartedAt, &r.FinishedAt)
if errors.Is(err, sql.ErrNoRows) {
return nil, nil
}
return r, err
}
// CreateFindingRetest atomically snapshots the source, creates its conversation
// and persists the first message. A finding row lock deduplicates simultaneous
// clicks across clients; an existing active run is returned without dispatching.
func (d *DB) CreateFindingRetest(ctx context.Context, findingID int64, notes string) (*FindingRetest, *Conversation, bool, error) {
tx, err := d.BeginTx(ctx, nil)
if err != nil {
return nil, nil, false, err
}
defer tx.Rollback()
var title string
var snapshot []byte
err = tx.QueryRowContext(ctx, `SELECT COALESCE(NULLIF(f.name,''), NULLIF(f.vulnclass,''), '未分类'),
jsonb_build_object('finding', to_jsonb(f),
'assets', COALESCE((SELECT jsonb_agg(to_jsonb(a)) FROM assets a WHERE f.asset_ids @> to_jsonb(ARRAY[a.id])), '[]'::jsonb),
'constraints', COALESCE((SELECT jsonb_agg(to_jsonb(c)) FROM task_constraints c JOIN tasks t ON t.exploration_id=c.exploration_id WHERE t.id=f.task_id), '[]'::jsonb))
FROM findings f WHERE f.id=$1 FOR UPDATE OF f`, findingID).Scan(&title, &snapshot)
if err != nil {
return nil, nil, false, err
}
r, err := scanRetest(tx.QueryRowContext(ctx, `SELECT `+retestCols+` FROM finding_retests WHERE finding_id=$1 AND status IN ('pending','running')`, findingID))
if err != nil {
return nil, nil, false, err
}
if r != nil {
return r, nil, false, nil
}
// Keep the title within the same limit as ordinary conversations.
if runes := []rune(title); len(runes) > 100 {
title = string(runes[:100])
}
c, err := scanConv(tx.QueryRowContext(ctx, `INSERT INTO conversations(agent_key,title) VALUES ($1,$2) RETURNING `+convCols,
FindingRetestAgentKey, fmt.Sprintf("复测 #%d · %s", findingID, title)))
if err != nil {
return nil, nil, false, err
}
r, err = scanRetest(tx.QueryRowContext(ctx, `INSERT INTO finding_retests(finding_id,conversation_id,notes,snapshot) VALUES ($1,$2,$3,$4) RETURNING `+retestCols,
findingID, c.ID, strings.TrimSpace(notes), snapshot))
if err != nil {
return nil, nil, false, err
}
msg := r.InitialMessage()
_, err = tx.ExecContext(ctx, `INSERT INTO conversation_activities(conversation_id,worker,kind,summary,detail) VALUES ($1,$2,'user',$3,$4)`,
c.ID, FindingRetestAgentKey, fmt.Sprintf("请复测漏洞 #%d", findingID), msg)
if err != nil {
return nil, nil, false, err
}
if err = tx.Commit(); err != nil {
return nil, nil, false, err
}
return r, &c, true, nil
}
func (r *FindingRetest) InitialMessage() string {
msg := fmt.Sprintf("请复测漏洞 #%d。先调用 get_finding_retest_context 读取本会话关联的原始证据与约束,再执行针对性验证,最后调用 record_finding_retest_result 保存结论。", r.FindingID)
if r.Notes != "" {
msg += "\n\n本次复测补充说明:\n" + r.Notes
}
return msg
}
func (d *DB) ListFindingRetests(findingID int64) ([]*FindingRetest, error) {
rows, err := d.Query(`SELECT `+retestCols+` FROM finding_retests WHERE finding_id=$1 ORDER BY id DESC`, findingID)
if err != nil {
return nil, err
}
defer rows.Close()
out := []*FindingRetest{}
for rows.Next() {
r, err := scanRetest(rows)
if err != nil {
return nil, err
}
out = append(out, r)
}
return out, rows.Err()
}
func (d *DB) FindingRetestForConversation(ctx context.Context, conversationID int64) (*FindingRetest, error) {
r, err := scanRetest(d.QueryRowContext(ctx, `SELECT `+retestCols+` FROM finding_retests WHERE conversation_id=$1`, conversationID))
if err != nil || r == nil {
return r, err
}
err = d.QueryRowContext(ctx, `SELECT snapshot FROM finding_retests WHERE id=$1`, r.ID).Scan(&r.Snapshot)
return r, err
}
// FailPendingRetestForConversation seals a conversation's unfinished retest when
// the runner could not even load it — the retest ID is unknown on that path, so
// the conversation ID is the only handle. Without it a transient read error
// leaves the row 'pending' forever: the findings list keeps showing 复测中 and
// every later 发起复测 is deduped against a run that is not happening, with only
// a process restart (RecoverFindingRetests) able to clear it.
func (d *DB) FailPendingRetestForConversation(conversationID int64, reason string) error {
_, err := d.Exec(`UPDATE finding_retests SET status='failed', error=$2, finished_at=now()
WHERE conversation_id=$1 AND status IN ('pending','running')`, conversationID, reason)
return err
}
func (d *DB) StartFindingRetest(ctx context.Context, id int64) (bool, error) {
res, err := d.ExecContext(ctx, `UPDATE finding_retests SET status='running', started_at=now() WHERE id=$1 AND status='pending'`, id)
if err != nil {
return false, err
}
n, err := res.RowsAffected()
return n == 1, err
}
// RecordFindingRetestResult never accepts a finding ID: ownership comes from the
// runtime conversation. Identical retries are safe; a second verdict is refused.
func (d *DB) RecordFindingRetestResult(ctx context.Context, conversationID int64, verdict, summary, evidence string) error {
if verdict != "reproduced" && verdict != "fixed" && verdict != "inconclusive" {
return errors.New("verdict 必须为 reproduced / fixed / inconclusive")
}
summary, evidence = strings.TrimSpace(summary), strings.TrimSpace(evidence)
if summary == "" || evidence == "" {
return errors.New("summary 与 evidence 不能为空;无法确认时说明实际检查及阻塞原因")
}
if len(summary) > 16000 || len(evidence) > 128000 {
return errors.New("复测结论过长(summary ≤ 16KB,evidence ≤ 128KB)")
}
res, err := d.ExecContext(ctx, `UPDATE finding_retests SET verdict=$2,summary=$3,evidence=$4
WHERE conversation_id=$1 AND status='running' AND (verdict='' OR (verdict=$2 AND summary=$3 AND evidence=$4))`, conversationID, verdict, summary, evidence)
if err != nil {
return err
}
if n, err := res.RowsAffected(); err != nil {
return err
} else if n == 0 {
return ErrRetestNotRunning
}
return nil
}
// FinishFindingRetest seals the result. Cancellation/failure takes precedence
// over a staged verdict so an interrupted test cannot appear successfully fixed.
// Only a newly completed fixed verdict updates triage, in the same transaction.
func (d *DB) FinishFindingRetest(id int64, status, reason string) error {
if status != "completed" && status != "failed" && status != "stopped" {
return errors.New("invalid terminal retest status")
}
tx, err := d.Begin()
if err != nil {
return err
}
defer tx.Rollback()
// Lock the finding before the retest, matching creation and cascading deletion.
var findingID int64
err = tx.QueryRow(`SELECT f.id FROM findings f WHERE f.id=(SELECT finding_id FROM finding_retests WHERE id=$1) FOR UPDATE OF f`, id).Scan(&findingID)
if errors.Is(err, sql.ErrNoRows) {
return nil // Finding/retest already deleted.
}
if err != nil {
return err
}
var finalStatus, verdict string
err = tx.QueryRow(`UPDATE finding_retests SET
status=CASE WHEN $2='completed' AND verdict='' THEN 'failed' ELSE $2 END,
error=CASE WHEN $2='completed' AND verdict='' THEN 'Agent 未保存复测结论,请查看会话后重新复测' ELSE $3 END,
finished_at=now() WHERE id=$1 AND status IN ('pending','running') RETURNING status,verdict`, id, status, reason).Scan(&finalStatus, &verdict)
if errors.Is(err, sql.ErrNoRows) {
return nil // A replay must not overwrite a later manual triage decision.
}
if err != nil {
return err
}
if finalStatus == "completed" && verdict == "fixed" {
if _, err := tx.Exec(`UPDATE findings SET status=$1 WHERE id=$2`, FindingFixed, findingID); err != nil {
return err
}
}
return tx.Commit()
}
func (d *DB) RecoverFindingRetests() error {
_, err := d.Exec(`UPDATE finding_retests SET status='stopped', error='服务重启,复测已中断,请重新发起', finished_at=now() WHERE status IN ('pending','running')`)
return err
}
+242
View File
@@ -0,0 +1,242 @@
package db
import (
"context"
"encoding/json"
"errors"
"strings"
"sync"
"testing"
)
func retestDB(t *testing.T) (*DB, int64) {
t.Helper()
d, err := Open(testDSN(t))
if err != nil {
t.Fatalf("open test database: %v", err)
}
fid, err := d.AddFinding(0, 0, "retest-test", "测试漏洞", "high", "original summary", "original evidence", "worker", nil)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
_, _ = d.Exec(`DELETE FROM conversations WHERE id IN (SELECT conversation_id FROM finding_retests WHERE finding_id=$1)`, fid)
_, _ = d.DeleteFinding(fid)
d.Close()
})
return d, fid
}
func TestRetestAtomicDeduplicationAndHistory(t *testing.T) {
d, fid := retestDB(t)
var wg sync.WaitGroup
var mu sync.Mutex
createdCount := 0
ids := make([]int64, 0, 8)
for range 8 {
wg.Go(func() {
r, c, created, err := d.CreateFindingRetest(t.Context(), fid, " 修复版本 v2 ")
if err != nil {
t.Error(err)
return
}
mu.Lock()
defer mu.Unlock()
ids = append(ids, r.ID)
if created {
createdCount++
if c == nil {
t.Error("created without conversation")
}
}
})
}
wg.Wait()
if createdCount != 1 || len(ids) != 8 {
t.Fatalf("created=%d ids=%v", createdCount, ids)
}
for _, id := range ids {
if id != ids[0] {
t.Fatal("duplicate active retests", ids)
}
}
rows, err := d.ListFindingRetests(fid)
if err != nil || len(rows) != 1 {
t.Fatalf("history=%v err=%v", rows, err)
}
r := rows[0]
if r.Snapshot != nil {
t.Fatal("history leaks large snapshot")
}
ctx := t.Context()
full, err := d.FindingRetestForConversation(ctx, *r.ConversationID)
if err != nil || !strings.Contains(string(full.Snapshot), "original evidence") {
t.Fatalf("snapshot=%+v err=%v", full, err)
}
var messages int
if err := d.QueryRow(`SELECT count(*) FROM conversation_activities WHERE conversation_id=$1 AND kind='user'`, *r.ConversationID).Scan(&messages); err != nil || messages != 1 {
t.Fatalf("messages=%d err=%v", messages, err)
}
_, _ = d.Exec(`UPDATE findings SET evidence='changed evidence' WHERE id=$1`, fid)
full, err = d.FindingRetestForConversation(ctx, *r.ConversationID)
if err != nil || strings.Contains(string(full.Snapshot), "changed evidence") {
t.Fatal("snapshot changed", err)
}
if err = d.RecordFindingRetestResult(ctx, *r.ConversationID, "fixed", "summary", "proof"); !errors.Is(err, ErrRetestNotRunning) {
t.Fatal("pending accepted result", err)
}
if ok, err := d.StartFindingRetest(ctx, r.ID); err != nil || !ok {
t.Fatalf("start=%t %v", ok, err)
}
if err := d.RecordFindingRetestResult(ctx, 0, "fixed", "summary", "proof"); !errors.Is(err, ErrRetestNotRunning) {
t.Fatal("unscoped write accepted", err)
}
for range 2 {
if err := d.RecordFindingRetestResult(ctx, *r.ConversationID, "fixed", "修复验证通过", "正常对照可用,原触发条件失效"); err != nil {
t.Fatal(err)
}
}
if err := d.RecordFindingRetestResult(ctx, *r.ConversationID, "reproduced", "different", "proof"); !errors.Is(err, ErrRetestNotRunning) {
t.Fatal("overwrote staged result", err)
}
if err := d.FinishFindingRetest(r.ID, "completed", ""); err != nil {
t.Fatal(err)
}
if err := d.RecordFindingRetestResult(ctx, *r.ConversationID, "fixed", "summary", "proof"); !errors.Is(err, ErrRetestNotRunning) {
t.Fatal("overwrote sealed result", err)
}
f, _ := d.GetFinding(fid)
if f.Status != FindingFixed || f.Evidence != "changed evidence" || f.Report != "" {
t.Fatal("fixed retest did not update only triage", f)
}
next, _, created, err := d.CreateFindingRetest(ctx, fid, "second")
if err != nil || !created || next.ID == r.ID {
t.Fatalf("new history=%+v created=%t err=%v", next, created, err)
}
if err := d.FinishFindingRetest(next.ID, "completed", ""); err != nil {
t.Fatal(err)
}
rows, _ = d.ListFindingRetests(fid)
if len(rows) != 2 || rows[0].Status != "failed" || rows[0].Error == "" || rows[1].Verdict != "fixed" {
t.Fatal("missing verdict treated as successful", rows)
}
}
func TestRetestDeletionAndRestart(t *testing.T) {
d, fid := retestDB(t)
r, c, _, err := d.CreateFindingRetest(t.Context(), fid, "delete")
if err != nil {
t.Fatal(err)
}
if err := d.DeleteConversation(c.ID); err != nil {
t.Fatal(err)
}
rows, _ := d.ListFindingRetests(fid)
if len(rows) != 1 || rows[0].ConversationID != nil || rows[0].Status != "stopped" {
t.Fatal(rows)
}
next, c2, created, err := d.CreateFindingRetest(t.Context(), fid, "restart")
if err != nil || !created {
t.Fatal("deleted session blocks retry", err)
}
if ok, err := d.StartFindingRetest(t.Context(), next.ID); err != nil || !ok {
t.Fatal(err)
}
if err := d.RecoverFindingRetests(); err != nil {
t.Fatal(err)
}
if err := d.FinishFindingRetest(next.ID, "completed", ""); err != nil {
t.Fatal(err)
}
full, _ := d.FindingRetestForConversation(t.Context(), c2.ID)
if full.Status != "stopped" || full.FinishedAt == nil {
t.Fatal("restart result overwritten", full)
}
if _, err := d.DeleteFinding(fid); err != nil {
t.Fatal(err)
}
var count int
_ = d.QueryRow(`SELECT count(*) FROM finding_retests WHERE id IN ($1,$2)`, r.ID, next.ID).Scan(&count)
if count != 0 {
t.Fatal("finding deletion did not cascade")
}
_ = d.DeleteConversation(c2.ID)
}
func TestRetestValidationAndRollback(t *testing.T) {
d, fid := retestDB(t)
ctx, cancel := context.WithCancel(t.Context())
cancel()
if _, _, _, err := d.CreateFindingRetest(ctx, fid, ""); err == nil {
t.Fatal("cancelled creation succeeded")
}
for _, args := range [][3]string{{"unknown", "summary", "proof"}, {"fixed", " ", "proof"}, {"fixed", "summary", ""}, {"fixed", strings.Repeat("x", 16001), "proof"}} {
if err := d.RecordFindingRetestResult(t.Context(), 0, args[0], args[1], args[2]); err == nil {
t.Fatal("bad result accepted", args[0])
}
}
r, c, _, err := d.CreateFindingRetest(t.Context(), fid, "snapshot")
if err != nil {
t.Fatal(err)
}
full, err := d.FindingRetestForConversation(t.Context(), c.ID)
var snap map[string]json.RawMessage
if err != nil || json.Unmarshal(full.Snapshot, &snap) != nil || len(snap["finding"]) == 0 {
t.Fatal("invalid snapshot", err)
}
if r.Notes != "snapshot" {
t.Fatal("notes lost")
}
}
func TestRetestFixedTriageOnlyAfterSuccessfulCompletion(t *testing.T) {
for _, tc := range []struct{ verdict, terminal, want string }{
{"fixed", "completed", FindingFixed},
{"fixed", "failed", FindingInProgress},
{"fixed", "stopped", FindingInProgress},
{"reproduced", "completed", FindingInProgress},
{"inconclusive", "completed", FindingInProgress},
{"", "completed", FindingInProgress},
} {
t.Run(tc.verdict+"/"+tc.terminal, func(t *testing.T) {
d, fid := retestDB(t)
if _, err := d.SetFindingStatus(fid, FindingInProgress); err != nil {
t.Fatal(err)
}
r, c, _, err := d.CreateFindingRetest(t.Context(), fid, "check triage")
if err != nil {
t.Fatal(err)
}
if _, err := d.StartFindingRetest(t.Context(), r.ID); err != nil {
t.Fatal(err)
}
if tc.verdict != "" {
if err := d.RecordFindingRetestResult(t.Context(), c.ID, tc.verdict, "summary", "proof"); err != nil {
t.Fatal(err)
}
}
f, err := d.GetFinding(fid)
if err != nil || f.Status != FindingInProgress {
t.Fatal("staged verdict changed triage", err, f)
}
if err := d.FinishFindingRetest(r.ID, tc.terminal, ""); err != nil {
t.Fatal(err)
}
f, err = d.GetFinding(fid)
if err != nil || f.Status != tc.want || f.Evidence != "original evidence" {
t.Fatalf("finding=%+v err=%v want=%s", f, err, tc.want)
}
// Re-delivering completion must not undo a later user decision.
if _, err := d.SetFindingStatus(fid, FindingIgnored); err != nil {
t.Fatal(err)
}
if err := d.FinishFindingRetest(r.ID, "completed", ""); err != nil {
t.Fatal(err)
}
f, err = d.GetFinding(fid)
if err != nil || f.Status != FindingIgnored {
t.Fatal("replayed completion overwrote triage", err, f)
}
})
}
}
+459
View File
@@ -0,0 +1,459 @@
package db
import (
"context"
"crypto/sha256"
"database/sql"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
)
var (
ErrEvidenceConflict = errors.New("流量证据已变更,请刷新后重试")
ErrFindingNotFound = errors.New("漏洞不存在")
ErrEvidenceNotFound = errors.New("流量证据不存在")
)
// This lock covers the evidence filesystem as well as its SQL references. All
// processes sharing the database use it, including readers, exports and GC.
const findingEvidenceLockKey int64 = 7337741004
func (d *DB) WithEvidenceTx(ctx context.Context, fn func(*sql.Tx) error) error {
tx, err := d.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
if _, err = tx.ExecContext(ctx, `SELECT pg_advisory_xact_lock($1)`, findingEvidenceLockKey); err != nil {
return err
}
if err = fn(tx); err != nil {
return err
}
return tx.Commit()
}
type TrafficRef struct {
TrafficID string `json:"traffic_id"`
Role string `json:"role"`
Note string `json:"note"`
}
func NormalizeTrafficRefs(refs []TrafficRef) ([]TrafficRef, error) {
out := make([]TrafficRef, 0, len(refs))
seen := map[string]bool{}
for _, ref := range refs {
ref.TrafficID = strings.TrimSpace(ref.TrafficID)
if ref.TrafficID == "" {
return nil, errors.New("traffic_id 不能为空")
}
if ref.Role == "" {
ref.Role = "supporting"
}
if !ValidTrafficRole(ref.Role) {
return nil, fmt.Errorf("无效的流量用途 %q", ref.Role)
}
if !seen[ref.TrafficID] {
out = append(out, ref)
seen[ref.TrafficID] = true
}
}
return out, nil
}
func ValidTrafficRole(role string) bool {
return role == "baseline" || role == "proof" || role == "verification" || role == "supporting"
}
type TrafficEvidenceSnapshot struct {
ID string `json:"id"`
SourceTrafficID string `json:"source_traffic_id"`
CapturedAt int64 `json:"captured_at"`
URL string `json:"url"`
Method string `json:"method"`
Status int `json:"status"`
ContentType string `json:"content_type"`
ReqHead string `json:"req_head,omitempty"`
RespHead string `json:"resp_head,omitempty"`
ReqHash string `json:"req_hash"`
RespHash string `json:"resp_hash"`
ReqLen int64 `json:"req_len"`
RespLen int64 `json:"resp_len"`
CreatedAt time.Time `json:"created_at"`
}
type FindingTrafficBinding struct {
ID int64 `json:"id,string"`
FindingID int64 `json:"finding_id,string"`
SnapshotID string `json:"snapshot_id"`
Role string `json:"role"`
Note string `json:"note"`
Position int `json:"position"`
CreatedAt time.Time `json:"created_at"`
Snapshot TrafficEvidenceSnapshot `json:"snapshot"`
}
type FindingTraffic struct {
FindingID int64 `json:"finding_id,string"`
Version int64 `json:"version"`
ReportVersion int64 `json:"report_version"`
Bindings []FindingTrafficBinding `json:"bindings"`
}
type PreparedTrafficEvidence struct {
Ref TrafficRef
Snapshot TrafficEvidenceSnapshot
}
// Normalize makes the snapshot's text columns safe for PostgreSQL. URL and the
// head blocks come straight off the wire, so a target answering with a non-UTF-8
// header (a GBK `Content-Disposition: filename=…`, a NUL byte) would otherwise
// abort the INSERT and roll back the whole finding — losing a confirmed finding
// over a malformed response header. Applied before hashing so the ID always
// matches the bytes that actually land in the table.
func (s TrafficEvidenceSnapshot) Normalize() TrafficEvidenceSnapshot {
s.SourceTrafficID = utf8Clean(s.SourceTrafficID)
s.URL = utf8Clean(s.URL)
s.Method = utf8Clean(s.Method)
s.ContentType = utf8Clean(s.ContentType)
s.ReqHead = utf8Clean(s.ReqHead)
s.RespHead = utf8Clean(s.RespHead)
return s
}
func TrafficSnapshotID(snapshot TrafficEvidenceSnapshot) string {
snapshot = snapshot.Normalize()
snapshot.ID = ""
snapshot.CreatedAt = time.Time{}
raw, _ := json.Marshal(snapshot)
sum := sha256.Sum256(raw)
return hex.EncodeToString(sum[:])
}
// LockTaskEvidenceTx serializes writes with archive queueing (which locks the
// same task row). Once queued, its snapshot must not acquire new evidence.
func LockTaskEvidenceTx(tx *sql.Tx, taskID int64) error {
if taskID == 0 {
return nil
}
var deleted sql.NullTime
if err := tx.QueryRow(`SELECT deleted_at FROM tasks WHERE id=$1 FOR UPDATE`, taskID).Scan(&deleted); err != nil {
return err
}
if deleted.Valid {
return ErrTaskArchiveState
}
var state string
err := tx.QueryRow(`SELECT state FROM task_archives WHERE task_id=$1`, taskID).Scan(&state)
if err != nil && !errors.Is(err, sql.ErrNoRows) {
return err
}
if err == nil && state != ArchiveFailed {
return ErrTaskArchiveState
}
return nil
}
func LockFindingEvidenceTx(tx *sql.Tx, findingID int64, version *int64) error {
var taskID sql.NullInt64
if err := tx.QueryRow(`SELECT task_id FROM findings WHERE id=$1`, findingID).Scan(&taskID); err != nil {
if errors.Is(err, sql.ErrNoRows) {
return ErrFindingNotFound
}
return err
}
if err := LockTaskEvidenceTx(tx, taskID.Int64); err != nil {
return err
}
var current int64
if err := tx.QueryRow(`SELECT evidence_version FROM findings WHERE id=$1 FOR UPDATE`, findingID).Scan(&current); err != nil {
if errors.Is(err, sql.ErrNoRows) {
return ErrFindingNotFound
}
return err
}
if version != nil && *version != current {
return ErrEvidenceConflict
}
return nil
}
func InsertEvidenceSnapshotTx(tx *sql.Tx, s TrafficEvidenceSnapshot) error {
if s.ID != TrafficSnapshotID(s) {
return errors.New("证据快照元数据哈希不匹配")
}
// The ID was computed over the normalized form; store those same bytes.
id := s.ID
s = s.Normalize()
s.ID = id
if s.CreatedAt.IsZero() {
s.CreatedAt = time.Now().UTC()
}
_, err := tx.Exec(`INSERT INTO traffic_evidence_snapshots
(id,source_traffic_id,captured_at,url,method,status,content_type,req_head,resp_head,req_hash,resp_hash,req_len,resp_len,created_at)
VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14) ON CONFLICT(id) DO NOTHING`,
s.ID, s.SourceTrafficID, s.CapturedAt, s.URL, s.Method, s.Status, s.ContentType, s.ReqHead, s.RespHead, s.ReqHash, s.RespHash, s.ReqLen, s.RespLen, s.CreatedAt)
return err
}
func AddFindingTrafficTx(tx *sql.Tx, findingID int64, prepared []PreparedTrafficEvidence) error {
var position int
if err := tx.QueryRow(`SELECT COALESCE(MAX(position)+1,0) FROM finding_traffic_bindings WHERE finding_id=$1`, findingID).Scan(&position); err != nil {
return err
}
changed := false
for _, item := range prepared {
if err := InsertEvidenceSnapshotTx(tx, item.Snapshot); err != nil {
return err
}
if _, err := tx.Exec(`UPDATE traffic_evidence_snapshots SET unreferenced_at=NULL WHERE id=$1`, item.Snapshot.ID); err != nil {
return err
}
res, err := tx.Exec(`INSERT INTO finding_traffic_bindings(finding_id,snapshot_id,role,note,position)
VALUES($1,$2,$3,$4,$5) ON CONFLICT(finding_id,snapshot_id) DO NOTHING`, findingID, item.Snapshot.ID, item.Ref.Role, item.Ref.Note, position)
if err != nil {
return err
}
n, err := res.RowsAffected()
if err != nil {
return err
}
if n > 0 {
changed = true
position++
}
}
if changed {
return bumpEvidenceVersionTx(tx, findingID)
}
return nil
}
func bumpEvidenceVersionTx(tx *sql.Tx, findingID int64) error {
_, err := tx.Exec(`UPDATE findings SET evidence_version=evidence_version+1 WHERE id=$1`, findingID)
return err
}
func FindingTrafficTx(tx *sql.Tx, findingID int64) (*FindingTraffic, error) {
out := &FindingTraffic{FindingID: findingID, Bindings: []FindingTrafficBinding{}}
if err := tx.QueryRow(`SELECT evidence_version,report_evidence_version FROM findings WHERE id=$1`, findingID).Scan(&out.Version, &out.ReportVersion); err != nil {
if errors.Is(err, sql.ErrNoRows) {
return nil, ErrFindingNotFound
}
return nil, err
}
rows, err := tx.Query(`SELECT b.id,b.finding_id,b.snapshot_id,b.role,b.note,b.position,b.created_at,to_jsonb(s)
FROM finding_traffic_bindings b JOIN traffic_evidence_snapshots s ON s.id=b.snapshot_id WHERE b.finding_id=$1 ORDER BY b.position,b.id`, findingID)
if err != nil {
return nil, err
}
defer rows.Close()
for rows.Next() {
var b FindingTrafficBinding
var raw []byte
if err := rows.Scan(&b.ID, &b.FindingID, &b.SnapshotID, &b.Role, &b.Note, &b.Position, &b.CreatedAt, &raw); err != nil {
return nil, err
}
if err := json.Unmarshal(raw, &b.Snapshot); err != nil {
return nil, err
}
out.Bindings = append(out.Bindings, b)
}
return out, rows.Err()
}
func (d *DB) GetFindingTraffic(ctx context.Context, findingID int64) (out *FindingTraffic, err error) {
err = d.WithEvidenceTx(ctx, func(tx *sql.Tx) error { var e error; out, e = FindingTrafficTx(tx, findingID); return e })
return
}
func (d *DB) EditFindingTraffic(ctx context.Context, findingID, bindingID, version int64, role, note *string, remove bool, order []int64) error {
if role != nil && !ValidTrafficRole(*role) {
return errors.New("无效的流量用途")
}
return d.WithEvidenceTx(ctx, func(tx *sql.Tx) error {
if err := LockFindingEvidenceTx(tx, findingID, &version); err != nil {
return err
}
if order != nil {
current, err := FindingTrafficTx(tx, findingID)
if err != nil {
return err
}
if len(order) != len(current.Bindings) {
return ErrEvidenceConflict
}
ids := map[int64]bool{}
for _, b := range current.Bindings {
ids[b.ID] = true
}
for position, id := range order {
if !ids[id] {
return ErrEvidenceConflict
}
delete(ids, id)
if _, err := tx.Exec(`UPDATE finding_traffic_bindings SET position=$1 WHERE finding_id=$2 AND id=$3`, position, findingID, id); err != nil {
return err
}
}
} else {
var res sql.Result
var err error
if remove {
res, err = tx.Exec(`DELETE FROM finding_traffic_bindings WHERE finding_id=$1 AND id=$2`, findingID, bindingID)
} else {
res, err = tx.Exec(`UPDATE finding_traffic_bindings SET role=COALESCE($3,role),note=COALESCE($4,note) WHERE finding_id=$1 AND id=$2`, findingID, bindingID, role, note)
}
if err != nil {
return err
}
n, err := res.RowsAffected()
if err != nil {
return err
}
if n == 0 {
return ErrEvidenceNotFound
}
}
return bumpEvidenceVersionTx(tx, findingID)
})
}
type RecordFindingInput struct {
TaskID, ExplorationID, IntentID int64
VulnClass, Name, Severity, Summary, Evidence, Worker string
AssetIDs []int64
}
type RecordedFinding struct {
FindingID int64 `json:"finding_id,string"`
NodeID int64 `json:"finding_node_id,string"`
Traffic *FindingTraffic `json:"traffic"`
}
func RecordFindingTx(tx *sql.Tx, in RecordFindingInput, prepared []PreparedTrafficEvidence) (*RecordedFinding, error) {
if err := LockTaskEvidenceTx(tx, in.TaskID); err != nil {
return nil, err
}
if in.TaskID > 0 {
var expID int64
if err := tx.QueryRow(`SELECT exploration_id FROM tasks WHERE id=$1`, in.TaskID).Scan(&expID); err != nil {
return nil, err
}
if expID != in.ExplorationID {
return nil, errors.New("漏洞所属任务与探索记录不匹配")
}
}
if in.IntentID > 0 {
var ok bool
if err := tx.QueryRow(`SELECT EXISTS(SELECT 1 FROM exploration_nodes WHERE id=$1 AND exploration_id=$2 AND kind='intent')`, in.IntentID, in.ExplorationID).Scan(&ok); err != nil {
return nil, err
}
if !ok {
return nil, errors.New("intent_id 必须是本任务的意图(关联任务意图只读)")
}
}
payload, _ := json.Marshal(map[string]any{"vulnclass": in.VulnClass, "name": in.Name, "severity": in.Severity, "summary": in.Summary, "evidence": map[string]any{"by": in.Worker, "poc": in.Evidence}})
out := &RecordedFinding{}
if err := tx.QueryRow(`INSERT INTO exploration_nodes(exploration_id,kind,payload,priority,state,origin)
VALUES($1,'finding',$2,9,'confirmed',$3) RETURNING id`, in.ExplorationID, string(payload), in.Worker).Scan(&out.NodeID); err != nil {
return nil, err
}
for _, asset := range in.AssetIDs {
if _, err := tx.Exec(`INSERT INTO exploration_anchors(node_id,asset_id) VALUES($1,$2) ON CONFLICT DO NOTHING`, out.NodeID, asset); err != nil {
return nil, err
}
}
if in.IntentID > 0 {
if _, err := tx.Exec(`INSERT INTO exploration_edges(exploration_id,src_id,rel,dst_id) VALUES($1,$2,$3,$4)`, in.ExplorationID, in.IntentID, RelYields, out.NodeID); err != nil {
return nil, err
}
}
assets := in.AssetIDs
if assets == nil {
assets = []int64{}
}
raw, _ := json.Marshal(assets)
if err := tx.QueryRow(`INSERT INTO findings(task_id,node_id,vulnclass,name,severity,summary,evidence,worker,asset_ids)
VALUES(NULLIF($1,0),$2,$3,$4,$5,$6,$7,$8,$9) RETURNING id`, in.TaskID, out.NodeID, in.VulnClass, in.Name, in.Severity, in.Summary, in.Evidence, in.Worker, string(raw)).Scan(&out.FindingID); err != nil {
return nil, err
}
if err := AddFindingTrafficTx(tx, out.FindingID, prepared); err != nil {
return nil, err
}
var err error
out.Traffic, err = FindingTrafficTx(tx, out.FindingID)
return out, err
}
// RecordFinding is the atomic legacy/no-recorder path used by tools and tests.
func (s *ExplorationStore) RecordFinding(ctx context.Context, in RecordFindingInput) (out *RecordedFinding, err error) {
in.ExplorationID = s.expID
err = s.db.WithEvidenceTx(ctx, func(tx *sql.Tx) error { var e error; out, e = RecordFindingTx(tx, in, nil); return e })
return
}
func (d *DB) FindingIDByNodeID(nodeID int64) (id int64, err error) {
err = d.QueryRow(`SELECT id FROM findings WHERE node_id=$1`, nodeID).Scan(&id)
return
}
// PopulateFindingTrafficIDs only enriches already-visible nodes. It performs no
// discovery or ID guessing, and leaves the legacy node ID unchanged.
func (s *ExplorationStore) PopulateFindingTrafficIDs(nodes []*Node) error {
byID := map[int64]*Node{}
var args []any
var placeholders []string
for _, n := range nodes {
if n == nil || n.Kind != KindFinding || byID[n.ID] != nil {
continue
}
byID[n.ID] = n
args = append(args, n.ID)
placeholders = append(placeholders, fmt.Sprintf("$%d", len(args)))
}
if len(args) == 0 {
return nil
}
rows, err := s.db.Query(`SELECT f.id,f.node_id,(SELECT count(*) FROM finding_traffic_bindings b WHERE b.finding_id=f.id) FROM findings f WHERE f.node_id IN (`+strings.Join(placeholders, ",")+`)`, args...)
if err != nil {
return err
}
defer rows.Close()
for rows.Next() {
var findingID, nodeID int64
var count int
if err := rows.Scan(&findingID, &nodeID, &count); err != nil {
return err
}
n := byID[nodeID]
n.FindingID, n.FindingNodeID, n.TrafficCount = findingID, nodeID, count
}
return rows.Err()
}
func (d *DB) SetFindingReportVersionByNodeID(ctx context.Context, nodeID int64, report string, version *int64) (n int64, err error) {
err = d.WithEvidenceTx(ctx, func(tx *sql.Tx) error {
var id int64
if err := tx.QueryRow(`SELECT id FROM findings WHERE node_id=$1`, nodeID).Scan(&id); err != nil {
if errors.Is(err, sql.ErrNoRows) {
return nil
}
return err
}
if err := LockFindingEvidenceTx(tx, id, version); err != nil {
return err
}
res, err := tx.Exec(`UPDATE findings SET report=$2,report_evidence_version=COALESCE($3::bigint,CASE WHEN evidence_version=0 THEN 0 ELSE -1 END) WHERE id=$1`, id, report, version)
if err != nil {
return err
}
n, err = res.RowsAffected()
return err
})
return
}
+85
View File
@@ -0,0 +1,85 @@
package db
import (
"database/sql"
"encoding/json"
"errors"
"fmt"
)
func ArchiveEvidenceSnapshots(snapshot *TaskArchiveSnapshot) ([]TrafficEvidenceSnapshot, error) {
var out []TrafficEvidenceSnapshot
if rawRowCount(snapshot.Tables["traffic_evidence_snapshots"]) == 0 {
return out, nil
}
if snapshot.FormatVersion < 3 {
return nil, ErrTaskArchiveFormatMismatch
}
err := json.Unmarshal(snapshot.Tables["traffic_evidence_snapshots"], &out)
return out, err
}
func restoreFindingTrafficTx(tx *sql.Tx, snapshot *TaskArchiveSnapshot) error {
snapshots, err := ArchiveEvidenceSnapshots(snapshot)
if err != nil {
return err
}
allowed := map[string]bool{}
for _, v := range snapshots {
if allowed[v.ID] {
return errors.New("duplicate archived evidence snapshot")
}
allowed[v.ID] = true
if err = InsertEvidenceSnapshotTx(tx, v); err != nil {
return err
}
}
rows, err := decodeArchiveRows(snapshot.Tables["finding_traffic_bindings"])
if err != nil {
return err
}
if len(rows) > 0 && snapshot.FormatVersion < 3 {
return ErrTaskArchiveFormatMismatch
}
for _, row := range rows {
fid, ok := jsonInt64(row["finding_id"])
if !ok {
return errors.New("invalid archived evidence finding id")
}
sid, _ := row["snapshot_id"].(string)
role, _ := row["role"].(string)
if !allowed[sid] || !ValidTrafficRole(role) {
return errors.New("invalid archived evidence binding")
}
var owned bool
if err = tx.QueryRow(`SELECT EXISTS(SELECT 1 FROM findings WHERE id=$1 AND task_id=$2)`, fid, snapshot.TaskID).Scan(&owned); err != nil {
return err
}
if !owned {
return fmt.Errorf("evidence references finding outside archived task: %d", fid)
}
}
if len(rows) > 0 {
raw, _ := json.Marshal(rows)
if _, err = tx.Exec(`INSERT INTO finding_traffic_bindings SELECT * FROM json_populate_recordset(NULL::finding_traffic_bindings,$1::json)`, string(raw)); err != nil {
return err
}
_, err = tx.Exec(`UPDATE traffic_evidence_snapshots s SET unreferenced_at=NULL WHERE EXISTS(SELECT 1 FROM finding_traffic_bindings b WHERE b.snapshot_id=s.id)`)
}
return err
}
func normalizeArchivedFindingVersions(raw json.RawMessage) (json.RawMessage, error) {
rows, err := decodeArchiveRows(raw)
if err != nil {
return nil, err
}
for _, row := range rows {
for _, key := range []string{"evidence_version", "report_evidence_version"} {
if row[key] == nil {
row[key] = 0
}
}
}
return json.Marshal(rows)
}
+87
View File
@@ -0,0 +1,87 @@
package db
import (
"encoding/json"
"fmt"
"testing"
)
func TestFindingEvidenceLockNamespace(t *testing.T) {
for _, reserved := range []int64{7337741001, 7337741002, 7337741003} {
if findingEvidenceLockKey == reserved {
t.Fatal("evidence lock collides with migration/test/company lock")
}
}
}
func TestFindingTrafficLegacyArchiveDefaults(t *testing.T) {
d, err := Open(testDSN(t))
if err != nil {
t.Fatal(err)
}
defer d.Close()
if err = d.EnsureLLMRecordsTable(); err != nil {
t.Fatal(err)
}
if err = d.EnsureLLMUsageTable(); err != nil {
t.Fatal(err)
}
for _, version := range []int{1, 2} {
t.Run(fmt.Sprint(version), func(t *testing.T) {
task, err := d.CreateTask("legacy archive evidence defaults", "fixture", nil, 0, 0)
if err != nil {
t.Fatal(err)
}
defer func() { d.Exec(`DELETE FROM task_archives WHERE task_id=$1`, task.ID); d.DeleteTask(task.ID) }()
f, err := d.Exploration(task.ExplorationID).RecordFinding(t.Context(), RecordFindingInput{TaskID: task.ID, ExplorationID: task.ExplorationID, Summary: "legacy", Severity: "low"})
if err != nil {
t.Fatal(err)
}
if err = d.SetPaused(task.ID, true); err != nil {
t.Fatal(err)
}
archive, err := d.QueueTaskArchive(task.ID)
if err != nil {
t.Fatal(err)
}
if _, err = d.ClaimTaskArchiveJob(t.Context()); err != nil {
t.Fatal(err)
}
snapshot, err := d.SnapshotTaskArchive(task.ID)
if err != nil {
t.Fatal(err)
}
if err = d.CompleteTaskArchive(archive.ID, snapshot, "/tmp/legacy-test.tar.zst", "fixture", 1, 1); err != nil {
t.Fatal(err)
}
snapshot.FormatVersion = version
delete(snapshot.Tables, "finding_traffic_bindings")
delete(snapshot.Tables, "traffic_evidence_snapshots")
var rows []map[string]any
if err = json.Unmarshal(snapshot.Tables["findings"], &rows); err != nil {
t.Fatal(err)
}
for _, row := range rows {
delete(row, "evidence_version")
delete(row, "report_evidence_version")
}
snapshot.Tables["findings"], _ = json.Marshal(rows)
if _, err = d.QueueTaskArchiveRestore(archive.ID); err != nil {
t.Fatal(err)
}
if _, err = d.ClaimTaskArchiveJob(t.Context()); err != nil {
t.Fatal(err)
}
if _, err = d.RestoreTaskArchive(archive.ID, snapshot, 0); err != nil {
t.Fatal(err)
}
got, err := d.GetFindingTraffic(t.Context(), f.FindingID)
if err != nil {
t.Fatal(err)
}
if got.Version != 0 || got.ReportVersion != 0 || len(got.Bindings) != 0 {
t.Fatal(got)
}
})
}
}
+40 -27
View File
@@ -1,6 +1,7 @@
package db
import (
"context"
"database/sql"
"encoding/json"
"errors"
@@ -14,6 +15,11 @@ import (
// DBFinding is a row in the standalone findings table. It persists across task
// deletion unless the caller explicitly requests related finding cleanup.
type DBFinding struct {
TrafficCount int
EvidenceVersion int64
ReportEvidenceVersion int64
TrafficBindings []FindingTrafficBinding // populated only for export
ID int64
TaskID *int64
NodeID *int64
@@ -35,7 +41,8 @@ const (
FindingPending = "pending" // 待处理
FindingInProgress = "in_progress" // 处理中
FindingConfirmed = "confirmed" // 已确认(真实漏洞,未修复)
FindingResolved = "resolved" // 已处理(已修复)
FindingResolved = "resolved" // 已处理
FindingFixed = "fixed" // 已修复
FindingFalsePositive = "false_positive" // 误报
FindingIgnored = "ignored" // 忽略
FindingDuplicate = "duplicate" // 重复
@@ -45,7 +52,7 @@ const (
// ValidFindingStatus reports whether s is a known triage state.
func ValidFindingStatus(s string) bool {
switch s {
case FindingPending, FindingInProgress, FindingConfirmed, FindingResolved,
case FindingPending, FindingInProgress, FindingConfirmed, FindingResolved, FindingFixed,
FindingFalsePositive, FindingIgnored, FindingDuplicate, FindingRiskAccepted:
return true
}
@@ -97,7 +104,8 @@ func (d *DB) AddFinding(taskID, nodeID int64, vulnclass, name, severity, summary
// list query selects, so scanFinding stays in sync across callers.
const findingSelectCols = `f.id, f.task_id, f.node_id, f.vulnclass, COALESCE(f.name, ''), f.severity, f.summary,
f.evidence, f.worker, f.asset_ids, COALESCE(f.status, 'pending'), f.created_at,
COALESCE(t.description, '') AS task_description`
COALESCE(t.description, '') AS task_description, f.evidence_version, f.report_evidence_version,
(SELECT count(*) FROM finding_traffic_bindings b WHERE b.finding_id=f.id)`
// scanFindings materializes rows selected via findingSelectCols.
func scanFindings(rows interface {
@@ -110,7 +118,7 @@ func scanFindings(rows interface {
f := &DBFinding{}
var aidsJSON string
if err := rows.Scan(&f.ID, &f.TaskID, &f.NodeID, &f.VulnClass, &f.Name, &f.Severity,
&f.Summary, &f.Evidence, &f.Worker, &aidsJSON, &f.Status, &f.CreatedAt, &f.TaskDescription); err != nil {
&f.Summary, &f.Evidence, &f.Worker, &aidsJSON, &f.Status, &f.CreatedAt, &f.TaskDescription, &f.EvidenceVersion, &f.ReportEvidenceVersion, &f.TrafficCount); err != nil {
return nil, err
}
_ = json.Unmarshal([]byte(aidsJSON), &f.AssetIDs)
@@ -503,7 +511,7 @@ func (d *DB) ListFindingsForExport(f FindingFilter, ids []int64) ([]*DBFinding,
var aidsJSON string
if err := rows.Scan(&f.ID, &f.TaskID, &f.NodeID, &f.VulnClass, &f.Name, &f.Severity,
&f.Summary, &f.Evidence, &f.Worker, &aidsJSON, &f.Status, &f.CreatedAt,
&f.TaskDescription, &f.Report); err != nil {
&f.TaskDescription, &f.EvidenceVersion, &f.ReportEvidenceVersion, &f.TrafficCount, &f.Report); err != nil {
return nil, err
}
_ = json.Unmarshal([]byte(aidsJSON), &f.AssetIDs)
@@ -631,7 +639,7 @@ func (d *DB) GetFinding(id int64) (*DBFinding, error) {
WHERE f.id = $1`, id).Scan(
&f.ID, &f.TaskID, &f.NodeID, &f.VulnClass, &f.Name, &f.Severity,
&f.Summary, &f.Evidence, &f.Worker, &aidsJSON, &f.Status, &f.CreatedAt,
&f.TaskDescription, &f.Report)
&f.TaskDescription, &f.EvidenceVersion, &f.ReportEvidenceVersion, &f.TrafficCount, &f.Report)
if err == sql.ErrNoRows {
return nil, nil
}
@@ -647,20 +655,27 @@ func (d *DB) GetFinding(id int64) (*DBFinding, error) {
// list, the per-task 发现 Tab, and the exploration graph alike. Deleting the node
// cascades its edges + node_assets and nulls any activity referencing it. Returns
// rows affected (0 = no finding with that id).
func (d *DB) DeleteFinding(id int64) (int64, error) {
var nodeID *int64
err := d.QueryRow(`DELETE FROM findings WHERE id=$1 RETURNING node_id`, id).Scan(&nodeID)
if err == sql.ErrNoRows {
return 0, nil
}
if err != nil {
return 0, err
}
if nodeID != nil {
// best-effort: the finding is already gone; a stray node must not fail the op.
_, _ = d.Exec(`DELETE FROM exploration_nodes WHERE id=$1 AND kind='finding'`, *nodeID)
}
return 1, nil
func (d *DB) DeleteFinding(id int64) (n int64, err error) {
err = d.WithEvidenceTx(context.Background(), func(tx *sql.Tx) error {
if err := LockFindingEvidenceTx(tx, id, nil); err != nil {
if errors.Is(err, ErrFindingNotFound) {
return nil
}
return err
}
var nodeID sql.NullInt64
if err := tx.QueryRow(`DELETE FROM findings WHERE id=$1 RETURNING node_id`, id).Scan(&nodeID); err != nil {
return err
}
if nodeID.Valid {
if _, err := tx.Exec(`DELETE FROM exploration_nodes WHERE id=$1 AND kind='finding'`, nodeID.Int64); err != nil {
return err
}
}
n = 1
return nil
})
return
}
// DeleteFindingsByTask removes all findings rows of a task. The originating
@@ -687,11 +702,7 @@ func (d *DB) SetFindingStatus(id int64, status string) (int64, error) {
// whose node_id matches — report_finding returns that node id, so an agent tool
// can address the finding it just created. Returns rows affected (0 when no row).
func (d *DB) SetFindingReportByNodeID(nodeID int64, report string) (int64, error) {
res, err := d.Exec(`UPDATE findings SET report=$1 WHERE node_id=$2`, report, nodeID)
if err != nil {
return 0, err
}
return res.RowsAffected()
return d.SetFindingReportVersionByNodeID(context.Background(), nodeID, report, nil)
}
// setFindingCol updates one text column on the standalone finding row AND mirrors
@@ -737,6 +748,8 @@ func (d *DB) SetFindingVulnClass(id int64, vulnclass string) (int64, error) {
// FindingMeta is the standalone-row data (id, triage state, anchored assets) the
// per-task view grafts onto its exploration-node findings.
type FindingMeta struct {
TrafficCount int
ID int64
Status string
AssetIDs []int64
@@ -757,7 +770,7 @@ func (d *DB) FindingMetaByNodeID(taskID int64) (map[int64]FindingMeta, error) {
if taskID <= 0 {
return out, nil
}
rows, err := d.Query(`SELECT node_id, id, COALESCE(status,'pending'), asset_ids FROM findings
rows, err := d.Query(`SELECT node_id, id, COALESCE(status,'pending'), asset_ids, (SELECT count(*) FROM finding_traffic_bindings b WHERE b.finding_id=findings.id) FROM findings
WHERE task_id=$1 AND node_id IS NOT NULL`, taskID)
if err != nil {
return out, err
@@ -767,7 +780,7 @@ func (d *DB) FindingMetaByNodeID(taskID int64) (map[int64]FindingMeta, error) {
var nid int64
var m FindingMeta
var aidsJSON string
if err := rows.Scan(&nid, &m.ID, &m.Status, &aidsJSON); err != nil {
if err := rows.Scan(&nid, &m.ID, &m.Status, &aidsJSON, &m.TrafficCount); err != nil {
return out, err
}
_ = json.Unmarshal([]byte(aidsJSON), &m.AssetIDs)
+42 -15
View File
@@ -3,6 +3,7 @@ package db
import (
"database/sql"
"encoding/json"
"strings"
"time"
)
@@ -34,6 +35,7 @@ type InterceptPending struct {
ToolName string `json:"tool_name"`
ToolInput json.RawMessage `json:"tool_input"`
Status string `json:"status"`
DecisionSource string `json:"decision_source"`
Reason string `json:"reason"` // 规则 message 或模型判定理由(前缀 [模型])
DecidedAt *time.Time `json:"decided_at"`
CreatedAt time.Time `json:"created_at"`
@@ -106,7 +108,7 @@ func (d *DB) ToggleInterceptRule(id int64, enabled bool) error {
// CreateInterceptPending inserts a pending approval record and returns its ID.
// convID == 0 → conversation_id stored as NULL (background task).
// taskID == "" → task_id stored as NULL.
func (d *DB) CreateInterceptPending(ruleID, convID int64, taskID, agentName, toolName string, input []byte, reason string) (int64, error) {
func (d *DB) CreateInterceptPending(ruleID, convID int64, taskID, agentName, toolName string, input []byte, reason string, audits ...*InterceptAudit) (int64, error) {
raw := json.RawMessage(input)
if len(raw) == 0 {
raw = json.RawMessage("{}")
@@ -126,9 +128,9 @@ func (d *DB) CreateInterceptPending(ruleID, convID int64, taskID, agentName, too
}
var id int64
err := d.QueryRow(`
INSERT INTO intercept_pending(rule_id, conversation_id, task_id, agent_name, tool_name, tool_input, reason)
VALUES ($1, $2, $3, $4, $5, $6, $7) RETURNING id`,
ruleIDPtr, convIDPtr, taskIDPtr, agentName, toolName, raw, reason).Scan(&id)
INSERT INTO intercept_pending(rule_id, conversation_id, task_id, agent_name, tool_name, tool_input, reason, decision_source, audit)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) RETURNING id`,
ruleIDPtr, convIDPtr, taskIDPtr, agentName, toolName, raw, reason, interceptSource(ruleID, reason), firstAudit(audits)).Scan(&id)
return id, err
}
@@ -142,7 +144,7 @@ func (d *DB) DecideInterceptPending(id int64, status string) error {
// (status = 'allowed' | 'denied'), decided_at stamped now. Used to log allow/deny
// rule matches for observability — they don't block and need no user action, so unlike
// CreateInterceptPending (which starts 'pending') this records the outcome directly.
func (d *DB) CreateDecidedIntercept(ruleID, convID int64, taskID, agentName, toolName string, input []byte, status, reason string) (int64, error) {
func (d *DB) CreateDecidedIntercept(ruleID, convID int64, taskID, agentName, toolName string, input []byte, status, reason string, audits ...*InterceptAudit) (int64, error) {
raw := json.RawMessage(input)
if len(raw) == 0 {
raw = json.RawMessage("{}")
@@ -162,17 +164,17 @@ func (d *DB) CreateDecidedIntercept(ruleID, convID int64, taskID, agentName, too
}
var id int64
err := d.QueryRow(`
INSERT INTO intercept_pending(rule_id, conversation_id, task_id, agent_name, tool_name, tool_input, status, reason, decided_at)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, NOW()) RETURNING id`,
ruleIDPtr, convIDPtr, taskIDPtr, agentName, toolName, raw, status, reason).Scan(&id)
INSERT INTO intercept_pending(rule_id, conversation_id, task_id, agent_name, tool_name, tool_input, status, reason, decided_at, decision_source, audit)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, NOW(), $9, $10) RETURNING id`,
ruleIDPtr, convIDPtr, taskIDPtr, agentName, toolName, raw, status, reason, interceptSource(ruleID, reason), firstAudit(audits)).Scan(&id)
return id, err
}
const interceptPendingCols = `id, rule_id, conversation_id, task_id, agent_name, tool_name, tool_input, status, reason, decided_at, created_at`
const interceptPendingCols = `id, rule_id, conversation_id, task_id, agent_name, tool_name, tool_input, status, reason, decided_at, created_at, decision_source`
func scanInterceptPending(s interface{ Scan(...any) error }, p *InterceptPending) error {
return s.Scan(&p.ID, &p.RuleID, &p.ConversationID, &p.TaskID, &p.AgentName,
&p.ToolName, &p.ToolInput, &p.Status, &p.Reason, &p.DecidedAt, &p.CreatedAt)
&p.ToolName, &p.ToolInput, &p.Status, &p.Reason, &p.DecidedAt, &p.CreatedAt, &p.DecisionSource)
}
// ListPendingIntercepts returns all unresolved approval requests, newest first.
@@ -218,20 +220,45 @@ func scanInterceptApprovalRow(rows interface{ Scan(...any) error }, r *Intercept
return rows.Scan(
&r.ID, &r.RuleID, &r.ConversationID, &r.TaskID, &r.AgentName,
&r.ToolName, &r.ToolInput, &r.Status, &r.Reason, &r.DecidedAt, &r.CreatedAt,
&r.ConvTitle, &r.ConvAgentKey, &r.RuleName,
&r.DecisionSource, &r.ConvTitle, &r.ConvAgentKey, &r.RuleName,
)
}
const approvalRowSelect = `
SELECT ip.id, ip.rule_id, ip.conversation_id, ip.task_id, ip.agent_name,
ip.tool_name, ip.tool_input, ip.status, ip.reason, ip.decided_at, ip.created_at,
const approvalRowColumns = `ip.id, ip.rule_id, ip.conversation_id, ip.task_id, ip.agent_name,
ip.tool_name, ip.tool_input, ip.status, ip.reason, ip.decided_at, ip.created_at, ip.decision_source,
COALESCE(c.title,'') AS conv_title,
COALESCE(c.agent_key,'') AS conv_agent_key,
COALESCE(ir.name,'') AS rule_name
COALESCE(ir.name,'') AS rule_name`
const approvalRowJoins = `
FROM intercept_pending ip
LEFT JOIN conversations c ON c.id = ip.conversation_id
LEFT JOIN intercept_rules ir ON ir.id = ip.rule_id`
const approvalRowSelect = `SELECT ` + approvalRowColumns + approvalRowJoins
const approvalRowSelectWithAudit = `SELECT ` + approvalRowColumns + `, ip.audit` + approvalRowJoins
func interceptSource(ruleID int64, reason string) string {
if ruleID != 0 {
return "rule"
}
if strings.HasPrefix(reason, "[模型]") {
return "model"
}
return "unknown"
}
func firstAudit(audits []*InterceptAudit) any {
if len(audits) == 0 || audits[0] == nil {
return nil
}
raw, err := json.Marshal(audits[0])
if err != nil {
return nil
}
return raw
}
// ListAllIntercepts returns up to limit intercept_pending rows (newest first)
// joined with conversation and rule info.
func (d *DB) ListAllIntercepts(limit int) ([]InterceptApprovalRow, error) {
+112
View File
@@ -0,0 +1,112 @@
package db
import (
"database/sql"
"encoding/json"
"time"
)
// InterceptContextEntry is a bounded, recorded session event, not model reasoning.
type InterceptContextEntry struct {
Kind string `json:"kind"`
Tool string `json:"tool,omitempty"`
ToolUseID string `json:"tool_use_id,omitempty"`
Text string `json:"text"`
IsError bool `json:"is_error,omitempty"`
Truncated bool `json:"truncated,omitempty"`
}
// InterceptAudit is captured at review time. It is deliberately excluded from
// polling/list responses; old rows have no audit instead of reconstructed data.
type InterceptAudit struct {
RunID string `json:"run_id,omitempty"`
ToolUseID string `json:"tool_use_id,omitempty"`
Correlation string `json:"correlation"` // exact | ambiguous | unavailable
InputDigest string `json:"input_digest"`
UserMessage string `json:"user_message"`
UserTruncated bool `json:"user_truncated,omitempty"`
Context []InterceptContextEntry `json:"context"`
ContextTruncated bool `json:"context_truncated,omitempty"`
CapturedAt time.Time `json:"captured_at"`
ModelFallback bool `json:"model_fallback,omitempty"`
InitialAction string `json:"initial_action"`
InitialReason string `json:"initial_reason"`
EffectiveAction string `json:"effective_action,omitempty"`
DecisionReason string `json:"decision_reason,omitempty"`
RuleName string `json:"rule_name,omitempty"`
ConfigDigest string `json:"config_digest,omitempty"`
ProfileID int64 `json:"profile_id,omitempty"`
ExecutionStatus string `json:"execution_status"`
Output string `json:"output,omitempty"`
OutputTruncated bool `json:"output_truncated,omitempty"`
ExecutionEndedAt *time.Time `json:"execution_ended_at,omitempty"`
}
type InterceptDetail struct {
InterceptApprovalRow
Audit *InterceptAudit `json:"audit"`
}
func (d *DB) GetInterceptDetail(id int64) (*InterceptDetail, error) {
var out InterceptDetail
var raw []byte
err := d.QueryRow(approvalRowSelectWithAudit+` WHERE ip.id=$1`, id).Scan(
&out.ID, &out.RuleID, &out.ConversationID, &out.TaskID, &out.AgentName,
&out.ToolName, &out.ToolInput, &out.Status, &out.Reason, &out.DecidedAt, &out.CreatedAt,
&out.DecisionSource, &out.ConvTitle, &out.ConvAgentKey, &out.RuleName, &raw,
)
if err == sql.ErrNoRows {
return nil, nil
}
if err != nil {
return nil, err
}
if len(raw) > 0 {
if err := json.Unmarshal(raw, &out.Audit); err != nil {
return nil, err
}
}
return &out, nil
}
// ResolveIntercept atomically settles a pending request. A timeout cannot
// overwrite a human decision and repeat decisions cannot rewrite history.
func (d *DB) ResolveIntercept(id int64, status, action, reason string) (bool, error) {
execution := "not_executed"
if action == "allow" {
execution = "awaiting_result"
}
patch, err := json.Marshal(map[string]any{
"effective_action": action, "decision_reason": reason, "execution_status": execution,
})
if err != nil {
return false, err
}
r, err := d.Exec(`UPDATE intercept_pending SET status=$2, decided_at=NOW(),
audit=CASE WHEN audit IS NULL THEN NULL ELSE audit || $3::jsonb ||
CASE WHEN $4='allow' AND audit->>'correlation' IS DISTINCT FROM 'exact'
THEN '{"execution_status":"unknown"}'::jsonb ELSE '{}'::jsonb END END
WHERE id=$1 AND status='pending'`, id, status, patch, action)
if err != nil {
return false, err
}
n, err := r.RowsAffected()
return n == 1, err
}
// CompleteIntercept only updates the exact recorded call after it was allowed.
// A blocked tool_result must never be presented as a failed execution.
func (d *DB) CompleteIntercept(id int64, runID, toolUseID, status, output string, truncated bool) error {
patch, err := json.Marshal(map[string]any{
"execution_status": status, "output": output, "output_truncated": truncated,
"execution_ended_at": time.Now().UTC(),
})
if err != nil {
return err
}
_, err = d.Exec(`UPDATE intercept_pending SET audit=audit || $4::jsonb
WHERE id=$1 AND audit->>'run_id'=$2 AND audit->>'tool_use_id'=$3
AND audit->>'effective_action'='allow' AND audit->>'execution_status'='awaiting_result'`,
id, runID, toolUseID, patch)
return err
}
+167
View File
@@ -0,0 +1,167 @@
package db
import (
"encoding/json"
"strings"
"sync"
"sync/atomic"
"testing"
)
func TestInterceptDetails(t *testing.T) {
d, err := Open(testDSN(t))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = d.Close() })
create := func(t *testing.T, audit *InterceptAudit) int64 {
t.Helper()
id, err := d.CreateInterceptPending(0, 0, "approval-detail-test", "test", "Write", []byte(`{"path":"report.md"}`), "[模型] 请确认", audit)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _, _ = d.Exec(`DELETE FROM intercept_pending WHERE id=$1`, id) })
return id
}
t.Run("legacy and lazy payload", func(t *testing.T) {
id := create(t, nil)
got, err := d.GetInterceptDetail(id)
if err != nil || got == nil || got.Audit != nil {
t.Fatalf("legacy: %+v %v", got, err)
}
create(t, &InterceptAudit{UserMessage: "snapshot-only-marker", InitialAction: "ask"})
items, err := d.ListTaskIntercepts("approval-detail-test")
if err != nil {
t.Fatal(err)
}
raw, _ := json.Marshal(items)
if strings.Contains(string(raw), "snapshot-only-marker") || strings.Contains(string(raw), `"audit"`) {
t.Fatal("snapshot leaked into list response")
}
missing, err := d.GetInterceptDetail(-1)
if err != nil || missing != nil {
t.Fatal("missing record not reported")
}
})
t.Run("decision race and exact output", func(t *testing.T) {
id := create(t, &InterceptAudit{RunID: "run", ToolUseID: "call", Correlation: "exact", InitialAction: "ask", ExecutionStatus: "not_started"})
var wins atomic.Int32
var wg sync.WaitGroup
for range 8 {
wg.Go(func() {
ok, err := d.ResolveIntercept(id, "allowed", "allow", "人工允许执行")
if err != nil {
t.Error(err)
}
if ok {
wins.Add(1)
}
})
}
wg.Wait()
if wins.Load() != 1 {
t.Fatalf("%d decisions won", wins.Load())
}
ok, err := d.ResolveIntercept(id, "timeout", "deny", "late timeout")
if err != nil || ok {
t.Fatal("timeout overwrote decision")
}
if err := d.CompleteIntercept(id, "different-run", "call", "succeeded", "WRONG", false); err != nil {
t.Fatal(err)
}
got, _ := d.GetInterceptDetail(id)
if got.Audit.Output != "" {
t.Fatal("cross-run result attached")
}
if err := d.CompleteIntercept(id, "run", "call", "failed", "permission denied", true); err != nil {
t.Fatal(err)
}
got, err = d.GetInterceptDetail(id)
if err != nil || got.Status != "allowed" || got.Audit.InitialAction != "ask" || got.Audit.ExecutionStatus != "failed" || !got.Audit.OutputTruncated {
t.Fatalf("wrong details: %+v %v", got, err)
}
if err := d.CompleteIntercept(id, "run", "call", "succeeded", "late duplicate", false); err != nil {
t.Fatal(err)
}
got, _ = d.GetInterceptDetail(id)
if got.Audit.Output != "permission denied" {
t.Fatal("duplicate result rewrote output")
}
})
t.Run("denied output and timeout allow", func(t *testing.T) {
id := create(t, &InterceptAudit{RunID: "run", ToolUseID: "call", ExecutionStatus: "not_started"})
if _, err := d.ResolveIntercept(id, "denied", "deny", "人工拒绝"); err != nil {
t.Fatal(err)
}
if err := d.CompleteIntercept(id, "run", "call", "failed", "Blocked by hook", false); err != nil {
t.Fatal(err)
}
got, _ := d.GetInterceptDetail(id)
if got.Audit.ExecutionStatus != "not_executed" || got.Audit.Output != "" {
t.Fatal("denial presented as executed")
}
id = create(t, &InterceptAudit{RunID: "run2", ToolUseID: "call2", Correlation: "exact", InitialAction: "ask"})
if _, err := d.ResolveIntercept(id, "timeout", "allow", "超时允许"); err != nil {
t.Fatal(err)
}
if err := d.CompleteIntercept(id, "run2", "call2", "succeeded", "ok", false); err != nil {
t.Fatal(err)
}
got, _ = d.GetInterceptDetail(id)
if got.Status != "timeout" || got.Audit.EffectiveAction != "allow" || got.Audit.ExecutionStatus != "succeeded" {
t.Fatal("timeout action lost")
}
})
t.Run("archive compatibility", func(t *testing.T) {
for _, legacy := range []bool{true, false} {
id := create(t, &InterceptAudit{InitialAction: "ask", ExecutionStatus: "not_started"})
var raw []byte
if err := d.QueryRow(`SELECT row_to_json(ip) FROM intercept_pending ip WHERE id=$1`, id).Scan(&raw); err != nil {
t.Fatal(err)
}
var row map[string]any
if err := json.Unmarshal(raw, &row); err != nil {
t.Fatal(err)
}
if legacy {
delete(row, "audit")
delete(row, "decision_source")
}
archived, _ := json.Marshal([]map[string]any{row})
tx, err := d.Begin()
if err != nil {
t.Fatal(err)
}
defer tx.Rollback()
if _, err := tx.Exec(`DELETE FROM intercept_pending WHERE id=$1`, id); err != nil {
t.Fatal(err)
}
if err := restoreInterceptRows(tx, archived); err != nil {
t.Fatalf("legacy=%t: %v", legacy, err)
}
var status, source string
var auditJSON []byte
if err := tx.QueryRow(`SELECT status, decision_source, audit FROM intercept_pending WHERE id=$1`, id).Scan(&status, &source, &auditJSON); err != nil {
t.Fatal(err)
}
if status != "timeout" || source != "model" {
t.Fatalf("restored %s/%s", status, source)
}
if legacy && len(auditJSON) > 0 {
t.Fatal("fabricated legacy audit")
}
if !legacy {
var a InterceptAudit
if err := json.Unmarshal(auditJSON, &a); err != nil {
t.Fatal(err)
}
if a.InitialAction != "ask" || a.EffectiveAction != "deny" || a.ExecutionStatus != "not_executed" {
t.Fatalf("restored audit: %+v", a)
}
}
if err := tx.Rollback(); err != nil {
t.Fatal(err)
}
}
})
}
+71 -1
View File
@@ -1024,6 +1024,11 @@ CREATE INDEX IF NOT EXISTS idx_intercept_pending_status ON intercept_pending(sta
CREATE INDEX IF NOT EXISTS idx_intercept_pending_task ON intercept_pending(task_id, created_at DESC);
-- 补旧库:reason 列(已发版,加列要带 IF NOT EXISTS)。
ALTER TABLE intercept_pending ADD COLUMN IF NOT EXISTS reason TEXT NOT NULL DEFAULT '';
-- Detail payloads are lazy-loaded; NULL preserves the meaning of legacy history.
ALTER TABLE intercept_pending ADD COLUMN IF NOT EXISTS audit JSONB;
ALTER TABLE intercept_pending ADD COLUMN IF NOT EXISTS decision_source TEXT NOT NULL DEFAULT '';
UPDATE intercept_pending SET decision_source=CASE WHEN rule_id IS NOT NULL THEN 'rule'
WHEN reason LIKE '[模型]%' THEN 'model' ELSE 'unknown' END WHERE decision_source='';
-- =====================================================================
-- L. 漏洞发现持久化
@@ -1041,7 +1046,7 @@ CREATE TABLE IF NOT EXISTS findings (
evidence TEXT NOT NULL DEFAULT '',
worker TEXT NOT NULL DEFAULT '',
asset_ids JSONB NOT NULL DEFAULT '[]',
-- 处置状态:pending 待处理 / in_progress 处理中 / confirmed 已确认 / resolved 已处理 /
-- 处置状态:pending 待处理 / in_progress 处理中 / confirmed 已确认 / resolved 已处理 / fixed 已修复 /
-- false_positive 误报 / ignored 忽略 / duplicate 重复 / risk_accepted 风险接受。
-- 取值不加 CHECK:旧库靠下面的 ALTER 补列,CHECK 无法回填,统一由 server 侧白名单校验。
status TEXT NOT NULL DEFAULT 'pending',
@@ -1058,6 +1063,71 @@ CREATE INDEX IF NOT EXISTS idx_findings_status ON findings(status, created_at DE
-- 「按资产」视图靠 asset_ids @> '[<id>]' 反查发现,没有这个 GIN 索引就是全表扫。
CREATE INDEX IF NOT EXISTS idx_findings_asset_ids ON findings USING GIN(asset_ids jsonb_path_ops);
-- 手动复测属于独立会话;结论与原漏洞处置状态分开保存。
CREATE TABLE IF NOT EXISTS finding_retests (
id BIGSERIAL PRIMARY KEY,
finding_id BIGINT NOT NULL REFERENCES findings(id) ON DELETE CASCADE,
conversation_id BIGINT UNIQUE REFERENCES conversations(id) ON DELETE SET NULL,
status TEXT NOT NULL DEFAULT 'pending' CHECK (status IN ('pending','running','completed','failed','stopped')),
verdict TEXT NOT NULL DEFAULT '' CHECK (verdict IN ('','reproduced','fixed','inconclusive')),
notes TEXT NOT NULL DEFAULT '',
snapshot JSONB NOT NULL,
summary TEXT NOT NULL DEFAULT '',
evidence TEXT NOT NULL DEFAULT '',
error TEXT NOT NULL DEFAULT '',
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
started_at TIMESTAMPTZ,
finished_at TIMESTAMPTZ
);
CREATE INDEX IF NOT EXISTS idx_finding_retests_history ON finding_retests(finding_id, id DESC);
CREATE UNIQUE INDEX IF NOT EXISTS idx_finding_retests_active ON finding_retests(finding_id)
WHERE status IN ('pending','running');
-- 删除会话保留复测记录,同时解除尚未结束的复测占用。
CREATE OR REPLACE FUNCTION stop_deleted_conversation_retest() RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN
UPDATE finding_retests SET status='stopped', error='复测会话已删除', finished_at=now()
WHERE conversation_id=OLD.id AND status IN ('pending','running');
RETURN OLD;
END;
$$;
DROP TRIGGER IF EXISTS trg_conversation_retest_delete ON conversations;
CREATE TRIGGER trg_conversation_retest_delete BEFORE DELETE ON conversations
FOR EACH ROW EXECUTE FUNCTION stop_deleted_conversation_retest();
ALTER TABLE findings ADD COLUMN IF NOT EXISTS evidence_version BIGINT NOT NULL DEFAULT 0;
ALTER TABLE findings ADD COLUMN IF NOT EXISTS report_evidence_version BIGINT NOT NULL DEFAULT 0;
CREATE TABLE IF NOT EXISTS traffic_evidence_snapshots (
id TEXT PRIMARY KEY,
source_traffic_id TEXT NOT NULL,
captured_at BIGINT NOT NULL,
url TEXT NOT NULL,
method TEXT NOT NULL,
status INTEGER NOT NULL,
content_type TEXT NOT NULL DEFAULT '',
req_head TEXT NOT NULL,
resp_head TEXT NOT NULL,
req_hash TEXT NOT NULL,
resp_hash TEXT NOT NULL,
req_len BIGINT NOT NULL,
resp_len BIGINT NOT NULL,
unreferenced_at TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE TABLE IF NOT EXISTS finding_traffic_bindings (
id BIGSERIAL PRIMARY KEY,
finding_id BIGINT NOT NULL REFERENCES findings(id) ON DELETE CASCADE,
snapshot_id TEXT NOT NULL REFERENCES traffic_evidence_snapshots(id),
role TEXT NOT NULL DEFAULT 'supporting',
note TEXT NOT NULL DEFAULT '',
position INTEGER NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
UNIQUE(finding_id, snapshot_id)
);
CREATE INDEX IF NOT EXISTS idx_finding_traffic_order ON finding_traffic_bindings(finding_id, position, id);
CREATE INDEX IF NOT EXISTS idx_finding_traffic_snapshot ON finding_traffic_bindings(snapshot_id);
-- =====================================================================
-- M. 后端日志持久化
-- =====================================================================
+2
View File
@@ -587,6 +587,8 @@ func (d *DB) snapshotTaskArchive(taskID int64, llmRecords io.Writer) (*TaskArchi
{"task_llm_profiles", `SELECT * FROM task_llm_profiles WHERE task_id=$1 ORDER BY position`, []any{taskID}},
{"task_scope", `SELECT * FROM task_scope WHERE task_id=$1 ORDER BY id`, []any{taskID}},
{"findings", `SELECT * FROM findings WHERE task_id=$1 ORDER BY id`, []any{taskID}},
{"finding_traffic_bindings", `SELECT b.* FROM finding_traffic_bindings b JOIN findings f ON f.id=b.finding_id WHERE f.task_id=$1 ORDER BY b.finding_id,b.position,b.id`, []any{taskID}},
{"traffic_evidence_snapshots", `SELECT s.* FROM traffic_evidence_snapshots s WHERE EXISTS(SELECT 1 FROM finding_traffic_bindings b JOIN findings f ON f.id=b.finding_id WHERE b.snapshot_id=s.id AND f.task_id=$1) ORDER BY s.id`, []any{taskID}},
{"llm_records", `SELECT * FROM llm_records WHERE COALESCE(task_id,'')=$1 ORDER BY id`, []any{strconv.FormatInt(taskID, 10)}},
{"llm_usage", `SELECT * FROM llm_usage WHERE COALESCE(task_id,'')=$1 OR exploration_id=$2 ORDER BY id`, []any{strconv.FormatInt(taskID, 10), expID}},
{"skill_usage", `SELECT * FROM skill_usage WHERE task_id=$1 OR exploration_id=$2 ORDER BY id`, []any{taskID, expID}},
+25
View File
@@ -236,6 +236,10 @@ WHERE archive.id=$1 FOR UPDATE OF archive,task`, archiveID).Scan(&taskID, &expID
if err != nil {
return nil, err
}
remappedTables["findings"], err = normalizeArchivedFindingVersions(remappedTables["findings"])
if err != nil {
return nil, err
}
// Insert graph rows in foreign-key order. The archived stub has no graph rows,
// so an ID conflict signals external corruption and must stop the restore.
for _, table := range []string{"exploration_nodes", "exploration_edges", "exploration_anchors", "task_constraints", "activity"} {
@@ -268,6 +272,9 @@ WHERE archive.id=$1 FOR UPDATE OF archive,task`, archiveID).Scan(&taskID, &expID
return nil, fmt.Errorf("restore %s: %w", table, err)
}
}
if err := restoreFindingTrafficTx(tx, snapshot); err != nil {
return nil, fmt.Errorf("restore finding traffic: %w", err)
}
if streamedLLMRecords != "" {
count, err := insertArchiveJSONSequenceRows(tx, "llm_records", llmRecords)
if err != nil {
@@ -665,10 +672,28 @@ func restoreInterceptRows(tx *sql.Tx, raw json.RawMessage) error {
return err
}
for _, row := range rows {
// Archives predating approval snapshots lack this NOT NULL column.
if source, _ := row["decision_source"].(string); source == "" {
source = "unknown"
if row["rule_id"] != nil {
source = "rule"
} else if reason, _ := row["reason"].(string); strings.HasPrefix(reason, "[模型]") {
source = "model"
}
row["decision_source"] = source
}
if status, _ := row["status"].(string); status == "pending" {
row["status"] = "timeout"
row["reason"] = "任务归档期间审批已超时"
row["decided_at"] = time.Now().UTC()
if audit, ok := row["audit"].(map[string]any); ok {
audit["effective_action"] = "deny"
audit["decision_reason"] = "任务归档期间审批已超时"
audit["execution_status"] = "not_executed"
}
} else if audit, ok := row["audit"].(map[string]any); ok && audit["execution_status"] == "awaiting_result" {
// An archived run cannot resume its former result callback.
audit["execution_status"] = "unknown"
}
}
if len(rows) == 0 {