diff --git a/db/exploration.go b/db/exploration.go index 4e679c5..5b0f79c 100644 --- a/db/exploration.go +++ b/db/exploration.go @@ -398,6 +398,23 @@ RETURNING id`, s.expID, a.NodeID, utf8Clean(a.Worker), utf8Clean(a.Kind), utf8Cl return id, err } +// ExplorationDiag probes, at the moment of an activity FK violation (23503), why the +// parent row is unreachable. It reports whether the exploration row still exists, how +// many task rows reference it (a live task should keep exactly 1 — and RESTRICT on +// tasks.exploration_id means the exploration CANNOT be deleted while that row lives), +// and the current MAX(explorations.id). Together these tell apart the failure modes: +// - expExists=false, taskRefs=0 → the whole row set is gone (DB reset / wrong DB). +// - expExists=false, taskRefs=1 → impossible under RESTRICT; would mean a broken FK. +// - expExists=true → the write's expID is NOT this exploration (stale/ +// wrong id in the in-memory store); compare against Store.ID(). +func (d *DB) ExplorationDiag(expID int64) (expExists bool, taskRefs int, maxExpID int64, err error) { + err = d.QueryRow(`SELECT + EXISTS(SELECT 1 FROM explorations WHERE id=$1), + (SELECT COUNT(*) FROM tasks WHERE exploration_id=$1), + COALESCE((SELECT MAX(id) FROM explorations),0)`, expID).Scan(&expExists, &taskRefs, &maxExpID) + return +} + // TokenTotal sums token usage across ALL workers for this exploration (whole-task // total). Same source as TokenStatsByWorker (kind='result' rows), just ungrouped. func (s *ExplorationStore) TokenTotal() (TokenUsage, error) { diff --git a/server/engine.go b/server/engine.go index c235d48..537853e 100644 --- a/server/engine.go +++ b/server/engine.go @@ -2,6 +2,7 @@ package server import ( "context" + "errors" "fmt" "log" "strconv" @@ -14,8 +15,16 @@ import ( "github.com/Autumn-27/artex/intercept" "github.com/Autumn-27/norma/harness" "github.com/Autumn-27/norma/llm" + "github.com/jackc/pgx/v5/pgconn" ) +// isFKViolation reports whether err is a Postgres foreign-key violation (SQLSTATE +// 23503) — e.g. an activity insert whose exploration_id has no parent row. +func isFKViolation(err error) bool { + var pgErr *pgconn.PgError + return errors.As(err, &pgErr) && pgErr.Code == "23503" +} + // model_error(provider/API 故障:LLM 层瞬时重试耗尽,或流已开始后中途断流) // 收场的 work 不是「试过没做完」也不是真失败,而是外部抖动。默认把它当永久 // blocked 会白丢一条意图,所以这里对该终态额外重跑几次,每次之间退避一下,给 @@ -273,6 +282,19 @@ func (e *Engine) emitActivity(t *Task, r db.Activity) { // trace — a tool_use whose tool_result was lost shows as "执行中" forever. log.Printf("[activity] task %s 重试后仍失败、丢弃该记录 (kind=%s tool=%s tuid=%s): %v", t.ID, r.Kind, r.Tool, r.ToolUseID, err) + // On the FK-parent failure (23503) probe the live DB so the next occurrence + // records WHY the exploration is unreachable instead of just that it is. + if isFKViolation(err) { + storeID := t.Store.ID() + exists, refs, maxID, dErr := e.m.pg.ExplorationDiag(storeID) + if dErr != nil { + log.Printf("[activity] task %s FK 诊断查询失败 (store.expID=%d task.ExpID=%d): %v", + t.ID, storeID, t.ExpID, dErr) + } else { + log.Printf("[activity] task %s FK 诊断: store.expID=%d task.ExpID=%d exploration存在=%v 引用它的task数=%d 当前MAX(exploration.id)=%d", + t.ID, storeID, t.ExpID, exists, refs, maxID) + } + } e.touch(t.ID) return } @@ -300,8 +322,8 @@ func (e *Engine) appendActivity(t *Task, r db.Activity) (int64, error) { } return id, nil } - log.Printf("[activity] task %s 写入失败 (第 %d/3 次, worker=%s kind=%s tool=%s): %v", - t.ID, attempt, r.Worker, r.Kind, r.Tool, err) + log.Printf("[activity] task %s 写入失败 (第 %d/3 次, worker=%s kind=%s tool=%s expID=%d): %v", + t.ID, attempt, r.Worker, r.Kind, r.Tool, t.Store.ID(), err) time.Sleep(time.Duration(attempt) * 25 * time.Millisecond) } return 0, err diff --git a/web/src/app/(main)/system/logs/page.tsx b/web/src/app/(main)/system/logs/page.tsx index cb89f45..9ba1fca 100644 --- a/web/src/app/(main)/system/logs/page.tsx +++ b/web/src/app/(main)/system/logs/page.tsx @@ -31,7 +31,9 @@ const levelDot: Record = { function fmtTime(ts: string) { const d = new Date(ts); - return isNaN(d.getTime()) ? "" : d.toLocaleTimeString("zh-CN", { hour12: false }); + if (isNaN(d.getTime())) return ""; + const p = (n: number) => String(n).padStart(2, "0"); + return `${d.getFullYear()}-${p(d.getMonth() + 1)}-${p(d.getDate())} ${p(d.getHours())}:${p(d.getMinutes())}:${p(d.getSeconds())}`; } export default function LogsPage() {