mirror of
https://github.com/RuoJi6/ARTEX.git
synced 2026-10-27 17:03:49 +08:00
feat(activity): 写失败时带 expID,并对外键(23503)现场诊断
- AppendActivity 失败日志补上 store.expID,直观看到写的是哪个 exploration - 丢弃记录时若为外键违反(SQLSTATE 23503),调用新增的 ExplorationDiag 现场查库: exploration 是否存在 / 引用它的 task 数 / 当前 MAX(exploration.id), 一行日志区分「父行真没了」vs「expID 拿错了」vs「外键未按 RESTRICT 建」 - 系统日志页时间戳补上年月日(原来只有时分秒)
This commit is contained in:
1 parent
daee98b33a
commit
ea833ad992
3 files changed
+44
-3
No files matched your search
@@ -398,6 +398,23 @@ RETURNING id`, s.expID, a.NodeID, utf8Clean(a.Worker), utf8Clean(a.Kind), utf8Cl
|
||||
return id, err
|
||||
}
|
||||
|
||||
// ExplorationDiag probes, at the moment of an activity FK violation (23503), why the
|
||||
// parent row is unreachable. It reports whether the exploration row still exists, how
|
||||
// many task rows reference it (a live task should keep exactly 1 — and RESTRICT on
|
||||
// tasks.exploration_id means the exploration CANNOT be deleted while that row lives),
|
||||
// and the current MAX(explorations.id). Together these tell apart the failure modes:
|
||||
// - expExists=false, taskRefs=0 → the whole row set is gone (DB reset / wrong DB).
|
||||
// - expExists=false, taskRefs=1 → impossible under RESTRICT; would mean a broken FK.
|
||||
// - expExists=true → the write's expID is NOT this exploration (stale/
|
||||
// wrong id in the in-memory store); compare against Store.ID().
|
||||
func (d *DB) ExplorationDiag(expID int64) (expExists bool, taskRefs int, maxExpID int64, err error) {
|
||||
err = d.QueryRow(`SELECT
|
||||
EXISTS(SELECT 1 FROM explorations WHERE id=$1),
|
||||
(SELECT COUNT(*) FROM tasks WHERE exploration_id=$1),
|
||||
COALESCE((SELECT MAX(id) FROM explorations),0)`, expID).Scan(&expExists, &taskRefs, &maxExpID)
|
||||
return
|
||||
}
|
||||
|
||||
// TokenTotal sums token usage across ALL workers for this exploration (whole-task
|
||||
// total). Same source as TokenStatsByWorker (kind='result' rows), just ungrouped.
|
||||
func (s *ExplorationStore) TokenTotal() (TokenUsage, error) {
|
||||
|
||||
+24
-2
@@ -2,6 +2,7 @@ package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"strconv"
|
||||
@@ -14,8 +15,16 @@ import (
|
||||
"github.com/Autumn-27/artex/intercept"
|
||||
"github.com/Autumn-27/norma/harness"
|
||||
"github.com/Autumn-27/norma/llm"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
)
|
||||
|
||||
// isFKViolation reports whether err is a Postgres foreign-key violation (SQLSTATE
|
||||
// 23503) — e.g. an activity insert whose exploration_id has no parent row.
|
||||
func isFKViolation(err error) bool {
|
||||
var pgErr *pgconn.PgError
|
||||
return errors.As(err, &pgErr) && pgErr.Code == "23503"
|
||||
}
|
||||
|
||||
// model_error(provider/API 故障:LLM 层瞬时重试耗尽,或流已开始后中途断流)
|
||||
// 收场的 work 不是「试过没做完」也不是真失败,而是外部抖动。默认把它当永久
|
||||
// blocked 会白丢一条意图,所以这里对该终态额外重跑几次,每次之间退避一下,给
|
||||
@@ -273,6 +282,19 @@ func (e *Engine) emitActivity(t *Task, r db.Activity) {
|
||||
// trace — a tool_use whose tool_result was lost shows as "执行中" forever.
|
||||
log.Printf("[activity] task %s 重试后仍失败、丢弃该记录 (kind=%s tool=%s tuid=%s): %v",
|
||||
t.ID, r.Kind, r.Tool, r.ToolUseID, err)
|
||||
// On the FK-parent failure (23503) probe the live DB so the next occurrence
|
||||
// records WHY the exploration is unreachable instead of just that it is.
|
||||
if isFKViolation(err) {
|
||||
storeID := t.Store.ID()
|
||||
exists, refs, maxID, dErr := e.m.pg.ExplorationDiag(storeID)
|
||||
if dErr != nil {
|
||||
log.Printf("[activity] task %s FK 诊断查询失败 (store.expID=%d task.ExpID=%d): %v",
|
||||
t.ID, storeID, t.ExpID, dErr)
|
||||
} else {
|
||||
log.Printf("[activity] task %s FK 诊断: store.expID=%d task.ExpID=%d exploration存在=%v 引用它的task数=%d 当前MAX(exploration.id)=%d",
|
||||
t.ID, storeID, t.ExpID, exists, refs, maxID)
|
||||
}
|
||||
}
|
||||
e.touch(t.ID)
|
||||
return
|
||||
}
|
||||
@@ -300,8 +322,8 @@ func (e *Engine) appendActivity(t *Task, r db.Activity) (int64, error) {
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
log.Printf("[activity] task %s 写入失败 (第 %d/3 次, worker=%s kind=%s tool=%s): %v",
|
||||
t.ID, attempt, r.Worker, r.Kind, r.Tool, err)
|
||||
log.Printf("[activity] task %s 写入失败 (第 %d/3 次, worker=%s kind=%s tool=%s expID=%d): %v",
|
||||
t.ID, attempt, r.Worker, r.Kind, r.Tool, t.Store.ID(), err)
|
||||
time.Sleep(time.Duration(attempt) * 25 * time.Millisecond)
|
||||
}
|
||||
return 0, err
|
||||
|
||||
@@ -31,7 +31,9 @@ const levelDot: Record<LogLine["level"], string> = {
|
||||
|
||||
function fmtTime(ts: string) {
|
||||
const d = new Date(ts);
|
||||
return isNaN(d.getTime()) ? "" : d.toLocaleTimeString("zh-CN", { hour12: false });
|
||||
if (isNaN(d.getTime())) return "";
|
||||
const p = (n: number) => String(n).padStart(2, "0");
|
||||
return `${d.getFullYear()}-${p(d.getMonth() + 1)}-${p(d.getDate())} ${p(d.getHours())}:${p(d.getMinutes())}:${p(d.getSeconds())}`;
|
||||
}
|
||||
|
||||
export default function LogsPage() {
|
||||
|
||||
Reference in new issue
Block a user