feat(intercept): 模型兜底审批单独计量 token 并在配置页展示

兜底审批(LLM judge)调模型判定漏网命令,其 token 消耗此前无法单独查看:
usage 行要么无 worker 归属、要么被并进复用的模型 profile 桶里。

- llmrec 增加 WithWorker 上下文覆盖,reviewer 调用前标注 worker='judge',
  让审批调用的计量行可被单独筛出(复用现有 llm_usage.worker 列,无需迁移)。
- db.JudgeUsageStats 返回累计总量 + 近 N 天每日序列(WHERE worker='judge')。
- 新增 GET /api/intercept/judge/usage。
- 配置页「模型兜底审批」开关下方新增用量卡片:调用次数 / 输入 / 输出 /
  缓存读写 五项统计 + 近 30 天每日消耗迷你柱图 + 刷新。
This commit is contained in:
Autumn-27 committed 2026-10-04 02:43:26 -04:00
1 parent d0033724a6
commit 86729b65dc
8 files changed
+242 -1

No files matched your search

+64
View File
@@ -259,6 +259,70 @@ ORDER BY day`, days)
return out, nil return out, nil
} }
// JudgeDayUsage is one UTC-day token bucket for the intercept fallback judge,
// for the config page's recent-spend sparkline.
type JudgeDayUsage struct {
Date string `json:"date"` // YYYY-MM-DD (UTC)
Calls int `json:"calls"`
InputTokens int `json:"input_tokens"`
OutputTokens int `json:"output_tokens"`
}
// JudgeUsage is the cumulative token spend of the intercept fallback judge
// (worker='judge' rows in the always-on ledger), plus a recent daily series.
type JudgeUsage struct {
Calls int `json:"calls"`
InputTokens int `json:"input_tokens"`
OutputTokens int `json:"output_tokens"`
CacheReadTokens int `json:"cache_read_tokens"`
CacheWriteTokens int `json:"cache_write_tokens"`
Daily []JudgeDayUsage `json:"daily"`
}
// JudgeUsageStats returns the fallback judge's all-time token totals and a
// per-day series over the past `days` days (default 30). Sourced from the
// always-on llm_usage ledger, so it is accurate across pool rotation and
// interrupted/failed judge calls. days only bounds the daily series; totals are
// all-time.
func (d *DB) JudgeUsageStats(days int) (JudgeUsage, error) {
if days <= 0 {
days = 30
}
var u JudgeUsage
err := d.QueryRow(`
SELECT COUNT(*), COALESCE(SUM(input_tokens),0), COALESCE(SUM(output_tokens),0),
COALESCE(SUM(cache_read),0), COALESCE(SUM(cache_write),0)
FROM llm_usage
WHERE worker = 'judge'`).Scan(&u.Calls, &u.InputTokens, &u.OutputTokens,
&u.CacheReadTokens, &u.CacheWriteTokens)
if err != nil {
return JudgeUsage{}, err
}
rows, err := d.Query(`
SELECT to_char(ts AT TIME ZONE 'UTC', 'YYYY-MM-DD') AS day,
COUNT(*), COALESCE(SUM(input_tokens),0), COALESCE(SUM(output_tokens),0)
FROM llm_usage
WHERE worker = 'judge' AND ts >= now() - ($1 * interval '1 day')
GROUP BY day
ORDER BY day`, days)
if err != nil {
return JudgeUsage{}, err
}
defer rows.Close()
u.Daily = []JudgeDayUsage{}
for rows.Next() {
var day JudgeDayUsage
if err := rows.Scan(&day.Date, &day.Calls, &day.InputTokens, &day.OutputTokens); err != nil {
return JudgeUsage{}, err
}
u.Daily = append(u.Daily, day)
}
if err := rows.Err(); err != nil {
return JudgeUsage{}, err
}
return u, nil
}
// ParseExpID turns the exploration-id segment parsed from a session string into an // ParseExpID turns the exploration-id segment parsed from a session string into an
// int64 (0 when empty/non-numeric, e.g. chat sessions keyed by conversation id). // int64 (0 when empty/non-numeric, e.g. chat sessions keyed by conversation id).
func ParseExpID(s string) int64 { func ParseExpID(s string) int64 {
+29
View File
@@ -17,6 +17,7 @@ import (
) )
type taskIDContextKey struct{} type taskIDContextKey struct{}
type workerContextKey struct{}
// WithTaskID attaches the owning task registry id to an LLM call. Session ids // WithTaskID attaches the owning task registry id to an LLM call. Session ids
// are based on exploration ids, which are not interchangeable with task ids. // are based on exploration ids, which are not interchangeable with task ids.
@@ -37,6 +38,28 @@ func TaskIDFrom(ctx context.Context) string {
return strings.TrimSpace(taskID) return strings.TrimSpace(taskID)
} }
// WithWorker overrides the agent-lane ("worker") label for an LLM call. The lane
// is normally parsed from the transcript session id (exp<N>-<role>); calls made
// outside the engine's worker/planner sessions — e.g. the intercept fallback judge
// — carry no such session, so they attach their lane explicitly here. This lets the
// usage ledger single out that spend (worker='judge') for the config page.
func WithWorker(ctx context.Context, worker string) context.Context {
worker = strings.TrimSpace(worker)
if worker == "" {
return ctx
}
return context.WithValue(ctx, workerContextKey{}, worker)
}
// workerFrom returns the explicit lane override, or "" when none is set.
func workerFrom(ctx context.Context) string {
if ctx == nil {
return ""
}
worker, _ := ctx.Value(workerContextKey{}).(string)
return strings.TrimSpace(worker)
}
// Recorder wraps an llm.Provider and records every completion call. // Recorder wraps an llm.Provider and records every completion call.
type Recorder struct { type Recorder struct {
inner llm.Provider inner llm.Provider
@@ -101,6 +124,9 @@ func (r *Recorder) Stream(ctx context.Context, req llm.CompletionRequest) iter.S
start := time.Now() start := time.Now()
session := transcript.SessionIDFrom(ctx) session := transcript.SessionIDFrom(ctx)
parsedID, worker := parseSession(session) parsedID, worker := parseSession(session)
if ov := workerFrom(ctx); ov != "" {
worker = ov
}
expID := db.ParseExpID(parsedID) expID := db.ParseExpID(parsedID)
taskID := TaskIDFrom(ctx) taskID := TaskIDFrom(ctx)
if taskID == "" { if taskID == "" {
@@ -198,6 +224,9 @@ func (r *Recorder) Complete(ctx context.Context, req llm.CompletionRequest) (llm
start := time.Now() start := time.Now()
session := transcript.SessionIDFrom(ctx) session := transcript.SessionIDFrom(ctx)
parsedID, worker := parseSession(session) parsedID, worker := parseSession(session)
if ov := workerFrom(ctx); ov != "" {
worker = ov
}
expID := db.ParseExpID(parsedID) expID := db.ParseExpID(parsedID)
taskID := TaskIDFrom(ctx) taskID := TaskIDFrom(ctx)
if taskID == "" { if taskID == "" {
+25
View File
@@ -14,9 +14,14 @@ import (
"github.com/Autumn-27/artex/db" "github.com/Autumn-27/artex/db"
"github.com/Autumn-27/artex/guard" "github.com/Autumn-27/artex/guard"
"github.com/Autumn-27/artex/intercept" "github.com/Autumn-27/artex/intercept"
"github.com/Autumn-27/artex/llmrec"
"github.com/Autumn-27/norma/llm" "github.com/Autumn-27/norma/llm"
) )
// judgeWorkerLane is the usage-ledger "worker" label for intercept fallback-judge
// calls, so judge spend can be queried apart from the worker/planner/main lanes.
const judgeWorkerLane = "judge"
// chatGuard returns a guard wired with the manager's interceptor, used for chat // chatGuard returns a guard wired with the manager's interceptor, used for chat
// conversations. Called once per applyLLM so a new LLM config always gets a fresh guard. // conversations. Called once per applyLLM so a new LLM config always gets a fresh guard.
func (s *Server) chatGuard() *guard.Guard { func (s *Server) chatGuard() *guard.Guard {
@@ -41,6 +46,9 @@ func (s *Server) wireInterceptReviewer() {
if !ok { if !ok {
return intercept.Decision{ProfileID: profileID}, fmt.Errorf("裁判模型 profile %d 不可用", profileID) return intercept.Decision{ProfileID: profileID}, fmt.Errorf("裁判模型 profile %d 不可用", profileID)
} }
// Tag this call's usage as the "judge" lane so the config page can report
// how much the fallback approval has spent, separate from model profiles.
ctx = llmrec.WithWorker(ctx, judgeWorkerLane)
text, err := reviewCompletion(ctx, prov, prompt, input) text, err := reviewCompletion(ctx, prov, prompt, input)
if err != nil { if err != nil {
return intercept.Decision{ProfileID: profileID}, err return intercept.Decision{ProfileID: profileID}, err
@@ -437,6 +445,23 @@ func (s *Server) interceptSetJudgeConfig(w http.ResponseWriter, r *http.Request)
writeJSON(w, 200, map[string]any{"ok": true}) writeJSON(w, 200, map[string]any{"ok": true})
} }
// interceptJudgeUsage returns the fallback judge's cumulative token spend plus a
// recent daily series, for the config page. ?days bounds the daily series (default 30).
func (s *Server) interceptJudgeUsage(w http.ResponseWriter, r *http.Request) {
pg := s.m.PG()
if pg == nil {
writeJSON(w, 200, db.JudgeUsage{Daily: []db.JudgeDayUsage{}})
return
}
days := atoiDefault(r.URL.Query().Get("days"), 30)
usage, err := pg.JudgeUsageStats(days)
if err != nil {
writeErr(w, 500, err.Error())
return
}
writeJSON(w, 200, usage)
}
// --- helpers --- // --- helpers ---
type interceptRuleReq struct { type interceptRuleReq struct {
+1
View File
@@ -930,6 +930,7 @@ func (s *Server) Handler() http.Handler {
mux.HandleFunc("PUT /api/intercept/tool-config", s.interceptSetToolConfig) mux.HandleFunc("PUT /api/intercept/tool-config", s.interceptSetToolConfig)
mux.HandleFunc("GET /api/intercept/judge", s.interceptGetJudgeConfig) mux.HandleFunc("GET /api/intercept/judge", s.interceptGetJudgeConfig)
mux.HandleFunc("PUT /api/intercept/judge", s.interceptSetJudgeConfig) mux.HandleFunc("PUT /api/intercept/judge", s.interceptSetJudgeConfig)
mux.HandleFunc("GET /api/intercept/judge/usage", s.interceptJudgeUsage) // 兜底审批累计 token 用量
// /api/* goes through CORS + JWT; everything else is served by the embedded // /api/* goes through CORS + JWT; everything else is served by the embedded
// frontend (public — auth is enforced client-side and on the API). With the // frontend (public — auth is enforced client-side and on the API). With the
+96 -1
View File
@@ -51,7 +51,53 @@ import {
import { Card, CardContent } from "@/components/ui/card"; import { Card, CardContent } from "@/components/ui/card";
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
import { api } from "@/lib/api"; import { api } from "@/lib/api";
import type { InterceptRule, InterceptAction, JudgeConfig, LLMProfile, Tool } from "@/lib/types"; import type {
InterceptAction,
InterceptRule,
JudgeConfig,
JudgeDayUsage,
JudgeUsage,
LLMProfile,
Tool,
} from "@/lib/types";
// fmtTokens 把 token 数压成紧凑写法(1.2k / 3.4M),用于审批用量统计。
function fmtTokens(n: number): string {
if (n >= 1_000_000) return (n / 1_000_000).toFixed(1) + "M";
if (n >= 1000) return (n / 1000).toFixed(1) + "k";
return String(n);
}
// JudgeStat 是一块统计数字(标签 + 数值)。
function JudgeStat({ label, value }: { label: string; value: string }) {
return (
<div className="rounded-lg border bg-muted/20 px-3 py-2">
<div className="text-[10px] text-muted-foreground">{label}</div>
<div className="mt-0.5 text-lg font-semibold tabular-nums">{value}</div>
</div>
);
}
// JudgeSparkbars 用纯 div 画近 N 天每日消耗(输入+输出)的迷你柱图,免图表库依赖。
function JudgeSparkbars({ daily }: { daily: JudgeDayUsage[] }) {
const max = Math.max(1, ...daily.map((d) => d.input_tokens + d.output_tokens));
return (
<div className="flex h-16 items-end gap-0.5">
{daily.map((d) => {
const total = d.input_tokens + d.output_tokens;
const h = Math.max(2, Math.round((total / max) * 100));
return (
<div
key={d.date}
title={`${d.date} · ${d.calls} 次 · ${fmtTokens(total)} tokens`}
className="min-w-[2px] flex-1 rounded-sm bg-violet-500/60 hover:bg-violet-500"
style={{ height: `${h}%` }}
/>
);
})}
</div>
);
}
// ---- tool scope ---- // ---- tool scope ----
@@ -164,6 +210,16 @@ function JudgeCard() {
const [profiles, setProfiles] = React.useState<LLMProfile[]>([]); const [profiles, setProfiles] = React.useState<LLMProfile[]>([]);
const [loading, setLoading] = React.useState(true); const [loading, setLoading] = React.useState(true);
const [saving, setSaving] = React.useState(false); const [saving, setSaving] = React.useState(false);
const [usage, setUsage] = React.useState<JudgeUsage | null>(null);
// 审批用量统计:失败不打断配置页,仅在开启时拉取。
const loadUsage = React.useCallback(async () => {
try {
setUsage(await api.interceptJudgeUsage(30));
} catch {
// 忽略:统计不可用不应影响配置编辑
}
}, []);
const load = React.useCallback(async () => { const load = React.useCallback(async () => {
setLoading(true); setLoading(true);
@@ -182,6 +238,11 @@ function JudgeCard() {
load(); load();
}, [load]); }, [load]);
// 开启后(含初次加载把开关读为 true 时)拉取审批用量统计。
React.useEffect(() => {
if (cfg.enabled) loadUsage();
}, [cfg.enabled, loadUsage]);
function patch(p: Partial<JudgeConfig>) { function patch(p: Partial<JudgeConfig>) {
setCfg((c) => ({ ...c, ...p })); setCfg((c) => ({ ...c, ...p }));
} }
@@ -237,6 +298,40 @@ function JudgeCard() {
</div> </div>
</div> </div>
{/* 审批 Token 用量统计(全局累计,独立于各模型配置) */}
{cfg.enabled && usage && (
<Card>
<CardContent className="p-4">
<div className="mb-3 flex items-center justify-between">
<div>
<p className="text-sm font-medium">审批 Token 消耗</p>
<p className="text-xs text-muted-foreground">
模型兜底审批累计用量,单独计量(worker=judge),不与各模型配置的统计混在一起
</p>
</div>
<Button variant="ghost" size="sm" className="h-7 text-xs" onClick={loadUsage}>
刷新
</Button>
</div>
<div className="grid grid-cols-2 gap-3 sm:grid-cols-5">
<JudgeStat label="审批调用" value={usage.calls.toLocaleString()} />
<JudgeStat label="输入 Token" value={fmtTokens(usage.input_tokens)} />
<JudgeStat label="输出 Token" value={fmtTokens(usage.output_tokens)} />
<JudgeStat label="缓存读" value={fmtTokens(usage.cache_read_tokens)} />
<JudgeStat label="缓存写" value={fmtTokens(usage.cache_write_tokens)} />
</div>
{usage.daily.length > 0 && (
<div className="mt-4">
<p className="mb-2 text-[10px] uppercase tracking-wider text-muted-foreground">
近 30 天每日消耗(输入 + 输出)
</p>
<JudgeSparkbars daily={usage.daily} />
</div>
)}
</CardContent>
</Card>
)}
{cfg.enabled && ( {cfg.enabled && (
<div className="grid gap-4 lg:grid-cols-5"> <div className="grid gap-4 lg:grid-cols-5">
{/* 左:提示词编辑器(直接展开,主区域) */} {/* 左:提示词编辑器(直接展开,主区域) */}
+2
View File
@@ -53,6 +53,7 @@ import type {
InterceptPending, InterceptPending,
InterceptRule, InterceptRule,
JudgeConfig, JudgeConfig,
JudgeUsage,
LLMPoolStatus, LLMPoolStatus,
LLMProfile, LLMProfile,
LLMRecordDetail, LLMRecordDetail,
@@ -1265,6 +1266,7 @@ export const api = {
// ---- intercept LLM judge (模型兜底审批,全局配置) ---- // ---- intercept LLM judge (模型兜底审批,全局配置) ----
interceptGetJudgeConfig: () => get<JudgeConfig>("/intercept/judge"), interceptGetJudgeConfig: () => get<JudgeConfig>("/intercept/judge"),
interceptSetJudgeConfig: (cfg: JudgeConfig) => put<{ ok: boolean }>("/intercept/judge", cfg), interceptSetJudgeConfig: (cfg: JudgeConfig) => put<{ ok: boolean }>("/intercept/judge", cfg),
interceptJudgeUsage: (days = 30) => get<JudgeUsage>(`/intercept/judge/usage?days=${days}`),
// ---- commands (tool execution history, any tool) ---- // ---- commands (tool execution history, any tool) ----
commands: (params?: { task?: string; q?: string; page?: number; size?: number }) => { commands: (params?: { task?: string; q?: string; page?: number; size?: number }) => {
+9
View File
@@ -2464,6 +2464,15 @@ function route(m: string, path: string, seg: string[], q: URLSearchParams, b: Re
ask_timeout_action: "deny", ask_timeout_action: "deny",
}; };
if (path === "/intercept/judge" && m === "PUT") return { ok: true }; if (path === "/intercept/judge" && m === "PUT") return { ok: true };
if (path === "/intercept/judge/usage" && m === "GET")
return {
calls: 0,
input_tokens: 0,
output_tokens: 0,
cache_read_tokens: 0,
cache_write_tokens: 0,
daily: [],
};
// ── 旁路提问(/btw):demo 无旁路会话 ── // ── 旁路提问(/btw):demo 无旁路会话 ──
// 必须显式命中:路径以 s 结尾会被下面的读兜底判成集合返回 [],items 就成了 undefined。 // 必须显式命中:路径以 s 结尾会被下面的读兜底判成集合返回 [],items 就成了 undefined。
+16
View File
@@ -1332,6 +1332,22 @@ export interface JudgeConfig {
ask_timeout_action: "allow" | "deny"; // 审批超时后的默认动作 ask_timeout_action: "allow" | "deny"; // 审批超时后的默认动作
} }
// JudgeUsage: 模型兜底审批(judge 通道)的累计 token 用量 + 近 N 天每日序列。
export interface JudgeDayUsage {
date: string; // YYYY-MM-DD (UTC)
calls: number;
input_tokens: number;
output_tokens: number;
}
export interface JudgeUsage {
calls: number;
input_tokens: number;
output_tokens: number;
cache_read_tokens: number;
cache_write_tokens: number;
daily: JudgeDayUsage[];
}
export interface InterceptApprovalFilter { export interface InterceptApprovalFilter {
status?: InterceptPending["status"]; status?: InterceptPending["status"];
decision_source?: "rule" | "model" | "unknown"; decision_source?: "rule" | "model" | "unknown";