mirror of
https://github.com/RuoJi6/ARTEX.git
synced 2026-10-27 17:03:49 +08:00
feat(intercept): 模型兜底审批单独计量 token 并在配置页展示
兜底审批(LLM judge)调模型判定漏网命令,其 token 消耗此前无法单独查看: usage 行要么无 worker 归属、要么被并进复用的模型 profile 桶里。 - llmrec 增加 WithWorker 上下文覆盖,reviewer 调用前标注 worker='judge', 让审批调用的计量行可被单独筛出(复用现有 llm_usage.worker 列,无需迁移)。 - db.JudgeUsageStats 返回累计总量 + 近 N 天每日序列(WHERE worker='judge')。 - 新增 GET /api/intercept/judge/usage。 - 配置页「模型兜底审批」开关下方新增用量卡片:调用次数 / 输入 / 输出 / 缓存读写 五项统计 + 近 30 天每日消耗迷你柱图 + 刷新。
This commit is contained in:
1 parent
d0033724a6
commit
86729b65dc
8 files changed
+242
-1
No files matched your search
@@ -259,6 +259,70 @@ ORDER BY day`, days)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// JudgeDayUsage is one UTC-day token bucket for the intercept fallback judge,
|
||||
// for the config page's recent-spend sparkline.
|
||||
type JudgeDayUsage struct {
|
||||
Date string `json:"date"` // YYYY-MM-DD (UTC)
|
||||
Calls int `json:"calls"`
|
||||
InputTokens int `json:"input_tokens"`
|
||||
OutputTokens int `json:"output_tokens"`
|
||||
}
|
||||
|
||||
// JudgeUsage is the cumulative token spend of the intercept fallback judge
|
||||
// (worker='judge' rows in the always-on ledger), plus a recent daily series.
|
||||
type JudgeUsage struct {
|
||||
Calls int `json:"calls"`
|
||||
InputTokens int `json:"input_tokens"`
|
||||
OutputTokens int `json:"output_tokens"`
|
||||
CacheReadTokens int `json:"cache_read_tokens"`
|
||||
CacheWriteTokens int `json:"cache_write_tokens"`
|
||||
Daily []JudgeDayUsage `json:"daily"`
|
||||
}
|
||||
|
||||
// JudgeUsageStats returns the fallback judge's all-time token totals and a
|
||||
// per-day series over the past `days` days (default 30). Sourced from the
|
||||
// always-on llm_usage ledger, so it is accurate across pool rotation and
|
||||
// interrupted/failed judge calls. days only bounds the daily series; totals are
|
||||
// all-time.
|
||||
func (d *DB) JudgeUsageStats(days int) (JudgeUsage, error) {
|
||||
if days <= 0 {
|
||||
days = 30
|
||||
}
|
||||
var u JudgeUsage
|
||||
err := d.QueryRow(`
|
||||
SELECT COUNT(*), COALESCE(SUM(input_tokens),0), COALESCE(SUM(output_tokens),0),
|
||||
COALESCE(SUM(cache_read),0), COALESCE(SUM(cache_write),0)
|
||||
FROM llm_usage
|
||||
WHERE worker = 'judge'`).Scan(&u.Calls, &u.InputTokens, &u.OutputTokens,
|
||||
&u.CacheReadTokens, &u.CacheWriteTokens)
|
||||
if err != nil {
|
||||
return JudgeUsage{}, err
|
||||
}
|
||||
rows, err := d.Query(`
|
||||
SELECT to_char(ts AT TIME ZONE 'UTC', 'YYYY-MM-DD') AS day,
|
||||
COUNT(*), COALESCE(SUM(input_tokens),0), COALESCE(SUM(output_tokens),0)
|
||||
FROM llm_usage
|
||||
WHERE worker = 'judge' AND ts >= now() - ($1 * interval '1 day')
|
||||
GROUP BY day
|
||||
ORDER BY day`, days)
|
||||
if err != nil {
|
||||
return JudgeUsage{}, err
|
||||
}
|
||||
defer rows.Close()
|
||||
u.Daily = []JudgeDayUsage{}
|
||||
for rows.Next() {
|
||||
var day JudgeDayUsage
|
||||
if err := rows.Scan(&day.Date, &day.Calls, &day.InputTokens, &day.OutputTokens); err != nil {
|
||||
return JudgeUsage{}, err
|
||||
}
|
||||
u.Daily = append(u.Daily, day)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return JudgeUsage{}, err
|
||||
}
|
||||
return u, nil
|
||||
}
|
||||
|
||||
// ParseExpID turns the exploration-id segment parsed from a session string into an
|
||||
// int64 (0 when empty/non-numeric, e.g. chat sessions keyed by conversation id).
|
||||
func ParseExpID(s string) int64 {
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
)
|
||||
|
||||
type taskIDContextKey struct{}
|
||||
type workerContextKey struct{}
|
||||
|
||||
// WithTaskID attaches the owning task registry id to an LLM call. Session ids
|
||||
// are based on exploration ids, which are not interchangeable with task ids.
|
||||
@@ -37,6 +38,28 @@ func TaskIDFrom(ctx context.Context) string {
|
||||
return strings.TrimSpace(taskID)
|
||||
}
|
||||
|
||||
// WithWorker overrides the agent-lane ("worker") label for an LLM call. The lane
|
||||
// is normally parsed from the transcript session id (exp<N>-<role>); calls made
|
||||
// outside the engine's worker/planner sessions — e.g. the intercept fallback judge
|
||||
// — carry no such session, so they attach their lane explicitly here. This lets the
|
||||
// usage ledger single out that spend (worker='judge') for the config page.
|
||||
func WithWorker(ctx context.Context, worker string) context.Context {
|
||||
worker = strings.TrimSpace(worker)
|
||||
if worker == "" {
|
||||
return ctx
|
||||
}
|
||||
return context.WithValue(ctx, workerContextKey{}, worker)
|
||||
}
|
||||
|
||||
// workerFrom returns the explicit lane override, or "" when none is set.
|
||||
func workerFrom(ctx context.Context) string {
|
||||
if ctx == nil {
|
||||
return ""
|
||||
}
|
||||
worker, _ := ctx.Value(workerContextKey{}).(string)
|
||||
return strings.TrimSpace(worker)
|
||||
}
|
||||
|
||||
// Recorder wraps an llm.Provider and records every completion call.
|
||||
type Recorder struct {
|
||||
inner llm.Provider
|
||||
@@ -101,6 +124,9 @@ func (r *Recorder) Stream(ctx context.Context, req llm.CompletionRequest) iter.S
|
||||
start := time.Now()
|
||||
session := transcript.SessionIDFrom(ctx)
|
||||
parsedID, worker := parseSession(session)
|
||||
if ov := workerFrom(ctx); ov != "" {
|
||||
worker = ov
|
||||
}
|
||||
expID := db.ParseExpID(parsedID)
|
||||
taskID := TaskIDFrom(ctx)
|
||||
if taskID == "" {
|
||||
@@ -198,6 +224,9 @@ func (r *Recorder) Complete(ctx context.Context, req llm.CompletionRequest) (llm
|
||||
start := time.Now()
|
||||
session := transcript.SessionIDFrom(ctx)
|
||||
parsedID, worker := parseSession(session)
|
||||
if ov := workerFrom(ctx); ov != "" {
|
||||
worker = ov
|
||||
}
|
||||
expID := db.ParseExpID(parsedID)
|
||||
taskID := TaskIDFrom(ctx)
|
||||
if taskID == "" {
|
||||
|
||||
@@ -14,9 +14,14 @@ import (
|
||||
"github.com/Autumn-27/artex/db"
|
||||
"github.com/Autumn-27/artex/guard"
|
||||
"github.com/Autumn-27/artex/intercept"
|
||||
"github.com/Autumn-27/artex/llmrec"
|
||||
"github.com/Autumn-27/norma/llm"
|
||||
)
|
||||
|
||||
// judgeWorkerLane is the usage-ledger "worker" label for intercept fallback-judge
|
||||
// calls, so judge spend can be queried apart from the worker/planner/main lanes.
|
||||
const judgeWorkerLane = "judge"
|
||||
|
||||
// chatGuard returns a guard wired with the manager's interceptor, used for chat
|
||||
// conversations. Called once per applyLLM so a new LLM config always gets a fresh guard.
|
||||
func (s *Server) chatGuard() *guard.Guard {
|
||||
@@ -41,6 +46,9 @@ func (s *Server) wireInterceptReviewer() {
|
||||
if !ok {
|
||||
return intercept.Decision{ProfileID: profileID}, fmt.Errorf("裁判模型 profile %d 不可用", profileID)
|
||||
}
|
||||
// Tag this call's usage as the "judge" lane so the config page can report
|
||||
// how much the fallback approval has spent, separate from model profiles.
|
||||
ctx = llmrec.WithWorker(ctx, judgeWorkerLane)
|
||||
text, err := reviewCompletion(ctx, prov, prompt, input)
|
||||
if err != nil {
|
||||
return intercept.Decision{ProfileID: profileID}, err
|
||||
@@ -437,6 +445,23 @@ func (s *Server) interceptSetJudgeConfig(w http.ResponseWriter, r *http.Request)
|
||||
writeJSON(w, 200, map[string]any{"ok": true})
|
||||
}
|
||||
|
||||
// interceptJudgeUsage returns the fallback judge's cumulative token spend plus a
|
||||
// recent daily series, for the config page. ?days bounds the daily series (default 30).
|
||||
func (s *Server) interceptJudgeUsage(w http.ResponseWriter, r *http.Request) {
|
||||
pg := s.m.PG()
|
||||
if pg == nil {
|
||||
writeJSON(w, 200, db.JudgeUsage{Daily: []db.JudgeDayUsage{}})
|
||||
return
|
||||
}
|
||||
days := atoiDefault(r.URL.Query().Get("days"), 30)
|
||||
usage, err := pg.JudgeUsageStats(days)
|
||||
if err != nil {
|
||||
writeErr(w, 500, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, usage)
|
||||
}
|
||||
|
||||
// --- helpers ---
|
||||
|
||||
type interceptRuleReq struct {
|
||||
|
||||
@@ -930,6 +930,7 @@ func (s *Server) Handler() http.Handler {
|
||||
mux.HandleFunc("PUT /api/intercept/tool-config", s.interceptSetToolConfig)
|
||||
mux.HandleFunc("GET /api/intercept/judge", s.interceptGetJudgeConfig)
|
||||
mux.HandleFunc("PUT /api/intercept/judge", s.interceptSetJudgeConfig)
|
||||
mux.HandleFunc("GET /api/intercept/judge/usage", s.interceptJudgeUsage) // 兜底审批累计 token 用量
|
||||
|
||||
// /api/* goes through CORS + JWT; everything else is served by the embedded
|
||||
// frontend (public — auth is enforced client-side and on the API). With the
|
||||
|
||||
@@ -51,7 +51,53 @@ import {
|
||||
import { Card, CardContent } from "@/components/ui/card";
|
||||
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
|
||||
import { api } from "@/lib/api";
|
||||
import type { InterceptRule, InterceptAction, JudgeConfig, LLMProfile, Tool } from "@/lib/types";
|
||||
import type {
|
||||
InterceptAction,
|
||||
InterceptRule,
|
||||
JudgeConfig,
|
||||
JudgeDayUsage,
|
||||
JudgeUsage,
|
||||
LLMProfile,
|
||||
Tool,
|
||||
} from "@/lib/types";
|
||||
|
||||
// fmtTokens 把 token 数压成紧凑写法(1.2k / 3.4M),用于审批用量统计。
|
||||
function fmtTokens(n: number): string {
|
||||
if (n >= 1_000_000) return (n / 1_000_000).toFixed(1) + "M";
|
||||
if (n >= 1000) return (n / 1000).toFixed(1) + "k";
|
||||
return String(n);
|
||||
}
|
||||
|
||||
// JudgeStat 是一块统计数字(标签 + 数值)。
|
||||
function JudgeStat({ label, value }: { label: string; value: string }) {
|
||||
return (
|
||||
<div className="rounded-lg border bg-muted/20 px-3 py-2">
|
||||
<div className="text-[10px] text-muted-foreground">{label}</div>
|
||||
<div className="mt-0.5 text-lg font-semibold tabular-nums">{value}</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// JudgeSparkbars 用纯 div 画近 N 天每日消耗(输入+输出)的迷你柱图,免图表库依赖。
|
||||
function JudgeSparkbars({ daily }: { daily: JudgeDayUsage[] }) {
|
||||
const max = Math.max(1, ...daily.map((d) => d.input_tokens + d.output_tokens));
|
||||
return (
|
||||
<div className="flex h-16 items-end gap-0.5">
|
||||
{daily.map((d) => {
|
||||
const total = d.input_tokens + d.output_tokens;
|
||||
const h = Math.max(2, Math.round((total / max) * 100));
|
||||
return (
|
||||
<div
|
||||
key={d.date}
|
||||
title={`${d.date} · ${d.calls} 次 · ${fmtTokens(total)} tokens`}
|
||||
className="min-w-[2px] flex-1 rounded-sm bg-violet-500/60 hover:bg-violet-500"
|
||||
style={{ height: `${h}%` }}
|
||||
/>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// ---- tool scope ----
|
||||
|
||||
@@ -164,6 +210,16 @@ function JudgeCard() {
|
||||
const [profiles, setProfiles] = React.useState<LLMProfile[]>([]);
|
||||
const [loading, setLoading] = React.useState(true);
|
||||
const [saving, setSaving] = React.useState(false);
|
||||
const [usage, setUsage] = React.useState<JudgeUsage | null>(null);
|
||||
|
||||
// 审批用量统计:失败不打断配置页,仅在开启时拉取。
|
||||
const loadUsage = React.useCallback(async () => {
|
||||
try {
|
||||
setUsage(await api.interceptJudgeUsage(30));
|
||||
} catch {
|
||||
// 忽略:统计不可用不应影响配置编辑
|
||||
}
|
||||
}, []);
|
||||
|
||||
const load = React.useCallback(async () => {
|
||||
setLoading(true);
|
||||
@@ -182,6 +238,11 @@ function JudgeCard() {
|
||||
load();
|
||||
}, [load]);
|
||||
|
||||
// 开启后(含初次加载把开关读为 true 时)拉取审批用量统计。
|
||||
React.useEffect(() => {
|
||||
if (cfg.enabled) loadUsage();
|
||||
}, [cfg.enabled, loadUsage]);
|
||||
|
||||
function patch(p: Partial<JudgeConfig>) {
|
||||
setCfg((c) => ({ ...c, ...p }));
|
||||
}
|
||||
@@ -237,6 +298,40 @@ function JudgeCard() {
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* 审批 Token 用量统计(全局累计,独立于各模型配置) */}
|
||||
{cfg.enabled && usage && (
|
||||
<Card>
|
||||
<CardContent className="p-4">
|
||||
<div className="mb-3 flex items-center justify-between">
|
||||
<div>
|
||||
<p className="text-sm font-medium">审批 Token 消耗</p>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
模型兜底审批累计用量,单独计量(worker=judge),不与各模型配置的统计混在一起
|
||||
</p>
|
||||
</div>
|
||||
<Button variant="ghost" size="sm" className="h-7 text-xs" onClick={loadUsage}>
|
||||
刷新
|
||||
</Button>
|
||||
</div>
|
||||
<div className="grid grid-cols-2 gap-3 sm:grid-cols-5">
|
||||
<JudgeStat label="审批调用" value={usage.calls.toLocaleString()} />
|
||||
<JudgeStat label="输入 Token" value={fmtTokens(usage.input_tokens)} />
|
||||
<JudgeStat label="输出 Token" value={fmtTokens(usage.output_tokens)} />
|
||||
<JudgeStat label="缓存读" value={fmtTokens(usage.cache_read_tokens)} />
|
||||
<JudgeStat label="缓存写" value={fmtTokens(usage.cache_write_tokens)} />
|
||||
</div>
|
||||
{usage.daily.length > 0 && (
|
||||
<div className="mt-4">
|
||||
<p className="mb-2 text-[10px] uppercase tracking-wider text-muted-foreground">
|
||||
近 30 天每日消耗(输入 + 输出)
|
||||
</p>
|
||||
<JudgeSparkbars daily={usage.daily} />
|
||||
</div>
|
||||
)}
|
||||
</CardContent>
|
||||
</Card>
|
||||
)}
|
||||
|
||||
{cfg.enabled && (
|
||||
<div className="grid gap-4 lg:grid-cols-5">
|
||||
{/* 左:提示词编辑器(直接展开,主区域) */}
|
||||
|
||||
@@ -53,6 +53,7 @@ import type {
|
||||
InterceptPending,
|
||||
InterceptRule,
|
||||
JudgeConfig,
|
||||
JudgeUsage,
|
||||
LLMPoolStatus,
|
||||
LLMProfile,
|
||||
LLMRecordDetail,
|
||||
@@ -1265,6 +1266,7 @@ export const api = {
|
||||
// ---- intercept LLM judge (模型兜底审批,全局配置) ----
|
||||
interceptGetJudgeConfig: () => get<JudgeConfig>("/intercept/judge"),
|
||||
interceptSetJudgeConfig: (cfg: JudgeConfig) => put<{ ok: boolean }>("/intercept/judge", cfg),
|
||||
interceptJudgeUsage: (days = 30) => get<JudgeUsage>(`/intercept/judge/usage?days=${days}`),
|
||||
|
||||
// ---- commands (tool execution history, any tool) ----
|
||||
commands: (params?: { task?: string; q?: string; page?: number; size?: number }) => {
|
||||
|
||||
@@ -2464,6 +2464,15 @@ function route(m: string, path: string, seg: string[], q: URLSearchParams, b: Re
|
||||
ask_timeout_action: "deny",
|
||||
};
|
||||
if (path === "/intercept/judge" && m === "PUT") return { ok: true };
|
||||
if (path === "/intercept/judge/usage" && m === "GET")
|
||||
return {
|
||||
calls: 0,
|
||||
input_tokens: 0,
|
||||
output_tokens: 0,
|
||||
cache_read_tokens: 0,
|
||||
cache_write_tokens: 0,
|
||||
daily: [],
|
||||
};
|
||||
|
||||
// ── 旁路提问(/btw):demo 无旁路会话 ──
|
||||
// 必须显式命中:路径以 s 结尾会被下面的读兜底判成集合返回 [],items 就成了 undefined。
|
||||
|
||||
@@ -1332,6 +1332,22 @@ export interface JudgeConfig {
|
||||
ask_timeout_action: "allow" | "deny"; // 审批超时后的默认动作
|
||||
}
|
||||
|
||||
// JudgeUsage: 模型兜底审批(judge 通道)的累计 token 用量 + 近 N 天每日序列。
|
||||
export interface JudgeDayUsage {
|
||||
date: string; // YYYY-MM-DD (UTC)
|
||||
calls: number;
|
||||
input_tokens: number;
|
||||
output_tokens: number;
|
||||
}
|
||||
export interface JudgeUsage {
|
||||
calls: number;
|
||||
input_tokens: number;
|
||||
output_tokens: number;
|
||||
cache_read_tokens: number;
|
||||
cache_write_tokens: number;
|
||||
daily: JudgeDayUsage[];
|
||||
}
|
||||
|
||||
export interface InterceptApprovalFilter {
|
||||
status?: InterceptPending["status"];
|
||||
decision_source?: "rule" | "model" | "unknown";
|
||||
|
||||
Reference in new issue
Block a user