diff --git a/db/skill_usage.go b/db/skill_usage.go new file mode 100644 index 0000000..61d3a63 --- /dev/null +++ b/db/skill_usage.go @@ -0,0 +1,200 @@ +package db + +import ( + "database/sql" + "strings" + "time" +) + +// SkillUsage is one Skill() invocation — the always-on skill call ledger. Written +// from the Skill meta-tool's OnInvoke hook (server/assembly.go), one row per load. +// Carries only dimensions (which skill, which agent, which task/session), never the +// caller's args text — args_len is kept so an "empty vs substantial context" split +// is still possible without storing prompt content, mirroring llm_usage. +// +// Rows deliberately outlive their task: skill_usage has no foreign keys, so deleting +// a task keeps its skill statistics intact (same rationale as llm_usage). +type SkillUsage struct { + Skill string `json:"skill"` // skill directory name (matches agent_skill_visibility.skill_name) + AgentKey string `json:"agent_key"` // worker / planner / mainagent / custom agent key + TaskID int64 `json:"task_id"` // 0 for non-task runs (chat sessions) + ExplorationID int64 `json:"exploration_id"` // 0 when unknown + IntentID int64 `json:"intent_id"` // worker's intent node; 0 for planner/mainagent/chat + SessionID string `json:"session_id"` // chat conversation id; empty for task runs + ArgsLen int `json:"args_len"` + Found bool `json:"found"` // false = the model named a skill that does not exist +} + +// InsertSkillUsage appends one ledger row. Best-effort: callers log and continue on +// error (a lost metering row must never break a skill invocation). +func (d *DB) InsertSkillUsage(u *SkillUsage) error { + _, err := d.Exec(` +INSERT INTO skill_usage(skill, agent_key, task_id, exploration_id, intent_id, session_id, args_len, found) +VALUES ($1, NULLIF($2,''), $3, $4, $5, NULLIF($6,''), $7, $8)`, + u.Skill, u.AgentKey, nullIfZero(u.TaskID), nullIfZero(u.ExplorationID), + nullIfZero(u.IntentID), u.SessionID, u.ArgsLen, u.Found) + return err +} + +func nullIfZero(v int64) any { + if v > 0 { + return v + } + return nil +} + +// SkillStat is one skill's aggregate usage, for the skills page. +type SkillStat struct { + Skill string `json:"skill"` + Calls int `json:"calls"` + Tasks int `json:"tasks"` // distinct tasks that loaded it (chat runs excluded) + Agents []string `json:"agents"` // agent keys that loaded it, most-used first + LastUsed *time.Time `json:"last_used"` // nil when never called +} + +// SkillStats aggregates the whole ledger grouped by skill, most-used first. Skills +// that were never invoked are absent — callers merge against the skill list on disk. +// Only resolved calls count; misses are reported separately by MissingSkillStats. +func (d *DB) SkillStats() ([]SkillStat, error) { + // agent keys come back as one comma-joined string rather than text[]: the pgx + // stdlib driver has no database/sql Scan target for arrays, and agent keys are + // [a-z0-9_-] so a comma join is unambiguous. + rows, err := d.Query(` +SELECT skill, COUNT(*) AS calls, + COUNT(DISTINCT task_id) AS tasks, + COALESCE(STRING_AGG(DISTINCT agent_key, ','), '') AS agents, + MAX(ts) AS last_used +FROM skill_usage +WHERE found +GROUP BY skill +ORDER BY COUNT(*) DESC, skill`) + if err != nil { + return nil, err + } + defer rows.Close() + out := []SkillStat{} + for rows.Next() { + var s SkillStat + var agents string + var lastUsed sql.NullTime + if err := rows.Scan(&s.Skill, &s.Calls, &s.Tasks, &agents, &lastUsed); err != nil { + return nil, err + } + s.Agents = []string{} + if agents != "" { + s.Agents = strings.Split(agents, ",") + } + if lastUsed.Valid { + t := lastUsed.Time + s.LastUsed = &t + } + out = append(out, s) + } + return out, rows.Err() +} + +// MissingSkillStats returns the skill names agents asked for that do not exist, +// most-requested first — the "wished it existed" gap list. Names come from the model +// so they are shown as-is (already length-capped at insert time). +func (d *DB) MissingSkillStats(limit int) ([]SkillStat, error) { + if limit <= 0 || limit > 200 { + limit = 20 + } + rows, err := d.Query(` +SELECT skill, COUNT(*) AS calls, + COALESCE(STRING_AGG(DISTINCT agent_key, ','), '') AS agents, + MAX(ts) AS last_used +FROM skill_usage +WHERE NOT found +GROUP BY skill +ORDER BY COUNT(*) DESC, skill +LIMIT $1`, limit) + if err != nil { + return nil, err + } + defer rows.Close() + out := []SkillStat{} + for rows.Next() { + var s SkillStat + var agents string + var lastUsed sql.NullTime + if err := rows.Scan(&s.Skill, &s.Calls, &agents, &lastUsed); err != nil { + return nil, err + } + s.Agents = []string{} + if agents != "" { + s.Agents = strings.Split(agents, ",") + } + if lastUsed.Valid { + t := lastUsed.Time + s.LastUsed = &t + } + out = append(out, s) + } + return out, rows.Err() +} + +// SkillCall is one row of a skill's recent-call list (detail panel). +type SkillCall struct { + TS time.Time `json:"ts"` + AgentKey string `json:"agent_key"` + TaskID int64 `json:"task_id"` + SessionID string `json:"session_id"` + ArgsLen int `json:"args_len"` +} + +// RecentSkillCalls returns the most recent invocations of one skill, newest first. +func (d *DB) RecentSkillCalls(skill string, limit int) ([]SkillCall, error) { + if limit <= 0 || limit > 200 { + limit = 50 + } + rows, err := d.Query(` +SELECT ts, COALESCE(agent_key,''), COALESCE(task_id,0), COALESCE(session_id,''), args_len +FROM skill_usage +WHERE skill = $1 AND found +ORDER BY ts DESC +LIMIT $2`, skill, limit) + if err != nil { + return nil, err + } + defer rows.Close() + out := []SkillCall{} + for rows.Next() { + var c SkillCall + if err := rows.Scan(&c.TS, &c.AgentKey, &c.TaskID, &c.SessionID, &c.ArgsLen); err != nil { + return nil, err + } + out = append(out, c) + } + return out, rows.Err() +} + +// SkillCallsByTask counts a task's skill loads, most-used first. Powers a per-task +// view of which procedures its agents actually reached for. +func (d *DB) SkillCallsByTask(taskID int64) ([]SkillStat, error) { + rows, err := d.Query(` +SELECT skill, COUNT(*) AS calls, MAX(ts) AS last_used +FROM skill_usage +WHERE task_id = $1 AND found +GROUP BY skill +ORDER BY COUNT(*) DESC, skill`, taskID) + if err != nil { + return nil, err + } + defer rows.Close() + out := []SkillStat{} + for rows.Next() { + var s SkillStat + var lastUsed sql.NullTime + if err := rows.Scan(&s.Skill, &s.Calls, &lastUsed); err != nil { + return nil, err + } + s.Agents = []string{} + if lastUsed.Valid { + t := lastUsed.Time + s.LastUsed = &t + } + out = append(out, s) + } + return out, rows.Err() +} diff --git a/db/skill_usage_test.go b/db/skill_usage_test.go new file mode 100644 index 0000000..4625696 --- /dev/null +++ b/db/skill_usage_test.go @@ -0,0 +1,116 @@ +package db + +import ( + "testing" +) + +// TestSkillUsageLedger writes a few rows and checks the three aggregates the +// skills page relies on: per-skill totals (resolved calls only), the miss list, +// and the recent-call list. Rows are namespaced by a unique skill name so the +// shared dev DB stays usable and the test cleans up after itself. +func TestSkillUsageLedger(t *testing.T) { + d, err := Open(testDSN(t)) + if err != nil { + t.Skipf("postgres unavailable (%v) — skipping", err) + } + defer d.Close() + + const ( + skillA = "zz-test-skill-a" + missing = "zz-test-skill-missing" + ) + cleanup := func() { + _, _ = d.Exec(`DELETE FROM skill_usage WHERE skill IN ($1,$2)`, skillA, missing) + } + cleanup() + defer cleanup() + + rows := []*SkillUsage{ + {Skill: skillA, AgentKey: "worker", TaskID: 991, ExplorationID: 5, IntentID: 7, ArgsLen: 12, Found: true}, + {Skill: skillA, AgentKey: "worker", TaskID: 991, ExplorationID: 5, ArgsLen: 0, Found: true}, + {Skill: skillA, AgentKey: "planner", TaskID: 992, ExplorationID: 6, Found: true}, + {Skill: skillA, AgentKey: "chatbot", SessionID: "conv-1", Found: true}, + {Skill: missing, AgentKey: "worker", TaskID: 991, Found: false}, + {Skill: missing, AgentKey: "worker", TaskID: 991, Found: false}, + } + for _, r := range rows { + if err := d.InsertSkillUsage(r); err != nil { + t.Fatalf("insert %s: %v", r.Skill, err) + } + } + + stats, err := d.SkillStats() + if err != nil { + t.Fatal(err) + } + var got *SkillStat + for i := range stats { + if stats[i].Skill == skillA { + got = &stats[i] + } + if stats[i].Skill == missing { + t.Fatalf("misses must not appear in SkillStats: %+v", stats[i]) + } + } + if got == nil { + t.Fatalf("skill %s absent from SkillStats", skillA) + } + if got.Calls != 4 { + t.Errorf("calls: want 4, got %d", got.Calls) + } + // 991 + 992; the chat row has a NULL task_id and COUNT(DISTINCT) skips NULLs. + if got.Tasks != 2 { + t.Errorf("tasks: want 2, got %d", got.Tasks) + } + if len(got.Agents) != 3 { + t.Errorf("agents: want 3 distinct, got %v", got.Agents) + } + if got.LastUsed == nil { + t.Error("last_used must be set") + } + + miss, err := d.MissingSkillStats(10) + if err != nil { + t.Fatal(err) + } + var missCalls int + for _, m := range miss { + if m.Skill == missing { + missCalls = m.Calls + } + } + if missCalls != 2 { + t.Errorf("missing calls: want 2, got %d", missCalls) + } + + calls, err := d.RecentSkillCalls(skillA, 10) + if err != nil { + t.Fatal(err) + } + if len(calls) != 4 { + t.Fatalf("recent calls: want 4, got %d", len(calls)) + } + // newest first + for i := 1; i < len(calls); i++ { + if calls[i].TS.After(calls[i-1].TS) { + t.Errorf("recent calls not newest-first at %d", i) + } + } + + byTask, err := d.SkillCallsByTask(991) + if err != nil { + t.Fatal(err) + } + var taskCalls int + for _, s := range byTask { + if s.Skill == skillA { + taskCalls = s.Calls + } + if s.Skill == missing { + t.Errorf("misses must not appear in SkillCallsByTask: %+v", s) + } + } + if taskCalls != 2 { + t.Errorf("task 991 calls: want 2, got %d", taskCalls) + } +} diff --git a/server/assembly.go b/server/assembly.go index 49fc162..426a9b6 100644 --- a/server/assembly.go +++ b/server/assembly.go @@ -137,7 +137,11 @@ func wireAgentAugment(pg *db.DB, skillDir string, hostTools func() ([]actool.Cor } return actool.RenderDeferredToolsBlock(reveal) } - extra = append(extra, reg.Tool()) + // Attribution for the usage ledger: ToolAugment only gets (ctx, agentKey), + // so the run's task/session ids ride in on the ctx (agent.RunInfo). Read it + // once here — this closure is rebuilt per run, so the captured value always + // belongs to this run. + extra = append(extra, meterSkillTool(reg.Tool(), pg, reg, a.Key, agent.RunInfoFrom(ctx))) } // host tools (traffic / orchestration / custom) — added to every agent's base; diff --git a/server/skill_usage.go b/server/skill_usage.go new file mode 100644 index 0000000..4d0906e --- /dev/null +++ b/server/skill_usage.go @@ -0,0 +1,88 @@ +package server + +import ( + "context" + "encoding/json" + "log" + "path/filepath" + "strings" + + "github.com/Autumn-27/norma/skill" + actool "github.com/Autumn-27/norma/tool" + + "github.com/Autumn-27/artex/agent" + "github.com/Autumn-27/artex/db" +) + +// maxLedgerSkillName caps the skill name stored for a MISS — that string comes +// straight from the model and is otherwise unbounded. +const maxLedgerSkillName = 128 + +// meteredSkill wraps the Skill meta-tool to append one skill_usage row per call. +// The metering lives here rather than in Registry.OnInvoke because OnInvoke only +// receives the resolved Skill: it never fires for an unknown skill name (norma +// returns a tool error before the hook) and it cannot see the caller's args. Both +// matter — a miss tells you which procedure the agents wished existed. +type meteredSkill struct { + actool.CoreTool + pg *db.DB + reg *skill.Registry + agentKey string + ri agent.RunInfo +} + +// meterSkillTool returns t unchanged when there is no DB to meter into. +func meterSkillTool(t actool.CoreTool, pg *db.DB, reg *skill.Registry, agentKey string, ri agent.RunInfo) actool.CoreTool { + if pg == nil { + return t + } + return &meteredSkill{CoreTool: t, pg: pg, reg: reg, agentKey: agentKey, ri: ri} +} + +func (m *meteredSkill) Call(ctx context.Context, input json.RawMessage, tc *actool.ToolContext) (actool.Result, error) { + m.record(input) + return m.CoreTool.Call(ctx, input, tc) +} + +// record appends the ledger row. Best-effort by design: a metering failure must +// never turn into a failed skill invocation, so every error path just logs. +func (m *meteredSkill) record(input json.RawMessage) { + var in struct { + Name string `json:"name"` + Args string `json:"args"` + } + if err := json.Unmarshal(input, &in); err != nil { + return // malformed call; norma will reject it too + } + name := strings.TrimSpace(in.Name) + if name == "" { + return + } + // Resolve to the directory name so the ledger keys match agent_skill_visibility + // and the skills page (a skill's display Name can differ from its directory). + found := false + if m.reg != nil { + if s, ok := m.reg.Get(name); ok { + found = true + if s.Dir != "" { + name = filepath.Base(s.Dir) + } + } + } + if len(name) > maxLedgerSkillName { + name = name[:maxLedgerSkillName] + } + err := m.pg.InsertSkillUsage(&db.SkillUsage{ + Skill: name, + AgentKey: m.agentKey, + TaskID: m.ri.TaskID, + ExplorationID: m.ri.ExplorationID, + IntentID: m.ri.IntentID, + SessionID: m.ri.SessionID, + ArgsLen: len(in.Args), + Found: found, + }) + if err != nil { + log.Printf("[skillusage] insert: %v", err) + } +} diff --git a/web/src/app/(main)/system/skills/page.tsx b/web/src/app/(main)/system/skills/page.tsx index ab6366d..41539df 100644 --- a/web/src/app/(main)/system/skills/page.tsx +++ b/web/src/app/(main)/system/skills/page.tsx @@ -12,13 +12,25 @@ import { PlusIcon, Trash2Icon, UploadIcon, + AlertTriangleIcon, } from "lucide-react"; +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from "@/components/ui/alert-dialog"; import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; import { Textarea } from "@/components/ui/textarea"; import { Checkbox } from "@/components/ui/checkbox"; +import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; import { ScrollArea } from "@/components/ui/scroll-area"; import { Badge } from "@/components/ui/badge"; import { @@ -31,7 +43,17 @@ import { import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { cn } from "@/lib/utils"; import { api } from "@/lib/api"; -import type { Agent, SkillItem, MCPServer } from "@/lib/types"; +import type { Agent, SkillItem, MCPServer, SkillCall, MissingSkill } from "@/lib/types"; + +function fmtTime(ts?: string) { + if (!ts) return "从未调用"; + return new Date(ts).toLocaleString("zh-CN", { + month: "2-digit", + day: "2-digit", + hour: "2-digit", + minute: "2-digit", + }); +} // ── Tree node ────────────────────────────────────────────────────────────── interface TreeNode { @@ -85,9 +107,22 @@ function buildTree(entries: string[]): TreeNode[] { if (fname) nodes.push({ name: fname, path: entry, type: "file", children: [] }); } } + sortNodes(root); return root; } +// sortNodes orders every level like a file explorer: directories before files, +// then case-insensitive by name. Recurses into children. +function sortNodes(nodes: TreeNode[]): void { + nodes.sort((a, b) => { + if (a.type !== b.type) return a.type === "dir" ? -1 : 1; + return a.name.localeCompare(b.name, undefined, { sensitivity: "base" }); + }); + for (const n of nodes) { + if (n.children.length > 0) sortNodes(n.children); + } +} + // ── State types ─────────────────────────────────────────────────────────── type Selected = | { skill: string; path: null } @@ -99,6 +134,164 @@ type Creating = { kind: "file" | "dir"; } | null; +type PendingDelete = + | { kind: "skill"; skill: string } + | { kind: "file"; skill: string; path: string } + | { kind: "dir"; skill: string; path: string } + | null; + +// ── Overview (empty-state) ────────────────────────────────────────────────── +// Shown when nothing is selected: a library-wide snapshot from data already loaded +// (the skill list carries per-skill usage; missing is fetched alongside). No extra +// requests — this is pure aggregation over props. +function SkillsOverview({ + skills, + missing, + onSelect, +}: { + skills: SkillItem[]; + missing: MissingSkill[]; + onSelect: (name: string) => void; +}) { + const agg = React.useMemo(() => { + const totalCalls = skills.reduce((n, s) => n + s.calls, 0); + const used = skills.filter((s) => s.calls > 0); + const ranked = [...used].sort((a, b) => b.calls - a.calls); + const neverUsed = skills.filter((s) => s.calls === 0); + const recent = skills + .filter((s) => s.last_used) + .sort((a, b) => (a.last_used! < b.last_used! ? 1 : -1)) + .slice(0, 6); + const missingCalls = missing.reduce((n, m) => n + m.calls, 0); + return { + totalCalls, + usedCount: used.length, + ranked, + topCalls: ranked[0]?.calls ?? 0, + neverUsed, + recent, + missingCalls, + }; + }, [skills, missing]); + + if (skills.length === 0) { + return ( +
暂无 Skill,点击左侧「新建」或「上传压缩包」开始
+选择左侧的 Skill 查看详情与调用记录,或从这里快速了解整体使用情况。
+{s.value}
+{s.label}
+ {s.hint &&{s.hint}
} ++ 还没有任何 Skill 调用记录。 +
+ ) : ( +暂无记录。
+ ) : ( +所有 Skill 都被调用过。
+ ) : ( +技能库 · agentskills.io 规范 · 按 Agent 授权可见
+技能库 · agentskills.io 规范 · 按 Agent 授权可见
++ Agent 点名调用、但技能库里不存在的 skill。按被点名次数排序。 +
+{m.skill}
+ {m.calls} 次
+ {fmtTime(m.last_used)}
+ 选择左侧 Skill 或文件
-暂无 MCP,可在「MCP」页添加。
- ) : ( -{selectedSkill.calls}
+总调用次数
+{selectedSkill.tasks}
+覆盖任务数
++ {fmtTime(selectedSkill.last_used)} +
+最近调用
+加载调用明细…
+ ) : usageCalls.length > 0 ? ( +还没有调用记录。
+ )} +暂无 MCP,可在「MCP」页添加。
+ ) : ( +