feat(skills): track usage and missing dependencies

2026-08-24 19:30:14 +08:00
1 parent 8dca38c4b9
commit fd2ddbc9d7
@@ -0,0 +1,200 @@
+package db
+
+import (
+	"database/sql"
+	"strings"
+	"time"
+)
+
+// SkillUsage is one Skill() invocation — the always-on skill call ledger. Written
+// from the Skill meta-tool's OnInvoke hook (server/assembly.go), one row per load.
+// Carries only dimensions (which skill, which agent, which task/session), never the
+// caller's args text — args_len is kept so an "empty vs substantial context" split
+// is still possible without storing prompt content, mirroring llm_usage.
+//
+// Rows deliberately outlive their task: skill_usage has no foreign keys, so deleting
+// a task keeps its skill statistics intact (same rationale as llm_usage).
+type SkillUsage struct {
+	Skill         string `json:"skill"`          // skill directory name (matches agent_skill_visibility.skill_name)
+	AgentKey      string `json:"agent_key"`      // worker / planner / mainagent / custom agent key
+	TaskID        int64  `json:"task_id"`        // 0 for non-task runs (chat sessions)
+	ExplorationID int64  `json:"exploration_id"` // 0 when unknown
+	IntentID      int64  `json:"intent_id"`      // worker's intent node; 0 for planner/mainagent/chat
+	SessionID     string `json:"session_id"`     // chat conversation id; empty for task runs
+	ArgsLen       int    `json:"args_len"`
+	Found         bool   `json:"found"` // false = the model named a skill that does not exist
+}
+
+// InsertSkillUsage appends one ledger row. Best-effort: callers log and continue on
+// error (a lost metering row must never break a skill invocation).
+func (d *DB) InsertSkillUsage(u *SkillUsage) error {
+	_, err := d.Exec(`
+INSERT INTO skill_usage(skill, agent_key, task_id, exploration_id, intent_id, session_id, args_len, found)
+VALUES ($1, NULLIF($2,''), $3, $4, $5, NULLIF($6,''), $7, $8)`,
+		u.Skill, u.AgentKey, nullIfZero(u.TaskID), nullIfZero(u.ExplorationID),
+		nullIfZero(u.IntentID), u.SessionID, u.ArgsLen, u.Found)
+	return err
+}
+
+func nullIfZero(v int64) any {
+	if v > 0 {
+		return v
+	}
+	return nil
+}
+
+// SkillStat is one skill's aggregate usage, for the skills page.
+type SkillStat struct {
+	Skill    string     `json:"skill"`
+	Calls    int        `json:"calls"`
+	Tasks    int        `json:"tasks"`     // distinct tasks that loaded it (chat runs excluded)
+	Agents   []string   `json:"agents"`    // agent keys that loaded it, most-used first
+	LastUsed *time.Time `json:"last_used"` // nil when never called
+}
+
+// SkillStats aggregates the whole ledger grouped by skill, most-used first. Skills
+// that were never invoked are absent — callers merge against the skill list on disk.
+// Only resolved calls count; misses are reported separately by MissingSkillStats.
+func (d *DB) SkillStats() ([]SkillStat, error) {
+	// agent keys come back as one comma-joined string rather than text[]: the pgx
+	// stdlib driver has no database/sql Scan target for arrays, and agent keys are
+	// [a-z0-9_-] so a comma join is unambiguous.
+	rows, err := d.Query(`
+SELECT skill, COUNT(*) AS calls,
+       COUNT(DISTINCT task_id) AS tasks,
+       COALESCE(STRING_AGG(DISTINCT agent_key, ','), '') AS agents,
+       MAX(ts) AS last_used
+FROM skill_usage
+WHERE found
+GROUP BY skill
+ORDER BY COUNT(*) DESC, skill`)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	out := []SkillStat{}
+	for rows.Next() {
+		var s SkillStat
+		var agents string
+		var lastUsed sql.NullTime
+		if err := rows.Scan(&s.Skill, &s.Calls, &s.Tasks, &agents, &lastUsed); err != nil {
+			return nil, err
+		}
+		s.Agents = []string{}
+		if agents != "" {
+			s.Agents = strings.Split(agents, ",")
+		}
+		if lastUsed.Valid {
+			t := lastUsed.Time
+			s.LastUsed = &t
+		}
+		out = append(out, s)
+	}
+	return out, rows.Err()
+}
+
+// MissingSkillStats returns the skill names agents asked for that do not exist,
+// most-requested first — the "wished it existed" gap list. Names come from the model
+// so they are shown as-is (already length-capped at insert time).
+func (d *DB) MissingSkillStats(limit int) ([]SkillStat, error) {
+	if limit <= 0 || limit > 200 {
+		limit = 20
+	}
+	rows, err := d.Query(`
+SELECT skill, COUNT(*) AS calls,
+       COALESCE(STRING_AGG(DISTINCT agent_key, ','), '') AS agents,
+       MAX(ts) AS last_used
+FROM skill_usage
+WHERE NOT found
+GROUP BY skill
+ORDER BY COUNT(*) DESC, skill
+LIMIT $1`, limit)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	out := []SkillStat{}
+	for rows.Next() {
+		var s SkillStat
+		var agents string
+		var lastUsed sql.NullTime
+		if err := rows.Scan(&s.Skill, &s.Calls, &agents, &lastUsed); err != nil {
+			return nil, err
+		}
+		s.Agents = []string{}
+		if agents != "" {
+			s.Agents = strings.Split(agents, ",")
+		}
+		if lastUsed.Valid {
+			t := lastUsed.Time
+			s.LastUsed = &t
+		}
+		out = append(out, s)
+	}
+	return out, rows.Err()
+}
+
+// SkillCall is one row of a skill's recent-call list (detail panel).
+type SkillCall struct {
+	TS        time.Time `json:"ts"`
+	AgentKey  string    `json:"agent_key"`
+	TaskID    int64     `json:"task_id"`
+	SessionID string    `json:"session_id"`
+	ArgsLen   int       `json:"args_len"`
+}
+
+// RecentSkillCalls returns the most recent invocations of one skill, newest first.
+func (d *DB) RecentSkillCalls(skill string, limit int) ([]SkillCall, error) {
+	if limit <= 0 || limit > 200 {
+		limit = 50
+	}
+	rows, err := d.Query(`
+SELECT ts, COALESCE(agent_key,''), COALESCE(task_id,0), COALESCE(session_id,''), args_len
+FROM skill_usage
+WHERE skill = $1 AND found
+ORDER BY ts DESC
+LIMIT $2`, skill, limit)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	out := []SkillCall{}
+	for rows.Next() {
+		var c SkillCall
+		if err := rows.Scan(&c.TS, &c.AgentKey, &c.TaskID, &c.SessionID, &c.ArgsLen); err != nil {
+			return nil, err
+		}
+		out = append(out, c)
+	}
+	return out, rows.Err()
+}
+
+// SkillCallsByTask counts a task's skill loads, most-used first. Powers a per-task
+// view of which procedures its agents actually reached for.
+func (d *DB) SkillCallsByTask(taskID int64) ([]SkillStat, error) {
+	rows, err := d.Query(`
+SELECT skill, COUNT(*) AS calls, MAX(ts) AS last_used
+FROM skill_usage
+WHERE task_id = $1 AND found
+GROUP BY skill
+ORDER BY COUNT(*) DESC, skill`, taskID)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	out := []SkillStat{}
+	for rows.Next() {
+		var s SkillStat
+		var lastUsed sql.NullTime
+		if err := rows.Scan(&s.Skill, &s.Calls, &lastUsed); err != nil {
+			return nil, err
+		}
+		s.Agents = []string{}
+		if lastUsed.Valid {
+			t := lastUsed.Time
+			s.LastUsed = &t
+		}
+		out = append(out, s)
+	}
+	return out, rows.Err()
+}
@@ -0,0 +1,116 @@
+package db
+
+import (
+	"testing"
+)
+
+// TestSkillUsageLedger writes a few rows and checks the three aggregates the
+// skills page relies on: per-skill totals (resolved calls only), the miss list,
+// and the recent-call list. Rows are namespaced by a unique skill name so the
+// shared dev DB stays usable and the test cleans up after itself.
+func TestSkillUsageLedger(t *testing.T) {
+	d, err := Open(testDSN(t))
+	if err != nil {
+		t.Skipf("postgres unavailable (%v) — skipping", err)
+	}
+	defer d.Close()
+
+	const (
+		skillA  = "zz-test-skill-a"
+		missing = "zz-test-skill-missing"
+	)
+	cleanup := func() {
+		_, _ = d.Exec(`DELETE FROM skill_usage WHERE skill IN ($1,$2)`, skillA, missing)
+	}
+	cleanup()
+	defer cleanup()
+
+	rows := []*SkillUsage{
+		{Skill: skillA, AgentKey: "worker", TaskID: 991, ExplorationID: 5, IntentID: 7, ArgsLen: 12, Found: true},
+		{Skill: skillA, AgentKey: "worker", TaskID: 991, ExplorationID: 5, ArgsLen: 0, Found: true},
+		{Skill: skillA, AgentKey: "planner", TaskID: 992, ExplorationID: 6, Found: true},
+		{Skill: skillA, AgentKey: "chatbot", SessionID: "conv-1", Found: true},
+		{Skill: missing, AgentKey: "worker", TaskID: 991, Found: false},
+		{Skill: missing, AgentKey: "worker", TaskID: 991, Found: false},
+	}
+	for _, r := range rows {
+		if err := d.InsertSkillUsage(r); err != nil {
+			t.Fatalf("insert %s: %v", r.Skill, err)
+		}
+	}
+
+	stats, err := d.SkillStats()
+	if err != nil {
+		t.Fatal(err)
+	}
+	var got *SkillStat
+	for i := range stats {
+		if stats[i].Skill == skillA {
+			got = &stats[i]
+		}
+		if stats[i].Skill == missing {
+			t.Fatalf("misses must not appear in SkillStats: %+v", stats[i])
+		}
+	}
+	if got == nil {
+		t.Fatalf("skill %s absent from SkillStats", skillA)
+	}
+	if got.Calls != 4 {
+		t.Errorf("calls: want 4, got %d", got.Calls)
+	}
+	// 991 + 992; the chat row has a NULL task_id and COUNT(DISTINCT) skips NULLs.
+	if got.Tasks != 2 {
+		t.Errorf("tasks: want 2, got %d", got.Tasks)
+	}
+	if len(got.Agents) != 3 {
+		t.Errorf("agents: want 3 distinct, got %v", got.Agents)
+	}
+	if got.LastUsed == nil {
+		t.Error("last_used must be set")
+	}
+
+	miss, err := d.MissingSkillStats(10)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var missCalls int
+	for _, m := range miss {
+		if m.Skill == missing {
+			missCalls = m.Calls
+		}
+	}
+	if missCalls != 2 {
+		t.Errorf("missing calls: want 2, got %d", missCalls)
+	}
+
+	calls, err := d.RecentSkillCalls(skillA, 10)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(calls) != 4 {
+		t.Fatalf("recent calls: want 4, got %d", len(calls))
+	}
+	// newest first
+	for i := 1; i < len(calls); i++ {
+		if calls[i].TS.After(calls[i-1].TS) {
+			t.Errorf("recent calls not newest-first at %d", i)
+		}
+	}
+
+	byTask, err := d.SkillCallsByTask(991)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var taskCalls int
+	for _, s := range byTask {
+		if s.Skill == skillA {
+			taskCalls = s.Calls
+		}
+		if s.Skill == missing {
+			t.Errorf("misses must not appear in SkillCallsByTask: %+v", s)
+		}
+	}
+	if taskCalls != 2 {
+		t.Errorf("task 991 calls: want 2, got %d", taskCalls)
+	}
+}