diff --git a/CHANGELOG.md b/CHANGELOG.md index a3fbdeb..b6e5616 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,86 @@ ## [Unreleased] +## [0.3.6] - 2026-08-27 + +### LLM + +#### 修复的问题 + +- 修复「非流式」开关保存后重新打开又变回「流式」的问题（#69）：配置列表接口的 DTO 漏了 `streaming` 字段，响应从不返回它，前端读到 `undefined` 后一律回落成默认流式；实际写入与 DB 存储本无问题，只是读不回来。DTO 补回该字段（不加 `omitempty`，`false` 也必须出现在响应里）。 +- 测试连接改为校验模型确有回复、并按该配置真实的收发模式来测（#65）：此前只判 HTTP 是否成功，请求通但模型零回复（思考烧光预算 / 正文被安全策略吞掉 / 兼容层丢 `content`）也报「连接成功」，与会话里「不回话」的表现割裂；且测试恒走流式，非流式配置测的其实是另一条通道。现在空回复直接判失败、成功时在提示里展示模型回复，并把当前流式开关一并带入测试，让「测试通过」与「会话跑得通」保持一致。 + +### Agent + +#### 修改的功能 + +- `list_facts` 改为分页 + 关键词过滤，避免事实多时单次调用撑爆上下文（#74）：默认返回最新 20 条，支持 `limit`（上限 100）/ `before` 游标 / `q` 摘要关键词，返回 `{facts, total, has_more, next_before}`；单条 summary 过长按字数截断（全文仍走 `node_detail`）。worker / planner 提示词同步为分页语义。旧库通过一次性迁移把新参数 schema 刷进工具目录表（`SeedTool` 首插入only，否则工具管理页显示「无参数」）。 + +### 工具 + +#### 新增的功能 + +- 「工具执行」页新增工具调用次数统计（#72）：工具栏「统计」按钮打开弹窗，按工具展示调用次数、占比与失败次数（次数降序）；沿用列表的任务 / 关键词筛选，统计整个结果集而非当前页，弹窗打开时才拉取。 + +### 依赖 + +#### 修改的功能 + +- 升级 norma v0.3.1 → v0.3.2：修复 OpenAI 系响应里被静默丢弃的 reasoning/refusal（Responses 补 `reasoning_text`）、reasoning 字段三名去重、空响应（零内容块）有界重试、压缩边界切断工具配对导致的网关 400（同时修复已烤进 transcript 的存量孤儿）。 + +### 贡献者 + +- [@Autumn-27](https://github.com/Autumn-27) + +## [0.3.5] - 2026-08-25 + +### LLM + +#### 新增的功能 + +- 每个 LLM 配置支持流式/非流式切换（默认流式）：开走流式 SSE；关走真·非流式（`stream:false`、一次性返回完整 JSON），可绕开部分网关糟糕的 SSE 实现（空帧、思考字段丢帧），代价是失去运行中的实时进度与实时 Token 计数。worker / planner / mainagent / chat / goals 均按当前激活配置动态取值；`llmpool` / `llmrec` / 任务运行时三层 Provider 包装均兼容非流式；`llm_profiles` 新增 `streaming` 列并 `ALTER` 补旧库（默认 `true`，旧配置无感）。 +- 支持 OpenAI Responses API 格式的 LLM 配置：每个配置新增第三种格式 `openai-responses`（打 `POST /v1/responses`），与 Chat Completions / Anthropic 并列；`BaseURL` 归一化、默认模型 `gpt-5`；`llm_profiles` 的 `format` 约束加入 `openai-responses` 并幂等迁移补旧库；前端格式下拉新增「OpenAI (Responses API)」。依赖升级 norma v0.3.1（含 `reasoning_content` 回传修复）。 +- 录制 LLM 请求/响应 HTTP 原文：在 HTTP transport 层捕获真实 wire body，保留归一化视图看不到的工具 schema、`tool_use` 块与原始 SSE 帧；norma 内部重试的多次尝试逐次保留；`llm_records` 新增 `raw_request` / `raw_response` 两列并 `ALTER` 补旧库；录制页详情面板增加「原文」视图切换与请求/响应复制按钮（兼容非安全上下文的 `execCommand` 回退）。 + +### 对话 + +#### 修改的功能 + +- 会话列表多选改为「多选」模式开关：默认列表不再常驻每行勾选框（观感更干净），表头改为「共 N 个 + 多选」按钮；点「多选」进入选择模式（勾选框、全选、批量删除），「完成」退出并清空选择，批量删除全部成功后自动退出（有失败则留在选择模式便于重试）；单条重命名 / 置顶 / 删除仍走每行 ⋯ 菜单。 + +### UI + +#### 修改的功能 + +- 任务管理、会话操作与流量查看体验增强（#57）：任务列表列排序偏好可持久化记忆、行内重命名改用图标直接触发（不再经菜单）、抽屉（sheet）交互与流量查看细节优化。 + +#### 修复的问题 + +- Worker 资产标签只显示域名与 IP，并正确处理空标签的场景。 + +### Agent + +#### 新增的功能 + +- planner 判目标时新增「量化验收核对」：目标含可量化验收条件（资产测试覆盖度达到 X%、拿到 N 个 flag、获得某权限）时，`prove_goal` 前必须核对 `graph_overview` 的实测值（`coverage.pct`、findings 计数等），实测未达标则禁止 `prove_goal`、改为派意图补足差距，不得以「主要部分已完成/大体达成」为由提前标 met。修复了目标要求覆盖度 100%、实测仅 40% 却被判完成的问题。 + +#### 修改的功能 + +- 重写 planner 提示词中「本轮 0 意图」的判定依据：原文将 0 意图描述为「最常见、最重要的原则」，会让规划者在目标未达成、范围内仍有未测面时过早收手。现改为只在两种具体情况下才应产出 0 意图——① 想到的方向都已被 open/running/recent_done 意图覆盖；② 下一步依赖当前正在运行的 work 的产出、而产出尚未出现（应等其跑完、图更新后的下一次唤醒再规划）。并补充反向约束：确有未被覆盖且不依赖在跑 work 的新方向、或目标未达成且仍有未测面时，不要因「0 意图常见」而收手。 +- 强化 worker 提示词中「否定结论的证据门槛」：对「不可注入/端口关闭/无登录入口」等可能让规划者放弃一整条方向的否定结论，要求下结论前先穷尽该意图内的合理手段（换编码/参数/路径/方法）；手段未走完或证据偏弱一律标 `confidence=inferred`，避免用轻率的 `observed` 否定把整条路线焊死（任务早期的错误否定尤其会把方向带偏且难以自愈）。 +- 上述 planner / worker 默认提示词变更通过一次性迁移（`reseedPlannerPrompt` / `reseedWorkerPrompt`，各带 settings flag 守卫）追加为新版本并切为当前版本；旧版本保留在版本历史中，自定义过提示词的用户可在版本记录里恢复。 + +### 运维 + +#### 新增的功能 + +- 新增 `reset-password.sh` 重置管理员（用户名固定 `ARTEX`）密码：支持 local / docker 两种部署，连接信息可显式指定或自动从 `--dsn`/`$ARTEX_PG_DSN`/`config.json` 读取；在库内用 `pgcrypto` 生成与后端登录兼容的 bcrypt 哈希并写回 `settings.auth.password_hash`，重置后无需重启服务。密码经环境变量传入、不进入进程 argv，并做转义防注入。 + +### 贡献者 + +- [@Autumn-27](https://github.com/Autumn-27) +- [@neouks](https://github.com/neouks) + ## [0.3.4] - 2026-08-24 ### UI @@ -243,7 +323,8 @@ - [@Autumn-27](https://github.com/Autumn-27) - [@neouks](https://github.com/neouks) -[Unreleased]: https://github.com/Autumn-27/ARTEX/compare/v0.3.4...HEAD +[Unreleased]: https://github.com/Autumn-27/ARTEX/compare/v0.3.5...HEAD +[0.3.5]: https://github.com/Autumn-27/ARTEX/compare/v0.3.4...v0.3.5 [0.3.4]: https://github.com/Autumn-27/ARTEX/compare/v0.3.3...v0.3.4 [0.3.3]: https://github.com/Autumn-27/ARTEX/compare/v0.3.2...v0.3.3 [0.3.2]: https://github.com/Autumn-27/ARTEX/compare/v0.3.1...v0.3.2 diff --git a/agent/blackboard_inheritance_test.go b/agent/blackboard_inheritance_test.go index ad4fe4b..4549ff5 100644 --- a/agent/blackboard_inheritance_test.go +++ b/agent/blackboard_inheritance_test.go @@ -326,7 +326,7 @@ func TestBlackboardToolsReadDirectSources(t *testing.T) { t.Fatalf("inherited host context missing: %#v", coverage) } - facts := callReadJSON(t, tools.listFacts(), `{}`).([]any) + facts := callReadJSON(t, tools.listFacts(), `{}`).(map[string]any)["facts"].([]any) seenSource, seenGrand := false, false for _, raw := range facts { item := raw.(map[string]any) diff --git a/agent/capture_usage_test.go b/agent/capture_usage_test.go index c76f79c..5b6389c 100644 --- a/agent/capture_usage_test.go +++ b/agent/capture_usage_test.go @@ -19,6 +19,17 @@ func (p captureUsageProvider) Stream(ctx context.Context, _ llm.CompletionReques return func(yield func(llm.StreamEvent, error) bool) { p.stream(ctx, yield) } } +func (p captureUsageProvider) Complete(ctx context.Context, req llm.CompletionRequest) (llm.Message, string, llm.Usage, error) { + acc := llm.NewAccumulator() + for ev, err := range p.Stream(ctx, req) { + if err != nil { + return llm.Message{}, "", llm.Usage{}, err + } + acc.Add(ev) + } + return acc.Message(), acc.StopReason, acc.Usage, nil +} + func TestCaptureRunPersistsUsageOnProviderFailure(t *testing.T) { wantErr := errors.New("provider failed after reporting usage") provider := captureUsageProvider{stream: func(_ context.Context, yield func(llm.StreamEvent, error) bool) { diff --git a/agent/chat.go b/agent/chat.go index 41be93b..0f38582 100644 --- a/agent/chat.go +++ b/agent/chat.go @@ -6,13 +6,13 @@ import ( "path/filepath" "time" + "github.com/Autumn-27/artex/db" + "github.com/Autumn-27/artex/guard" "github.com/Autumn-27/norma/agentcore" "github.com/Autumn-27/norma/llm" "github.com/Autumn-27/norma/permission" actool "github.com/Autumn-27/norma/tool" "github.com/Autumn-27/norma/transcript" - "github.com/Autumn-27/artex/db" - "github.com/Autumn-27/artex/guard" ) // ChatAgent is the generic, task-independent conversational runner behind the chat @@ -22,21 +22,28 @@ import ( // LS/Glob/Grep) plus whatever skills/MCP the key is made visible; NO pentest // graph/task context is injected (that stays exclusive to MainAgent). type ChatAgent struct { - prov llm.Provider - model string - workDir string - tx *transcript.Store - window int - proxyAddr string - proxyCACert string - webSearch WebSearchOpts - guard *guard.Guard // optional; nil disables intercept hooks for chat + prov llm.Provider + model string + workDir string + tx *transcript.Store + window int + proxyAddr string + proxyCACert string + webSearch WebSearchOpts + guard *guard.Guard // optional; nil disables intercept hooks for chat + nonStreamingFn func() bool // resolver: use non-streaming (Complete) path? (nil = streaming) } func NewChatAgent(prov llm.Provider, model, workDir string, tx *transcript.Store, window int) *ChatAgent { return &ChatAgent{prov: prov, model: model, workDir: workDir, tx: tx, window: window} } +// SetNonStreaming wires a resolver deciding whether chat runs use the +// non-streaming model path (true = non-streaming). nil/unset = streaming. +func (c *ChatAgent) SetNonStreaming(fn func() bool) { c.nonStreamingFn = fn } + +func (c *ChatAgent) nonStreaming() bool { return c.nonStreamingFn != nil && c.nonStreamingFn() } + // SetProxy points the chat agent's WebFetch/Bash at the recording proxy plus the // CA cert it trusts (empty addr = direct). Kept for parity with the other agents. func (c *ChatAgent) SetProxy(addr, caCert string) { c.proxyAddr, c.proxyCACert = addr, caCert } @@ -115,18 +122,19 @@ func (c *ChatAgent) Chat(ctx context.Context, agentKey, sessionID, message strin BraveSearchAPIKey: ws.BraveKey, TavilySearchAPIKey: ws.TavilyKey, WebSearchProxy: ws.Proxy, - BashEnv: proxyEnv(c.proxyAddr, c.proxyCACert), // Bash 子命令默认走代理+信任 CA - WorkingDir: sessionWorkDir, - MaxTurns: maxTurns, - MaxDuration: maxDuration, - Compaction: compactionConfig(c.window), - Todos: actool.NewTodoStore(), + BashEnv: proxyEnv(c.proxyAddr, c.proxyCACert), // Bash 子命令默认走代理+信任 CA + WorkingDir: sessionWorkDir, + MaxTurns: maxTurns, + MaxDuration: maxDuration, + Compaction: compactionConfig(c.window), + Todos: actool.NewTodoStore(), // large tool output spills to cmd-output/ under the session dir. // 截断上限用 SDK 默认(tool.Capture 的 30000 字符)。 ToolOutputDir: filepath.Join(sessionWorkDir, "cmd-output"), // 命中预算(步数)→ SDK 跑收尾:输出一句总结。Prompt 与收尾轮数按本 agent key 后台可编辑 // (自定义 agent 各自一份;留空/0 用通用默认:10 轮)。 - Settlement: wrapupSettlement(agentKey, nil), + Settlement: wrapupSettlement(agentKey, nil), + NonStreaming: c.nonStreaming(), // 该 profile 选非流式时走 Provider.Complete } if c.guard != nil { opts.Hooks = c.guard.Hooks() diff --git a/agent/goals.go b/agent/goals.go index 388515a..b98e9fe 100644 --- a/agent/goals.go +++ b/agent/goals.go @@ -98,13 +98,13 @@ func DecomposeGoals(ctx context.Context, prov llm.Provider, dataDir, goalText, d if prov == nil { return nil } - return DecomposeGoalsWithProvider(ctx, prov, dataDir, goalText, desc, as, ts, taskID, emit) + return DecomposeGoalsWithProvider(ctx, prov, dataDir, goalText, desc, as, ts, taskID, false, emit) } // DecomposeGoalsWithProvider is the task-runtime variant used when a task has an // ordered provider chain. It preserves the same tools and write behavior while // letting the caller own provider selection/failover. -func DecomposeGoalsWithProvider(ctx context.Context, prov llm.Provider, dataDir, goalText, desc string, as *db.AssetStore, ts *db.ExplorationStore, taskID int64, emit func(db.Activity)) []GoalSpec { +func DecomposeGoalsWithProvider(ctx context.Context, prov llm.Provider, dataDir, goalText, desc string, as *db.AssetStore, ts *db.ExplorationStore, taskID int64, nonStreaming bool, emit func(db.Activity)) []GoalSpec { if prov == nil { return nil } @@ -145,7 +145,8 @@ func DecomposeGoalsWithProvider(ctx context.Context, prov llm.Provider, dataDir, PermissionMode: acperm.ModeBypass, DisableBackgroundTasks: true, // 3 步(抽约束 → 登记范围 → 拆目标)各需一次工具调用,给足回合避免收尾前漏调 set_goals。 - MaxTurns: 8, + MaxTurns: 8, + NonStreaming: nonStreaming, // 该 profile 选非流式时走 Provider.Complete }, userMsg, captureEmit) // set_goals persisted the goals directly; read them back so the caller sees what // was written (empty slice ⇒ the LLM produced nothing ⇒ caller falls back). diff --git a/agent/mainagent.go b/agent/mainagent.go index 99531df..9c40eca 100644 --- a/agent/mainagent.go +++ b/agent/mainagent.go @@ -17,19 +17,26 @@ import ( // (→planner) or direct high-priority intents (→frontier). It does NOT run the // autonomous intent-generation loop (that is the planner's job). type MainAgent struct { - prov llm.Provider - model string - tx *transcript.Store // raw LLM conversation persistence (nil = off) - window int // context window in tokens (for compaction) - windowFn func() int // optional dynamic task-chain minimum - maxTurns int // max agent turns per run (0 = unlimited) - proxyAddr string // recording proxy for WebFetch (empty = direct) - proxyCACert string // recording proxy's CA cert path (HTTPS verify) - webSearch WebSearchOpts // web_search tool backend selection (off by default) - workDir string // shared work dir (surfaced in prompt as artifact-output target) - steerWork func(intentID int64, msg string) error // engine callback: steer a running work (nil = off) + prov llm.Provider + model string + tx *transcript.Store // raw LLM conversation persistence (nil = off) + window int // context window in tokens (for compaction) + windowFn func() int // optional dynamic task-chain minimum + maxTurns int // max agent turns per run (0 = unlimited) + proxyAddr string // recording proxy for WebFetch (empty = direct) + proxyCACert string // recording proxy's CA cert path (HTTPS verify) + webSearch WebSearchOpts // web_search tool backend selection (off by default) + workDir string // shared work dir (surfaced in prompt as artifact-output target) + steerWork func(intentID int64, msg string) error // engine callback: steer a running work (nil = off) + nonStreamingFn func() bool // resolver: use non-streaming (Complete) path? (nil = streaming) } +// SetNonStreaming wires a resolver deciding whether runs use the non-streaming +// model path (true = non-streaming). nil/unset = streaming (default). +func (m *MainAgent) SetNonStreaming(fn func() bool) { m.nonStreamingFn = fn } + +func (m *MainAgent) nonStreaming() bool { return m.nonStreamingFn != nil && m.nonStreamingFn() } + func NewMainAgent(prov llm.Provider, model, workDir string, tx *transcript.Store, window, maxTurns int) *MainAgent { return &MainAgent{prov: prov, model: model, workDir: workDir, tx: tx, window: window, maxTurns: maxTurns} } @@ -128,7 +135,8 @@ func (m *MainAgent) Chat(ctx context.Context, taskID int64, as *db.AssetStore, t Compaction: compactionConfig(m.compactionWindow()), // long chats stay within the window Todos: actool.NewTodoStore(), // 会话级临时待办（TodoWrite），纯规划用，退出即丢 // 命中预算(步数)→ SDK 跑收尾:向用户输出一句进展总结。Prompt 与收尾轮数可后台编辑(默认 10 轮)。 - Settlement: wrapupSettlement("mainagent", nil), + Settlement: wrapupSettlement("mainagent", nil), + NonStreaming: m.nonStreaming(), // 该 profile 选非流式时走 Provider.Complete } if m.tx != nil { // persist raw human↔AI conversation; one accumulating file per task opts.Transcript = m.tx diff --git a/agent/planner.go b/agent/planner.go index 21437cf..95802f8 100644 --- a/agent/planner.go +++ b/agent/planner.go @@ -20,19 +20,20 @@ import ( // assets, judges whether the task goal is met, and emits 0..N exploration intents // into the frontier. It is the sole intent generator. type Planner struct { - prov llm.Provider - model string - tx *transcript.Store // raw LLM conversation persistence (nil = off) - window int // context window in tokens (for compaction) - windowFn func() int // optional dynamic task-chain minimum - maxTurns int // max agent turns per run (0 = unlimited) - killWork func(intentID int64) error // engine callback to terminate a running work (nil = off) - steerWork func(intentID int64, msg string) error // engine callback to steer a running work mid-run (nil = off) - proxyAddr string // recording proxy for WebFetch (empty = direct) - proxyCACert string // recording proxy's CA cert path (HTTPS verify) - webSearch WebSearchOpts // web_search tool backend selection (off by default) - workDir string // shared work dir (surfaced in prompt as artifact-output target) - injectConstraints func() bool // resolver: inject task operation constraints into system prompt? (nil = yes) + prov llm.Provider + model string + tx *transcript.Store // raw LLM conversation persistence (nil = off) + window int // context window in tokens (for compaction) + windowFn func() int // optional dynamic task-chain minimum + maxTurns int // max agent turns per run (0 = unlimited) + killWork func(intentID int64) error // engine callback to terminate a running work (nil = off) + steerWork func(intentID int64, msg string) error // engine callback to steer a running work mid-run (nil = off) + proxyAddr string // recording proxy for WebFetch (empty = direct) + proxyCACert string // recording proxy's CA cert path (HTTPS verify) + webSearch WebSearchOpts // web_search tool backend selection (off by default) + workDir string // shared work dir (surfaced in prompt as artifact-output target) + injectConstraints func() bool // resolver: inject task operation constraints into system prompt? (nil = yes) + nonStreamingFn func() bool // resolver: use non-streaming (Complete) path? (nil = streaming) // todos keeps ONE plan-scratchpad per task (keyed by exploration id) so the // planner's multi-step plan survives across wake-ups — each Plan() is a fresh @@ -48,6 +49,12 @@ func NewPlanner(prov llm.Provider, model, workDir string, tx *transcript.Store, func (p *Planner) SetCompactionWindowResolver(fn func() int) { p.windowFn = fn } +// SetNonStreaming wires a resolver deciding whether runs use the non-streaming +// model path (true = non-streaming). nil/unset = streaming (default). +func (p *Planner) SetNonStreaming(fn func() bool) { p.nonStreamingFn = fn } + +func (p *Planner) nonStreaming() bool { return p.nonStreamingFn != nil && p.nonStreamingFn() } + func (p *Planner) compactionWindow() int { if p.windowFn != nil { return p.windowFn() @@ -116,7 +123,6 @@ func renderPlannerTodos(items []actool.Todo) string { // // "done" — a worker finished intent IntentID (its output conclusion is fetched). // "finding" — a worker reported a finding on intent IntentID (Detail = 摘要). -// "worker_message" — the human changed one Worker's direction (Detail = 新方向). // "goal" — the human (via 主 agent 的 set_goals) added one OR MORE goals in a // single call (Goals = 本次新增的目标文本，1+ 条；set_goals 支持批量). // "goal_deleted" — the human deleted a goal from 总览的目标管理 (Detail = 被删目标文本). @@ -124,17 +130,14 @@ func renderPlannerTodos(items []actool.Todo) string { // "cancelled" — the human deleted intent IntentID (Detail = 删除原因). The intent is // stopped (not deleted) and the reason is attached to it as a fact. type TriggerEvent struct { - Kind string - ActivityID int64 // persistent activity id; worker_message notifications use it for de-duplication - IntentID int64 - Detail string - Goals []string // Kind=="goal" 专用：本次 set_goals 新增的目标文本（1 条或多条） - OldGoal string // Kind=="goal_edited" 专用：修改前的目标文本 - NewGoal string // Kind=="goal_edited" 专用：修改后的目标文本 + Kind string + IntentID int64 + Detail string + Goals []string // Kind=="goal" 专用：本次 set_goals 新增的目标文本（1 条或多条） + OldGoal string // Kind=="goal_edited" 专用：修改前的目标文本 + NewGoal string // Kind=="goal_edited" 专用：修改后的目标文本 } -const TriggerWorkerMessage = db.PlannerEventWorkerMessage - // renderTriggers spells out the change(s) that fired this round: for a finished // worker — which intent + its output conclusion; for a finding — which intent + // what was found. Empty for time/heartbeat wakes. Reads the store (best-effort; @@ -159,9 +162,6 @@ func renderTriggers(ts *db.ExplorationStore, evs []TriggerEvent) string { b.WriteString(fmt.Sprintf("\n- 人修改了目标，由「%s」变为「%s」—— 请据新目标调整探索方向（原方向若已不适用请停派）。", ev.OldGoal, ev.NewGoal)) case "finding": b.WriteString(fmt.Sprintf("\n- 意图 #%d（%s）的 worker 报告了一个 finding：%s", ev.IntentID, intentSummary(ts, ev.IntentID), ev.Detail)) - case TriggerWorkerMessage: - b.WriteString(fmt.Sprintf("\n- 用户向意图 #%d（%s）的 Worker 下发了新的人工方向：%s —— 该方向已优先交给同一 Worker；后续规划必须以它替代或补充原意图描述，避免生成冲突、回退或重复方向。", - ev.IntentID, intentSummary(ts, ev.IntentID), truncOutput(ev.Detail, 2000))) case "cancelled": b.WriteString(fmt.Sprintf("\n- 意图 #%d 由用户删除，意图内容是：%s、删除原因是：%s。该意图已停止（不再执行），其原因已作为事实挂在该意图上；请据此重新规划。", ev.IntentID, intentSummary(ts, ev.IntentID), ev.Detail)) default: // "done" @@ -242,15 +242,18 @@ const plannerDefaultTmpl = `你是一个授权渗透测试系统的"规划者" 任务目标：{{.Goal}} -⚠️ 最重要的原则：**这一轮生成 0 个意图是完全正常、而且是最常见的结果。** 你不是"每次都要产出意图"的机器。绝大多数唤醒，frontier 里已有的意图已经覆盖了所有已知方向，你应当什么都不加、直接结束。重复/换措辞地生成已经存在的意图是严重错误。 +⚠️ **生成 0 个意图是合法、正常的结果，但它有【具体的正当理由】，不是"少派更稳"的默认姿态。** 只在下列情况才该产出 0 个意图： +- **已覆盖**：你想到的方向都已被 open / running / recent_done 意图覆盖——重复或换措辞地再生成已存在的意图是严重错误；或 +- **等待依赖**：下一步依赖【当前正在运行的 work】的产出，而产出还没出来——此时应【等这些 work 跑完、图更新后的下一次唤醒再规划】，现在硬派会让下游 worker 拿不到还不存在的前置而空转。 +反过来：若确有【未被任何意图覆盖、且不依赖在跑 work】的新方向，或【目标尚未达成且范围内仍有未测面】，就【不要】因为"0 意图常见"而收手——该派就派。别把 0 意图当成偷懒的默认。 决策流程（每次唤醒）： -1. **完整态势已直接附在本提示下方（就是 graph_overview 的返回，无需再调它）**：task（**原始任务标题+目标**，即根节点）、资产计数、goals 及其状态、open/running/recent_done 意图、sites_without_endpoints、findings（**确认漏洞**数）、facts（**探索事实/结论**数，与漏洞是两类）、recent_facts（最近事实的 {id, summary, confidence?}，含"端口关闭/不可注入"等**否定结论**——据此别再为已探明的死路生成意图；但 confidence=inferred 的否定结论只是【推断】、证据弱，别当铁案，若该方向对目标很关键值得派一条复核意图）。**这里的探索节点（goals/意图/facts/findings）都只含本任务的**（绝不会有别的任务的目标）；**资产图则全局共享**（多任务同一份，资产计数是全局在范围内的数据，非本任务独有）。需要更深的细节时才按需调：list_facts 列全部事实、list_findings 列全部漏洞、**node_detail(id)** 取某条的完整证据/详情（列表/recent_facts 只给摘要）。 +1. **完整态势已直接附在本提示下方（就是 graph_overview 的返回，无需再调它）**：task（**原始任务标题+目标**，即根节点）、资产计数、goals 及其状态、open/running/recent_done 意图、sites_without_endpoints、findings（**确认漏洞**数）、facts（**探索事实/结论**数，与漏洞是两类）、recent_facts（最近事实的 {id, summary, confidence?}，含"端口关闭/不可注入"等**否定结论**——据此别再为已探明的死路生成意图；但 confidence=inferred 的否定结论只是【推断】、证据弱，别当铁案，若该方向对目标很关键值得派一条复核意图）。**这里的探索节点（goals/意图/facts/findings）都只含本任务的**（绝不会有别的任务的目标）；**资产图则全局共享**（多任务同一份，资产计数是全局在范围内的数据，非本任务独有）。需要更深的细节时才按需调：list_facts 分页列事实（最新在前，默认 20 条，可传 q 关键词过滤、before 翻页，返回带 total/has_more）、list_findings 列全部漏洞、**node_detail(id)** 取某条的完整证据/详情（列表/recent_facts 只给摘要）。 - open_intents / running_intents / recent_done_intents——"哪些方向已经有意图在覆盖/已尝试"。每个意图还带 **parents（上游：它派生自哪些事实/意图）和 yields（下游：它产生了哪些事实/发现）**——这就是探索图的**血缘关系**，据此理解"哪些事实来自哪个方向、能否综合成新方向"。recent_facts 里每个事实带 **from_intent**（由哪个意图产生）。 - - worker_directions——用户在 Worker 对话里对每个意图最近一次下发的人工方向（持久化、重启后仍存在）。它比该意图最初的 summary 更新；判断在跑方向、补意图和使用 steer/kill 时必须以它为准，禁止重新派发与人工方向冲突的工作。 - sites_without_endpoints / findings——"哪些方向【可能】需要探索"。 - 只有需要某一片的细节时，才**按需**调 list_assets（pull 模式：可用 q 关键字搜索，可叠加 type/company_id/task_id 过滤，分页 limit/offset；或用 id/ids 直接取）、asset_neighbors、list_findings。资产图全局共享，别默认拉全量。资产中可能包含非本次任务涉及到的资产，所以需要主要出现非本次任务相关的资产时忽略这些资产。 2. 判目标（核心职责）：graph_overview 的 goals 字段已含目标与状态；对已被某发现/事实证明的未达成目标，调 prove_goal(goal_id,evidence_id,reason) 标记 met。**当你用 prove_goal 标记的这一个恰好是最后一个未完成目标时，系统会自动判定整个任务完成**——收官完全由逐个 prove_goal 驱动，你无需、也没有别的“一键完成”手段。 + ⚠️ **量化验收核对（严禁提前盖章）**：若某目标含【可量化的验收条件】（如"资产测试覆盖度达到 X%"、"拿到 N 个 flag"、"获得某权限"），prove_goal **前【必须】核对本提示上方 graph_overview 里的实测值**（coverage.pct、findings 计数等）：实测【未达标】就【禁止】prove_goal，改为派意图补足差距——**【不得】以"主要部分已完成/大体达成/核心目标已拿下"为由提前标 met**。例：目标要求覆盖度 100%、而 graph_overview 实测 coverage.pct=40%，则该目标【未达成】，继续派补测意图，不许 prove_goal。 2.5. **（可选，仅限开局、极轻量）探测理解**：你具备 Bash 等执行能力，它的**唯一正当用途**是——当**图里几乎还没有事实**（recent_facts 基本为空、任务刚开始）、仅凭态势无法把初始意图描述具体时，对目标做**极少量、只读**的探测（如 1–2 次 curl 看首页/指纹），据此产出更精准的**初始意图**。 ⚠️ **牢记你的身份边界：你是"规划者"，不是"执行者"。你在这里做的一切都只为【生成/说清意图】，绝不是【在 plan 里把活干了】。** 探测的**唯一合法产物是一句更精准的意图描述**——绝不能是漏洞的发现、验证、利用，也不能是端点/目录/参数的枚举结果。任何"我顺手把这个也测了/确认了"的念头都是越界：那是 worker 在 work 阶段该做的事，你只需把它**写成一个意图派下去**。 **三条硬性边界，务必守住**： @@ -361,7 +364,8 @@ func (p *Planner) Plan(ctx context.Context, taskID int64, as *db.AssetStore, ts // 命中【本轮】步数预算→ SDK 跑收尾:把本轮已想清楚的结论落地(该派的 add_intent、 // 能证的 prove_goal、串行链记 TodoWrite),而非停止规划——planner 之后仍会被反复唤醒。 // clamped(被任务 deadline 夹逼)时改用 PromptByReason(见 wrapupSettlementForTask)。 - Settlement: settle, + Settlement: settle, + NonStreaming: p.nonStreaming(), // 该 profile 选非流式时走 Provider.Complete } if p.tx != nil { // persist raw LLM conversation; one accumulating file per task's planner opts.Transcript = p.tx diff --git a/agent/provider.go b/agent/provider.go index 6011ada..4def9b5 100644 --- a/agent/provider.go +++ b/agent/provider.go @@ -19,6 +19,7 @@ import ( "strings" "time" + "github.com/Autumn-27/artex/llmrec" "github.com/Autumn-27/norma/agentcore" "github.com/Autumn-27/norma/compaction" "github.com/Autumn-27/norma/llm" @@ -51,6 +52,11 @@ type Config struct { // "" = 不发送(默认); "low"/"medium"/"high"/"xhigh"/"max" = 对应强度. // OpenAI 映射为顶层 reasoning_effort;Anthropic 映射为 output_config.effort. ReasoningEffort string + // Stream 控制该 profile 是否使用流式(SSE)接口。true(默认)= 流式;false = 真· + // 非流式(发 stream:false,一次性拿完整 JSON,走 Provider.Complete)。非流式可绕开 + // 某些网关糟糕的 SSE 实现(空帧、思考字段丢帧),代价是失去运行中的实时进度/实时 + // token 计数。映射为 agentcore.Options.NonStreaming = !Stream。 + Stream bool } // compaction window resolution bounds (in K tokens). Below the floor the @@ -116,6 +122,8 @@ func FromEnv() (Config, bool) { BaseURL: os.Getenv("ARTEX_LLM_BASE_URL"), Model: os.Getenv("ARTEX_LLM_MODEL"), Proxy: strings.TrimSpace(os.Getenv("ARTEX_LLM_PROXY")), + // 默认流式;ARTEX_LLM_STREAM=false/0/off 显式关闭走非流式。 + Stream: !isFalsy(os.Getenv("ARTEX_LLM_STREAM")), } switch prov { case "openai": @@ -124,6 +132,12 @@ func FromEnv() (Config, bool) { if c.Model == "" { c.Model = "gpt-4o" } + case "openai-responses": + c.Format = llm.FormatOpenAIResponses + c.APIKey = oaiKey + if c.Model == "" { + c.Model = "gpt-5" + } default: c.Format = llm.FormatAnthropic c.APIKey = anthKey @@ -147,6 +161,7 @@ func ConfigFrom(provider, model, baseURL, apiKey, proxy string) Config { BaseURL: strings.TrimRight(strings.TrimSpace(baseURL), "/"), APIKey: strings.TrimSpace(apiKey), Proxy: strings.TrimSpace(proxy), + Stream: true, // 默认流式;调用方按 profile 覆盖 } switch strings.TrimSpace(provider) { case "openai": @@ -156,6 +171,13 @@ func ConfigFrom(provider, model, baseURL, apiKey, proxy string) Config { if c.Model == "" { c.Model = "gpt-4o" } + case "openai-responses": + c.Format = llm.FormatOpenAIResponses + // provider appends "/responses"; tolerate a full endpoint URL. + c.BaseURL = strings.TrimRight(strings.TrimSuffix(c.BaseURL, "/responses"), "/") + if c.Model == "" { + c.Model = "gpt-5" + } default: c.Format = llm.FormatAnthropic // provider appends "/v1/messages". @@ -167,10 +189,23 @@ func ConfigFrom(provider, model, baseURL, apiKey, proxy string) Config { return c } +// isFalsy reports whether an env-var string explicitly requests "off". Empty or +// unrecognized → false (so an unset var keeps the streaming default). +func isFalsy(s string) bool { + switch strings.ToLower(strings.TrimSpace(s)) { + case "0", "false", "off", "no": + return true + } + return false +} + // Provider returns the short provider name ("anthropic"/"openai"). func (c Config) Provider() string { - if c.Format == llm.FormatOpenAI { + switch c.Format { + case llm.FormatOpenAI: return "openai" + case llm.FormatOpenAIResponses: + return "openai-responses" } return "anthropic" } @@ -247,10 +282,25 @@ func IsQuotaExhaustedMessage(message string) bool { type quotaAwareTransport struct{ base http.RoundTripper } func (t quotaAwareTransport) RoundTrip(req *http.Request) (*http.Response, error) { + // When LLM recording is on, the Recorder puts a Capture on the context so the + // raw wire bodies can be persisted. This is the only layer that still sees + // them: norma builds the request body internally and decodes the SSE response + // before either reaches the recorder. + capt := llmrec.CaptureFrom(req.Context()) + capt.SetRequest(requestBodySnapshot(req)) + resp, err := t.base.RoundTrip(req) - if err != nil || resp == nil || resp.StatusCode != http.StatusTooManyRequests { + if err != nil || resp == nil { return resp, err } + // Tee rather than read: a 200 is an SSE stream that must keep streaming. The + // 429 branch below reads through this wrapper, so its body lands in the + // capture before being replaced. + resp.Body = capt.TeeResponse(resp.StatusCode, resp.Body) + + if resp.StatusCode != http.StatusTooManyRequests { + return resp, nil + } body, readErr := io.ReadAll(resp.Body) _ = resp.Body.Close() resp.Body = io.NopCloser(bytes.NewReader(body)) @@ -265,6 +315,25 @@ func (t quotaAwareTransport) RoundTrip(req *http.Request) (*http.Response, error return resp, nil } +// requestBodySnapshot copies an outgoing request body without consuming it. +// norma builds every model request from a *bytes.Reader, so net/http populates +// GetBody and the copy has no effect on what gets sent. +func requestBodySnapshot(req *http.Request) string { + if req.GetBody == nil { + return "" + } + rc, err := req.GetBody() + if err != nil { + return "" + } + defer rc.Close() + b, err := io.ReadAll(rc) + if err != nil { + return "" + } + return string(b) +} + func quotaAwareHTTPClient(proxy string) (*http.Client, error) { transport := http.DefaultTransport.(*http.Transport).Clone() proxy = strings.TrimSpace(proxy) @@ -288,11 +357,12 @@ func quotaAwareHTTPClient(proxy string) (*http.Client, error) { } // TestConnection makes a minimal real completion to verify the provider/model/ -// endpoint/key actually work. Returns the round-trip latency. -func TestConnection(ctx context.Context, c Config) (time.Duration, error) { +// endpoint/key actually work. Returns the round-trip latency and the model's +// reply text. +func TestConnection(ctx context.Context, c Config) (time.Duration, string, error) { prov, err := c.NewProvider() if err != nil { - return 0, err + return 0, "", err } ctx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() @@ -302,12 +372,24 @@ func TestConnection(ctx context.Context, c Config) (time.Duration, error) { // 就撞到输出上限(finish=length)、被截断,连接测试虽仍算通(err=nil)但显示成 // "已中断/length/resume" 一团糟。给足预算让它把 OK 干净吐完(finish=stop)。 // EscalateMaxTokens 保持 false:不因截断而抬额重试,避免 resume 循环空烧。 - _, err = agentcore.Run(ctx, agentcore.Options{ + reply, err := agentcore.Run(ctx, agentcore.Options{ Provider: prov, SystemPrompt: []string{"你是连接测试。直接输出两个字符 OK 即可，不要思考、不要解释、不要别的。"}, PermissionMode: acperm.ModeBypass, MaxTurns: 1, MaxTokens: 8192, + NonStreaming: !c.Stream, // 用该 profile 的真实收发模式做连接测试 }, "ping") - return time.Since(start), err + lat := time.Since(start) + if err != nil { + return lat, "", err + } + // err==nil 还不够：请求通了但模型一个字都不吐的情况真实存在(思考把预算烧光、 + // 正文被安全策略吞掉、兼容层把 content 丢了)。这种配置在会话里就是"不回话", + // 测试却报成功——正是本项要消除的落差。没有可见正文一律判失败。 + reply = strings.TrimSpace(reply) + if reply == "" { + return lat, "", fmt.Errorf("模型无回复内容（请求已通，但未返回任何文本）") + } + return lat, reply, nil } diff --git a/agent/provider_capture_e2e_test.go b/agent/provider_capture_e2e_test.go new file mode 100644 index 0000000..3405f6d --- /dev/null +++ b/agent/provider_capture_e2e_test.go @@ -0,0 +1,131 @@ +package agent + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/Autumn-27/artex/llmrec" + "github.com/Autumn-27/norma/llm" +) + +// End-to-end through a real norma provider: the Capture rides the context into +// norma, survives its internal request building, and comes back holding the +// exact bytes buildBody() put on the wire. This is the load-bearing assumption +// of the whole feature — norma must propagate the caller's context down to +// http.NewRequestWithContext. +func TestCapturePropagatesThroughNormaProvider(t *testing.T) { + sse := strings.Join([]string{ + `event: message_start`, + `data: {"type":"message_start","message":{"id":"msg_1","usage":{"input_tokens":11,"output_tokens":1}}}`, + ``, + `event: content_block_delta`, + `data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"hello"}}`, + ``, + `event: message_delta`, + `data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":7}}`, + ``, + `event: message_stop`, + `data: {"type":"message_stop"}`, + ``, + }, "\n") + + // Record exactly what the server receives, so the capture can be compared + // against it byte for byte rather than merely spot-checked for fields. + var gotPath, serverSaw string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + b, err := io.ReadAll(r.Body) + if err != nil { + t.Errorf("server read body: %v", err) + } + serverSaw = string(b) + w.Header().Set("content-type", "text/event-stream") + _, _ = io.WriteString(w, sse) + })) + defer srv.Close() + + cfg := Config{ + Format: llm.FormatAnthropic, + BaseURL: srv.URL, + APIKey: "test-key", + Model: "claude-test", + } + prov, err := cfg.NewProvider() + if err != nil { + t.Fatalf("NewProvider: %v", err) + } + + ctx, capt := llmrec.NewCapture(context.Background()) + req := llm.CompletionRequest{ + System: []string{"you are a scanner"}, + Messages: []llm.Message{{Role: "user", Content: []llm.ContentBlock{{Type: "text", Text: "go"}}}}, + Tools: []llm.ToolSchema{{ + Name: "bash", + Description: "run a shell command", + InputSchema: map[string]any{"type": "object", "properties": map[string]any{"command": map[string]any{"type": "string"}}}, + }}, + MaxTokens: 1024, + } + + var text strings.Builder + for ev, err := range prov.Stream(ctx, req) { + if err != nil { + t.Fatalf("stream: %v", err) + } + if ev.Type == llm.SETextDelta { + text.WriteString(ev.Text) + } + } + if text.String() != "hello" { + t.Fatalf("stream text=%q, capture interfered with delivery", text.String()) + } + if gotPath != "/v1/messages" { + t.Fatalf("path=%q", gotPath) + } + + // The raw request must be what norma actually sent, not the recorder's + // re-serialization — which is exactly why it carries fields the normalized + // view drops. + raw := capt.RawRequest() + if raw == "" { + t.Fatal("no raw request captured — context did not reach the transport") + } + // The load-bearing claim: what we stored equals, byte for byte, what the + // server received — not merely "has the right fields". + if raw != serverSaw { + t.Fatalf("captured request != what the server received:\n got: %q\nsaw: %q", raw, serverSaw) + } + var body map[string]any + if err := json.Unmarshal([]byte(raw), &body); err != nil { + t.Fatalf("raw request is not valid JSON: %v\n%s", err, raw) + } + if body["model"] != "claude-test" { + t.Errorf("model=%v want claude-test (absent from CompletionRequest)", body["model"]) + } + if body["stream"] != true { + t.Errorf("stream=%v want true", body["stream"]) + } + // The full tool schema is the headline gain: the normalized view keeps names only. + tools, _ := body["tools"].([]any) + if len(tools) != 1 { + t.Fatalf("tools=%v", body["tools"]) + } + tool, _ := tools[0].(map[string]any) + if tool["description"] != "run a shell command" { + t.Errorf("tool description missing: %v", tool) + } + if tool["input_schema"] == nil { + t.Errorf("tool input_schema missing: %v", tool) + } + + // And the response is the untouched SSE frames, including the events the + // recorder never turns into stored output. + if capt.RawResponse() != sse { + t.Errorf("RawResponse mismatch:\n got: %q\nwant: %q", capt.RawResponse(), sse) + } +} diff --git a/agent/provider_capture_test.go b/agent/provider_capture_test.go new file mode 100644 index 0000000..514b2c7 --- /dev/null +++ b/agent/provider_capture_test.go @@ -0,0 +1,125 @@ +package agent + +import ( + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/Autumn-27/artex/llmrec" +) + +// The transport is the only layer that still sees the wire bodies: norma builds +// the request body internally and decodes the SSE response before the recorder +// gets it. This checks the round trip preserves both directions untouched. +func TestRoundTripCapturesRawBodies(t *testing.T) { + const sse = "event: message_start\ndata: {\"type\":\"message_start\"}\n\nevent: message_stop\ndata: {}\n\n" + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("content-type", "text/event-stream") + _, _ = io.WriteString(w, sse) + })) + defer srv.Close() + + client, err := quotaAwareHTTPClient("") + if err != nil { + t.Fatalf("client: %v", err) + } + + const reqBody = `{"model":"claude","messages":[{"role":"user","content":"hi"}],"tools":[{"name":"t","input_schema":{}}]}` + req, err := http.NewRequest(http.MethodPost, srv.URL, strings.NewReader(reqBody)) + if err != nil { + t.Fatalf("request: %v", err) + } + ctx, capt := llmrec.NewCapture(req.Context()) + req = req.WithContext(ctx) + + resp, err := client.Do(req) + if err != nil { + t.Fatalf("do: %v", err) + } + got, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read: %v", err) + } + _ = resp.Body.Close() + + if string(got) != sse { + t.Fatal("capture altered the response delivered to norma") + } + if capt.RawRequest() != reqBody { + t.Fatalf("RawRequest()=%q want %q", capt.RawRequest(), reqBody) + } + if capt.RawResponse() != sse { + t.Fatalf("RawResponse()=%q want the raw SSE frames", capt.RawResponse()) + } +} + +// A 429 body is read and replaced in-place by the quota check. Capturing must +// still see it, and the replacement body must remain readable downstream. +func TestRoundTripCaptures429BodyAlongsideQuotaRewrite(t *testing.T) { + const body = `{"error":{"message":"insufficient_quota"}}` + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusTooManyRequests) + _, _ = io.WriteString(w, body) + })) + defer srv.Close() + + client, err := quotaAwareHTTPClient("") + if err != nil { + t.Fatalf("client: %v", err) + } + req, err := http.NewRequest(http.MethodPost, srv.URL, strings.NewReader("{}")) + if err != nil { + t.Fatalf("request: %v", err) + } + ctx, capt := llmrec.NewCapture(req.Context()) + req = req.WithContext(ctx) + + resp, err := client.Do(req) + if err != nil { + t.Fatalf("do: %v", err) + } + defer resp.Body.Close() + + // Quota exhaustion is normalized to 402 so the router fails over. + if resp.StatusCode != http.StatusPaymentRequired { + t.Fatalf("status=%d want 402", resp.StatusCode) + } + if capt.RawResponse() != body { + t.Fatalf("RawResponse()=%q want %q", capt.RawResponse(), body) + } + rest, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read replaced body: %v", err) + } + if string(rest) != body { + t.Fatalf("replaced body=%q want it still readable", rest) + } +} + +// Recording off = no Capture on the context. The transport must behave exactly +// as before, including the quota rewrite. +func TestRoundTripWithoutCaptureIsUnchanged(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, "ok") + })) + defer srv.Close() + + client, err := quotaAwareHTTPClient("") + if err != nil { + t.Fatalf("client: %v", err) + } + resp, err := client.Post(srv.URL, "application/json", strings.NewReader("{}")) + if err != nil { + t.Fatalf("post: %v", err) + } + defer resp.Body.Close() + got, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read: %v", err) + } + if string(got) != "ok" { + t.Fatalf("body=%q", got) + } +} diff --git a/agent/provider_responses_test.go b/agent/provider_responses_test.go new file mode 100644 index 0000000..fea982a --- /dev/null +++ b/agent/provider_responses_test.go @@ -0,0 +1,30 @@ +package agent + +import ( + "testing" + + "github.com/Autumn-27/norma/llm" +) + +// TestConfigFromOpenAIResponses locks the openai-responses format wiring: +// ConfigFrom resolves the Responses format, strips a full /responses endpoint +// back to the API base, and Provider() round-trips the short name. NewProvider +// must build a working provider for it. +func TestConfigFromOpenAIResponses(t *testing.T) { + c := ConfigFrom("openai-responses", "gpt-5", "https://gw.example/v1/responses", "sk-x", "") + if c.Format != llm.FormatOpenAIResponses { + t.Fatalf("format=%v, want FormatOpenAIResponses", c.Format) + } + if c.BaseURL != "https://gw.example/v1" { + t.Fatalf("base_url=%q, want the /responses suffix stripped", c.BaseURL) + } + if c.Provider() != "openai-responses" { + t.Fatalf("Provider()=%q", c.Provider()) + } + if !c.Stream { // default streaming preserved + t.Fatal("Stream should default true") + } + if _, err := c.NewProvider(); err != nil { + t.Fatalf("NewProvider: %v", err) + } +} diff --git a/agent/tools.go b/agent/tools.go index 029f973..ebed6d8 100644 --- a/agent/tools.go +++ b/agent/tools.go @@ -48,38 +48,6 @@ func compactIntents(ns []*db.Node, parentsOf, yieldsOf map[int64][]int64) []map[ return out } -const workerDirectionOverviewLimit = 300 -const workerDirectionMessageRunes = 1200 - -// compactWorkerDirections keeps the latest durable human direction for each -// intent small enough for every Planner/Main Agent turn. The full message stays -// available in the Worker activity trace when an unusually long direction is -// truncated here. -func compactWorkerDirections(items []db.WorkerDirection) []map[string]any { - out := make([]map[string]any, 0, len(items)) - for _, item := range items { - message := item.Message - truncated := false - if runes := []rune(message); len(runes) > workerDirectionMessageRunes { - message = string(runes[:workerDirectionMessageRunes]) + "…" - truncated = true - } - row := map[string]any{ - "activity_id": item.ActivityID, - "intent_id": item.IntentID, - "intent_state": item.IntentState, - "intent_summary": item.IntentSummary, - "message": message, - "updated_at": item.CreatedAt, - } - if truncated { - row["truncated"] = true - } - out = append(out, row) - } - return out -} - // ToolSet exposes the PG-backed dual graph (asset + exploration) to an LLM agent. // One ToolSet is created per planner/worker run; per-run signals live here. type ToolSet struct { @@ -392,14 +360,6 @@ func (t *ToolSet) graphOverview() actool.CoreTool { // an empty context and always needs this first). func (t *ToolSet) graphOverviewData() map[string]any { out := map[string]any{} - // worker_directions is the persistent cross-agent steering channel. It has - // one latest accepted human message per local intent, so Planner and Main - // Agent do not have to infer a changed direction from the original summary. - if directions, err := t.ts.LatestWorkerDirections(workerDirectionOverviewLimit); err == nil { - out["worker_directions"] = compactWorkerDirections(directions) - } else { - out["worker_directions"] = []map[string]any{} - } // goals summary folded in so the planner needn't call list_goals each round. goals, _ := t.ts.ListByKind(db.KindGoal, 100) gsum := make([]map[string]any, 0, len(goals)) @@ -938,19 +898,64 @@ func (t *ToolSet) listFindings() actool.CoreTool { }) } +// factsPageSize is the default page size for list_facts. Facts pile up on long +// tasks; returning all of them at once (the old behaviour) could blow up the +// context, so default to the newest page and let the agent page/filter for more. +const factsPageSize = 20 + func (t *ToolSet) listFacts() actool.CoreTool { - return readTool("list_facts", "列本任务及直接关联任务的【探索事实/结论】(紧凑：id+摘要+状态)。关联任务条目带 source_task_id/inherited=true 且只读。详情用 node_detail(id)，漏洞看 list_findings。", - obj(map[string]any{}), - func(context.Context, json.RawMessage) (actool.Result, error) { - f, _ := t.ts.ListByKindWithSources(db.KindFact, 500) + return readTool("list_facts", "分页列本任务及直接关联任务的【探索事实/结论】，最新在前(紧凑：id+摘要+状态，摘要过长会截断，全文用 node_detail(id))。参数均可选：limit(默认 20，上限 100)、before(游标，传上一页返回的 next_before 取更旧的一页；省略/0=最新一页)、q(按摘要关键词过滤)。返回 {facts, total, has_more, next_before}：total 是过滤后的总数，has_more=true 时用 next_before 继续翻页。关联任务条目带 source_task_id/inherited=true 且只读。漏洞看 list_findings。", + obj(map[string]any{ + "limit": intp("返回条数，默认 20，上限 100"), + "before": intp("分页游标：只返回 id 小于该值的更旧事实；省略或 0 = 最新一页"), + "q": str("按事实摘要关键词过滤（不区分大小写）；省略 = 不过滤"), + }), + func(_ context.Context, in json.RawMessage) (actool.Result, error) { + var a struct { + Limit int `json:"limit"` + Before int64 `json:"before"` + Q string `json:"q"` + } + _ = json.Unmarshal(in, &a) + limit := a.Limit + if limit <= 0 { + limit = factsPageSize + } + if limit > 100 { + limit = 100 + } + f, hasMore, total, err := t.ts.ListByKindPageWithSources(db.KindFact, a.Before, limit, strings.TrimSpace(a.Q)) + if err != nil { + return actool.Result{}, err + } out := make([]map[string]any, 0, len(f)) for _, n := range f { - out = append(out, compactNode(n)) + out = append(out, compactFact(n)) } - return jsonResult(out) + res := map[string]any{"facts": out, "total": total, "has_more": hasMore} + if hasMore && len(f) > 0 { + res["next_before"] = f[len(f)-1].ID // 传回它取下一页(更旧的) + } + return jsonResult(res) }) } +// factSummaryMax caps a fact summary in list_facts output. Facts carry one-line +// conclusions, but nothing enforces brevity; a runaway summary must not bloat a +// whole page. Full text stays available via node_detail(id). +const factSummaryMax = 160 + +// compactFact is compactNode with the summary rune-capped for list_facts, so a +// page of facts stays bounded regardless of how long any single summary grew. +func compactFact(n *db.Node) map[string]any { + m := compactNode(n) + if s, ok := m["summary"].(string); ok && len([]rune(s)) > factSummaryMax { + m["summary"] = string([]rune(s)[:factSummaryMax]) + "…" + m["summary_truncated"] = true + } + return m +} + func (t *ToolSet) nodeDetail() actool.CoreTool { return readTool("node_detail", "按 id 取本任务或直接关联任务的【探索图节点】完整内容。继承节点带 source_task_id/inherited=true 且只读。仅限 list_facts/list_findings/graph_overview 返回的探索节点 id；资产请用 list_assets/asset_neighbors。", obj(map[string]any{"id": idp("探索图节点 id(非资产 id)")}, "id"), diff --git a/agent/worker.go b/agent/worker.go index 34f9188..84f693b 100644 --- a/agent/worker.go +++ b/agent/worker.go @@ -63,10 +63,12 @@ type Worker struct { // injected into the worker system prompt. Read per run so the settings toggle // takes effect without rebuilding the agent. nil = inject (default). injectConstraints func() bool + // nonStreamingFn resolves whether this run uses the non-streaming (Complete) + // path. Read per run so a profile/task toggle takes effect without rebuilding + // the agent. nil = streaming (default). + nonStreamingFn func() bool } -<<<<<<< Updated upstream -======= // WorkerSessionID returns the stable transcript key used by a worker intent. // Worker slots are reusable, so the intent id (rather than work#N) is the // session identity. Keep this helper public so the Worker message API and UI @@ -98,7 +100,6 @@ func (w *Worker) SetNonStreaming(fn func() bool) { w.nonStreamingFn = fn } func (w *Worker) nonStreaming() bool { return w.nonStreamingFn != nil && w.nonStreamingFn() } ->>>>>>> Stashed changes // SetConstraintInject wires a resolver deciding whether this task's operation // constraints get injected into the worker system prompt. nil = inject (default). func (w *Worker) SetConstraintInject(fn func() bool) { w.injectConstraints = fn } @@ -193,9 +194,9 @@ const workerDefaultTmpl = `你是一个授权渗透测试系统的"执行者"(wo 可用工具： - insert_assets：登记新资产（资产图）。**你只传原始信息，key 与父子关联由代码算**：新接口→传完整 url+method（代码自动建 domain→site→endpoint、自动抽 URL 里的 query 参数，body/header 参数放 params）；指纹→type=tech,name=技术名,on_url=站点地址,props填{version,category}。多个资产放 assets 数组一次批量登记。属性写在资产自己的 props 上，探索结论不要写这里。 -- record_fact：把探索【事实/结论】写入探索图并连到意图（传 intent_id）。正向/否定结论、观察、判断用它；**一次探索的多个观察汇总成一条事实**（summary 总结一句话 + detail 放细节），不要一个属性一条。真有多条不同结论才用 facts 数组。**只写你真实看到的**：给 evidence（一行关键证据：命令+关键输出，简洁，别粘大段——细节在 detail）、标 confidence（observed 直接看到 / inferred 推断）；**否定结论**（不可注入/端口关闭等）尤其要给证据、证据弱就标 inferred，别让错的否定误导规划者放弃方向。 +- record_fact：把探索【事实/结论】写入探索图并连到意图（传 intent_id）。正向/否定结论、观察、判断用它；**一次探索的多个观察汇总成一条事实**（summary 总结一句话 + detail 放细节），不要一个属性一条。真有多条不同结论才用 facts 数组。**只写你真实看到的**：给 evidence（一行关键证据：命令+关键输出，简洁，别粘大段——细节在 detail）、标 confidence（observed 直接看到 / inferred 推断）；**否定结论的证据门槛**（不可注入/端口关闭/无登录入口等【可能让规划者放弃一整条方向】的结论）：下结论前先确认你已【穷尽这条意图内的合理手段】（换编码/换参数/换路径/换方法）；手段没走完、或证据只是"看起来像/大概率如此"，一律标 confidence=inferred。**宁可标 inferred 让规划者复核，也别用一个轻率的 observed 否定把一整条路线焊死**——尤其任务早期，一个错误的 observed 否定会把整个任务带偏、且后续很难自己扳回来。 - report_finding：确认漏洞 → 记录(含 PoC，传 intent_id=你领到的意图id)。**只有你在本次运行里真实触发过该漏洞、拿到了可复现的证据（请求/响应或命令输出）才用它。** 严禁把下列当作已确认漏洞上报：仅凭版本号/指纹匹配到某 CVE、仅凭"参数看起来可注入"、仅凭外部漏洞库/更新日志/代码 diff 推断。**不要用查 CVE 库或"对比补丁版本"替代实际触发。** 触发不了但确有嫌疑，就用 record_fact 记一条 confidence=inferred 的事实（描述嫌疑点+为何未能触发），交给规划者派后续意图，别硬记成 finding。 -- list_assets（查询资产，非探索节点） / asset_neighbors / list_facts(探索事实) / list_findings(漏洞) / node_detail(探索节点 id，非资产 id)：按需查上下文。 +- list_assets（查询资产，非探索节点） / asset_neighbors / list_facts(探索事实，分页最新在前；默认 20 条，可传 q 关键词过滤、before 翻页) / list_findings(漏洞) / node_detail(探索节点 id，非资产 id)：按需查上下文。 只在授权范围内操作；若系统提示顶部附有【操作约束】，那是最高优先级红线——任何命令/探测在执行前先自检是否违反，违反即不做（哪怕它落在你领到的意图里）。完成本意图后用一句话总结你做了什么、写回了哪些事实。务实、克制、聚焦这一条意图。` @@ -384,6 +385,7 @@ func (w *Worker) execute(ctx context.Context, name string, taskID int64, as *db. ToolOutputDir: cmdOutDir(runDir), Compaction: compactionConfig(w.compactionWindow()), // long tool-heavy runs stay within the window Todos: actool.NewTodoStore(), // 会话级临时待办（TodoWrite），纯规划用，退出即丢 + NonStreaming: w.nonStreaming(), // 该 profile 选非流式时走 Provider.Complete } if hooks != nil { // typed-nil guard: only set when concrete (avoids harness panic) opts.Hooks = hooks diff --git a/db/commands.go b/db/commands.go index 9e6e2e6..3edefa6 100644 --- a/db/commands.go +++ b/db/commands.go @@ -19,18 +19,10 @@ type CommandRecord struct { CreatedAt time.Time `json:"created_at"` } -// ListCommands returns tool executions (tool_use + paired tool_result) across all -// explorations, with optional filtering and pagination. Covers every tool, not -// just Bash; q matches the tool name or its input. -func (d *DB) ListCommands(expID *int64, q string, page, size int) ([]CommandRecord, int, error) { - if size <= 0 { - size = 50 - } - if page < 0 { - page = 0 - } - offset := page * size - +// commandFilter builds the WHERE clause shared by the tool-execution list and +// its per-tool tally, so the summary always describes exactly the rows the table +// pages through. Returns the clause, its args, and the next placeholder index. +func commandFilter(expID *int64, q string) (string, []any, int) { where := `WHERE u.kind = 'tool_use'` args := []any{} argN := 1 @@ -45,6 +37,59 @@ func (d *DB) ListCommands(expID *int64, q string, page, size int) ([]CommandReco args = append(args, "%"+q+"%") argN++ } + return where, args, argN +} + +// ToolStat is one tool's execution tally for the usage summary. +type ToolStat struct { + Tool string `json:"tool"` + Total int `json:"total"` + Errors int `json:"errors"` +} + +// ToolStats counts executions grouped by tool under the same filters +// ListCommands takes. Unpaginated on purpose: the tally describes the whole +// filtered set, not the page currently on screen. +func (d *DB) ToolStats(expID *int64, q string) ([]ToolStat, error) { + where, args, _ := commandFilter(expID, q) + + rows, err := d.Query(` +SELECT COALESCE(NULLIF(u.tool,''),'-') AS tool, COUNT(*) AS total, + COUNT(*) FILTER (WHERE COALESCE(r.is_error,false)) AS errors +FROM activity u +LEFT JOIN activity r ON r.tool_use_id = u.tool_use_id AND r.kind = 'tool_result' +`+where+` +GROUP BY 1 +ORDER BY total DESC, tool ASC`, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + out := []ToolStat{} + for rows.Next() { + var s ToolStat + if err := rows.Scan(&s.Tool, &s.Total, &s.Errors); err != nil { + return nil, err + } + out = append(out, s) + } + return out, rows.Err() +} + +// ListCommands returns tool executions (tool_use + paired tool_result) across all +// explorations, with optional filtering and pagination. Covers every tool, not +// just Bash; q matches the tool name or its input. +func (d *DB) ListCommands(expID *int64, q string, page, size int) ([]CommandRecord, int, error) { + if size <= 0 { + size = 50 + } + if page < 0 { + page = 0 + } + offset := page * size + + where, args, argN := commandFilter(expID, q) // count var total int @@ -99,6 +144,13 @@ type LLMRecord struct { Error string `json:"error,omitempty"` RequestBody string `json:"request_body,omitempty"` ResponseBody string `json:"response_body,omitempty"` + // RawRequest / RawResponse are the untouched HTTP bodies exchanged with the + // provider — the request as buildBody() sent it (full tool schemas included) + // and the raw SSE frames. RequestBody/ResponseBody above are the normalized + // view, which drops tool schemas and tool_use blocks entirely. Empty for + // records written before this was added, or when the call never reached HTTP. + RawRequest string `json:"raw_request,omitempty"` + RawResponse string `json:"raw_response,omitempty"` } const llmRecordsSchema = ` @@ -118,7 +170,9 @@ CREATE TABLE IF NOT EXISTS llm_records ( status TEXT, error TEXT, request_body TEXT, - response_body TEXT + response_body TEXT, + raw_request TEXT, + raw_response TEXT ); CREATE INDEX IF NOT EXISTS idx_llm_records_ts ON llm_records(ts); CREATE INDEX IF NOT EXISTS idx_llm_records_session ON llm_records(session_id); @@ -129,6 +183,8 @@ const llmRecordsMigrate = ` ALTER TABLE llm_records ADD COLUMN IF NOT EXISTS task_id TEXT; ALTER TABLE llm_records ADD COLUMN IF NOT EXISTS worker TEXT; ALTER TABLE llm_records ADD COLUMN IF NOT EXISTS profile_name TEXT; +ALTER TABLE llm_records ADD COLUMN IF NOT EXISTS raw_request TEXT; +ALTER TABLE llm_records ADD COLUMN IF NOT EXISTS raw_response TEXT; ` // EnsureLLMRecordsTable creates the llm_records table if it does not exist. @@ -153,11 +209,12 @@ func (d *DB) EnsureLLMRecordsTable() error { // InsertLLMRecord stores one LLM call record. func (d *DB) InsertLLMRecord(r *LLMRecord) error { _, err := d.Exec(` -INSERT INTO llm_records(model, profile_name, session_id, task_id, worker, latency_ms, input_tokens, output_tokens, cache_read, cache_write, status, error, request_body, response_body) -VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14)`, +INSERT INTO llm_records(model, profile_name, session_id, task_id, worker, latency_ms, input_tokens, output_tokens, cache_read, cache_write, status, error, request_body, response_body, raw_request, raw_response) +VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16)`, r.Model, nullIfEmpty(r.ProfileName), r.SessionID, nullIfEmpty(r.TaskID), nullIfEmpty(r.Worker), r.LatencyMs, r.InputTokens, r.OutputTokens, r.CacheRead, r.CacheWrite, - r.Status, nullIfEmpty(r.Error), nullIfEmpty(r.RequestBody), nullIfEmpty(r.ResponseBody)) + r.Status, nullIfEmpty(r.Error), nullIfEmpty(r.RequestBody), nullIfEmpty(r.ResponseBody), + nullIfEmpty(r.RawRequest), nullIfEmpty(r.RawResponse)) return err } @@ -270,19 +327,21 @@ func (d *DB) DeleteLLMRecords(task string) (int64, error) { // GetLLMRecord returns a single LLM record with full request/response bodies. func (d *DB) GetLLMRecord(id int64) (*LLMRecord, error) { var r LLMRecord - var reqBody, respBody sql.NullString + var reqBody, respBody, rawReq, rawResp sql.NullString err := d.QueryRow(`SELECT id, ts, COALESCE(model,''), COALESCE(profile_name,''), COALESCE(session_id,''), COALESCE(task_id,''), COALESCE(worker,''), COALESCE(latency_ms,0), COALESCE(input_tokens,0), COALESCE(output_tokens,0), COALESCE(cache_read,0), COALESCE(cache_write,0), - COALESCE(status,''), COALESCE(error,''), request_body, response_body + COALESCE(status,''), COALESCE(error,''), request_body, response_body, raw_request, raw_response FROM llm_records WHERE id=$1`, id). Scan(&r.ID, &r.Ts, &r.Model, &r.ProfileName, &r.SessionID, &r.TaskID, &r.Worker, &r.LatencyMs, &r.InputTokens, &r.OutputTokens, &r.CacheRead, &r.CacheWrite, &r.Status, &r.Error, - &reqBody, &respBody) + &reqBody, &respBody, &rawReq, &rawResp) if err != nil { return nil, err } r.RequestBody = reqBody.String r.ResponseBody = respBody.String + r.RawRequest = rawReq.String + r.RawResponse = rawResp.String return &r, nil } diff --git a/db/config.go b/db/config.go index 40684d6..410994a 100644 --- a/db/config.go +++ b/db/config.go @@ -39,18 +39,23 @@ type LLMProfile struct { // usable when an agent/task binds it explicitly, it just never gets picked up // as a fallback target. PoolExclude bool `json:"pool_exclude"` + // Streaming selects the wire protocol: true (default) = streaming (SSE); + // false = real non-streaming (stream:false, single JSON response via + // Provider.Complete). Non-streaming sidesteps flaky gateway SSE at the cost of + // live in-run progress. Maps to agent.Config.Stream. + Streaming bool `json:"streaming"` } // profileCols is the read column list (hint variant, no api key) shared by the // list query; profileColsKey is the same with api_key for the single-row loads. -const profileCols = `id,name,format,COALESCE(base_url,''),COALESCE(proxy,''),model,COALESCE(api_key_hint,''),rate_per_second,rate_per_minute,context_window_k,COALESCE(reasoning_effort,''),is_default,priority,pool_exclude,COALESCE(thinking_type,'')` -const profileColsKey = `id,name,format,COALESCE(base_url,''),COALESCE(proxy,''),model,COALESCE(api_key,''),rate_per_second,rate_per_minute,context_window_k,COALESCE(reasoning_effort,''),is_default,priority,pool_exclude,COALESCE(thinking_type,'')` +const profileCols = `id,name,format,COALESCE(base_url,''),COALESCE(proxy,''),model,COALESCE(api_key_hint,''),rate_per_second,rate_per_minute,context_window_k,COALESCE(reasoning_effort,''),is_default,priority,pool_exclude,COALESCE(thinking_type,''),COALESCE(streaming,true)` +const profileColsKey = `id,name,format,COALESCE(base_url,''),COALESCE(proxy,''),model,COALESCE(api_key,''),rate_per_second,rate_per_minute,context_window_k,COALESCE(reasoning_effort,''),is_default,priority,pool_exclude,COALESCE(thinking_type,''),COALESCE(streaming,true)` // scanProfile reads one row in the profileCols / profileColsKey column order. The // 7th column lands in APIKeyHint or APIKey depending on which list the caller used. func scanProfile(sc interface{ Scan(...any) error }, into *string, p *LLMProfile) error { return sc.Scan(&p.ID, &p.Name, &p.Format, &p.BaseURL, &p.Proxy, &p.Model, into, - &p.RatePerSecond, &p.RatePerMinute, &p.ContextWindowK, &p.ReasoningEffort, &p.IsDefault, &p.Priority, &p.PoolExclude, &p.ThinkingType) + &p.RatePerSecond, &p.RatePerMinute, &p.ContextWindowK, &p.ReasoningEffort, &p.IsDefault, &p.Priority, &p.PoolExclude, &p.ThinkingType, &p.Streaming) } func (d *DB) ListProfiles() ([]*LLMProfile, error) { @@ -123,18 +128,18 @@ func (d *DB) SaveProfile(p *LLMProfile) (int64, error) { } if p.ID == 0 { var id int64 - err := d.QueryRow(`INSERT INTO llm_profiles(name,format,base_url,proxy,model,api_key,api_key_hint,rate_per_second,rate_per_minute,context_window_k,reasoning_effort,priority,pool_exclude,thinking_type) -VALUES ($1,$2,NULLIF($3,''),NULLIF($4,''),$5,NULLIF($6,''),NULLIF($7,''),$8,$9,$10,$11,$12,$13,$14) RETURNING id`, - p.Name, p.Format, p.BaseURL, p.Proxy, p.Model, p.APIKey, hint, p.RatePerSecond, p.RatePerMinute, p.ContextWindowK, p.ReasoningEffort, p.Priority, p.PoolExclude, p.ThinkingType).Scan(&id) + err := d.QueryRow(`INSERT INTO llm_profiles(name,format,base_url,proxy,model,api_key,api_key_hint,rate_per_second,rate_per_minute,context_window_k,reasoning_effort,priority,pool_exclude,thinking_type,streaming) +VALUES ($1,$2,NULLIF($3,''),NULLIF($4,''),$5,NULLIF($6,''),NULLIF($7,''),$8,$9,$10,$11,$12,$13,$14,$15) RETURNING id`, + p.Name, p.Format, p.BaseURL, p.Proxy, p.Model, p.APIKey, hint, p.RatePerSecond, p.RatePerMinute, p.ContextWindowK, p.ReasoningEffort, p.Priority, p.PoolExclude, p.ThinkingType, p.Streaming).Scan(&id) return id, err } if p.APIKey == "" { - _, err := d.Exec(`UPDATE llm_profiles SET name=$1,format=$2,base_url=NULLIF($3,''),proxy=NULLIF($4,''),model=$5,rate_per_second=$6,rate_per_minute=$7,context_window_k=$8,reasoning_effort=$9,priority=$10,pool_exclude=$11,thinking_type=$12 WHERE id=$13`, - p.Name, p.Format, p.BaseURL, p.Proxy, p.Model, p.RatePerSecond, p.RatePerMinute, p.ContextWindowK, p.ReasoningEffort, p.Priority, p.PoolExclude, p.ThinkingType, p.ID) + _, err := d.Exec(`UPDATE llm_profiles SET name=$1,format=$2,base_url=NULLIF($3,''),proxy=NULLIF($4,''),model=$5,rate_per_second=$6,rate_per_minute=$7,context_window_k=$8,reasoning_effort=$9,priority=$10,pool_exclude=$11,thinking_type=$12,streaming=$13 WHERE id=$14`, + p.Name, p.Format, p.BaseURL, p.Proxy, p.Model, p.RatePerSecond, p.RatePerMinute, p.ContextWindowK, p.ReasoningEffort, p.Priority, p.PoolExclude, p.ThinkingType, p.Streaming, p.ID) return p.ID, err } - _, err := d.Exec(`UPDATE llm_profiles SET name=$1,format=$2,base_url=NULLIF($3,''),proxy=NULLIF($4,''),model=$5,api_key=$6,api_key_hint=$7,rate_per_second=$8,rate_per_minute=$9,context_window_k=$10,reasoning_effort=$11,priority=$12,pool_exclude=$13,thinking_type=$14 WHERE id=$15`, - p.Name, p.Format, p.BaseURL, p.Proxy, p.Model, p.APIKey, hint, p.RatePerSecond, p.RatePerMinute, p.ContextWindowK, p.ReasoningEffort, p.Priority, p.PoolExclude, p.ThinkingType, p.ID) + _, err := d.Exec(`UPDATE llm_profiles SET name=$1,format=$2,base_url=NULLIF($3,''),proxy=NULLIF($4,''),model=$5,api_key=$6,api_key_hint=$7,rate_per_second=$8,rate_per_minute=$9,context_window_k=$10,reasoning_effort=$11,priority=$12,pool_exclude=$13,thinking_type=$14,streaming=$15 WHERE id=$16`, + p.Name, p.Format, p.BaseURL, p.Proxy, p.Model, p.APIKey, hint, p.RatePerSecond, p.RatePerMinute, p.ContextWindowK, p.ReasoningEffort, p.Priority, p.PoolExclude, p.ThinkingType, p.Streaming, p.ID) return p.ID, err } diff --git a/db/db.go b/db/db.go index 7bf2c36..20ec514 100644 --- a/db/db.go +++ b/db/db.go @@ -434,30 +434,30 @@ func (d *DB) seedDefaultInterceptRules() error { // 2. Python HTTP 客户端 .delete() 方法 // 3. JS/通用脚本里的 method: 'DELETE' / method="DELETE" { - name: "[内置] curl / wget 发送 DELETE 请求", - target: "tool_input", - typ: "regex", - pattern: `(?i)\bcurl\b[^|\n&;"]{0,300}(?:-X\s*DELETE|--request\s+DELETE|-XDELETE)|\bwget\b[^|\n&;"]{0,300}--method[=\s]+DELETE`, - action: "deny", - message: "禁止通过 curl/wget 发送 HTTP DELETE 请求，可能删除目标系统数据", + name: "[内置] curl / wget 发送 DELETE 请求", + target: "tool_input", + typ: "regex", + pattern: `(?i)\bcurl\b[^|\n&;"]{0,300}(?:-X\s*DELETE|--request\s+DELETE|-XDELETE)|\bwget\b[^|\n&;"]{0,300}--method[=\s]+DELETE`, + action: "deny", + message: "禁止通过 curl/wget 发送 HTTP DELETE 请求，可能删除目标系统数据", priority: 80, }, { - name: "[内置] Python HTTP 客户端 DELETE（requests/httpx/aiohttp）", - target: "tool_input", - typ: "regex", - pattern: `(?i)\b(?:requests|httpx|aiohttp|urllib\.request)\.delete\s*\(|session\.delete\s*\(|client\.delete\s*\(`, - action: "deny", - message: "禁止使用 Python HTTP 客户端发送 DELETE 请求", + name: "[内置] Python HTTP 客户端 DELETE（requests/httpx/aiohttp）", + target: "tool_input", + typ: "regex", + pattern: `(?i)\b(?:requests|httpx|aiohttp|urllib\.request)\.delete\s*\(|session\.delete\s*\(|client\.delete\s*\(`, + action: "deny", + message: "禁止使用 Python HTTP 客户端发送 DELETE 请求", priority: 80, }, { - name: "[内置] 脚本中声明 HTTP DELETE 方法（JS/通用）", - target: "tool_input", - typ: "regex", - pattern: `(?i)axios\.delete\s*\(|method\s*[:=]\s*['"]DELETE['"]`, - action: "deny", - message: "禁止在脚本中声明并发送 HTTP DELETE 请求", + name: "[内置] 脚本中声明 HTTP DELETE 方法（JS/通用）", + target: "tool_input", + typ: "regex", + pattern: `(?i)axios\.delete\s*\(|method\s*[:=]\s*['"]DELETE['"]`, + action: "deny", + message: "禁止在脚本中声明并发送 HTTP DELETE 请求", priority: 80, }, { diff --git a/db/exploration.go b/db/exploration.go index 1913198..1ffb92a 100644 --- a/db/exploration.go +++ b/db/exploration.go @@ -339,7 +339,6 @@ const ( IntentInterventionPending = "pending" IntentInterventionAccepted = "accepted" IntentInterventionRejected = "rejected" - PlannerEventWorkerMessage = "worker_message" ) // IntentIntervention is the durable control record for one human interruption. @@ -356,19 +355,6 @@ type IntentIntervention struct { CreatedAt time.Time } -// WorkerDirection is the latest accepted human direction for one Worker -// intent. The accepted activity is the durable planning event, so this view -// survives restarts and is shared by Planner and Main Agent context builders. -type WorkerDirection struct { - ActivityID int64 - IntentID int64 - RequestID string - Message string - IntentState string - IntentSummary string - CreatedAt time.Time -} - // lockIntentIntervention serializes request-id reservation/finalization across // processes. exploration_nodes.id is a global primary key, so the intent id is // sufficient input to the namespaced advisory key. @@ -577,7 +563,6 @@ WHERE a.exploration_id=$1 AND a.node_id=$2 "intervention_request_id": requestID, "intervention_message": message, "intervention_status": IntentInterventionAccepted, - "planner_event": PlannerEventWorkerMessage, "source": "user", }) if err != nil { @@ -624,52 +609,6 @@ WHERE a.exploration_id=$1 AND a.node_id=$2 return activity, false, nil } -// LatestWorkerDirections returns at most one accepted human direction per -// local intent, newest directions first. It reads the visible user activity -// rather than the hidden hand-off row so archive/restore and activity history -// share one authoritative record. -func (s *ExplorationStore) LatestWorkerDirections(limit int) ([]WorkerDirection, error) { - if limit <= 0 { - limit = 300 - } - rows, err := s.db.Query(` -SELECT activity_id,intent_id,request_id,message,intent_state,intent_summary,created_at -FROM ( - SELECT DISTINCT ON (a.node_id) - a.id AS activity_id, - a.node_id AS intent_id, - COALESCE(a.metadata->>'intervention_request_id','') AS request_id, - COALESCE(a.metadata->>'intervention_message',a.detail,'') AS message, - n.state AS intent_state, - COALESCE(n.payload->>'summary','') AS intent_summary, - a.created_at - FROM activity a - JOIN exploration_nodes n ON n.id=a.node_id AND n.exploration_id=a.exploration_id - WHERE a.exploration_id=$1 - AND a.kind='user' - AND n.kind='intent' - AND a.metadata->>'intervention_status'=$2 - AND a.metadata->>'planner_event'=$3 - ORDER BY a.node_id,a.id DESC -) latest -ORDER BY activity_id DESC -LIMIT $4`, s.expID, IntentInterventionAccepted, PlannerEventWorkerMessage, limit) - if err != nil { - return nil, err - } - defer rows.Close() - out := make([]WorkerDirection, 0) - for rows.Next() { - var item WorkerDirection - if err := rows.Scan(&item.ActivityID, &item.IntentID, &item.RequestID, &item.Message, - &item.IntentState, &item.IntentSummary, &item.CreatedAt); err != nil { - return nil, err - } - out = append(out, item) - } - return out, rows.Err() -} - // RecoverableIntentInterventions returns control operations that must be // reconstructed before ordinary frontier claiming after a process restart: // pending transcript commits, plus accepted hand-offs not yet claimed once. @@ -1108,6 +1047,40 @@ ORDER BY id DESC LIMIT $4`, s.expID, kind, before, limit+1) return nodes, hasMore, nil } +// listByKindPageFiltered is ListByKindPage with an optional summary keyword +// filter (payload->>'summary' ILIKE %q%). Fetches one extra row so the caller can +// probe hasMore. Empty q = no filter. Newest-first by id. +func (s *ExplorationStore) listByKindPageFiltered(kind string, before int64, limit int, q string) ([]*Node, error) { + where := `exploration_id=$1 AND kind=$2 AND ($3 <= 0 OR id < $3)` + args := []any{s.expID, kind, before} + if q != "" { + args = append(args, "%"+q+"%") + where += fmt.Sprintf(` AND payload->>'summary' ILIKE $%d`, len(args)) + } + args = append(args, limit+1) + rows, err := s.db.Query(`SELECT `+nodeCols+` FROM exploration_nodes +WHERE `+where+` ORDER BY id DESC LIMIT $`+fmt.Sprintf("%d", len(args)), args...) + if err != nil { + return nil, err + } + defer rows.Close() + return scanNodes(rows) +} + +// countByKindFiltered counts nodes of a kind under the same summary filter, for a +// page's total. Empty q = count all of that kind. +func (s *ExplorationStore) countByKindFiltered(kind, q string) (int, error) { + where := `exploration_id=$1 AND kind=$2` + args := []any{s.expID, kind} + if q != "" { + args = append(args, "%"+q+"%") + where += fmt.Sprintf(` AND payload->>'summary' ILIKE $%d`, len(args)) + } + var n int + err := s.db.QueryRow(`SELECT COUNT(*) FROM exploration_nodes WHERE `+where, args...).Scan(&n) + return n, err +} + // GetNode returns one node of this exploration by id (nil, nil if not found). func (s *ExplorationStore) GetNode(id int64) (*Node, error) { n, err := scanNode(s.db.QueryRow(`SELECT `+nodeCols+` FROM exploration_nodes WHERE id=$1 AND exploration_id=$2`, id, s.expID)) diff --git a/db/exploration_sources.go b/db/exploration_sources.go index 3ccbb96..c12d5ba 100644 --- a/db/exploration_sources.go +++ b/db/exploration_sources.go @@ -108,6 +108,53 @@ func (s *ExplorationStore) ListByKindWithSources(kind string, limit int) ([]*Nod return out, nil } +// ListByKindPageWithSources is the paginated, keyword-filterable sibling of +// ListByKindWithSources: it returns one newest-first page (id < before, before<=0 +// = newest) spanning this exploration and its direct sources, plus hasMore and the +// filtered total across all of them. Node ids are globally unique, so merging each +// store's own page and re-sorting by id DESC yields the true global page; fetching +// limit+1 per store guarantees the merged top-`limit` is complete. +func (s *ExplorationStore) ListByKindPageWithSources(kind string, before int64, limit int, q string) (nodes []*Node, hasMore bool, total int, err error) { + if limit <= 0 { + limit = 20 + } + own, err := s.listByKindPageFiltered(kind, before, limit, q) + if err != nil { + return nil, false, 0, err + } + merged := own + total, err = s.countByKindFiltered(kind, q) + if err != nil { + return nil, false, 0, err + } + + sources, err := s.DirectSourceStores() + if err != nil { + return nil, false, 0, err + } + for _, source := range sources { + page, err := source.Store.listByKindPageFiltered(kind, before, limit, q) + if err != nil { + return nil, false, 0, err + } + for _, n := range page { + merged = append(merged, markInheritedNode(n, source.Task.TaskID)) + } + cnt, err := source.Store.countByKindFiltered(kind, q) + if err != nil { + return nil, false, 0, err + } + total += cnt + } + + sort.Slice(merged, func(i, j int) bool { return merged[i].ID > merged[j].ID }) + hasMore = len(merged) > limit + if hasMore { + merged = merged[:limit] + } + return merged, hasMore, total, nil +} + // GetNodeWithSources reads a node only when it belongs to this exploration or // one of its direct sources. Inherited nodes are tagged so tool callers can keep // them read-only and show their provenance. diff --git a/db/llm_records_migrate_test.go b/db/llm_records_migrate_test.go new file mode 100644 index 0000000..135f06f --- /dev/null +++ b/db/llm_records_migrate_test.go @@ -0,0 +1,62 @@ +package db + +import ( + "testing" +) + +// The raw-body columns were added after release, so existing installs only get +// them through llmRecordsMigrate. This runs the real migration against the dev +// PG on a table stripped back to its pre-upgrade shape, inside a transaction +// that is always rolled back — PG does transactional DDL, so nothing persists. +func TestLLMRecordsMigrateAddsRawColumnsToOldTable(t *testing.T) { + d, err := Open(testDSN(t)) + if err != nil { + t.Skipf("postgres unavailable (%v) — skipping", err) + } + defer d.Close() + if err := d.EnsureLLMRecordsTable(); err != nil { + t.Fatalf("ensure: %v", err) + } + + tx, err := d.Begin() + if err != nil { + t.Fatalf("begin: %v", err) + } + defer tx.Rollback() //nolint:errcheck // the test never commits + + // Roll the table back to how a pre-upgrade install looks. + if _, err := tx.Exec(`ALTER TABLE llm_records DROP COLUMN IF EXISTS raw_request, DROP COLUMN IF EXISTS raw_response`); err != nil { + t.Fatalf("simulate old table: %v", err) + } + if _, err := tx.Exec(llmRecordsMigrate); err != nil { + t.Fatalf("migrate: %v", err) + } + + for _, col := range []string{"raw_request", "raw_response"} { + var n int + if err := tx.QueryRow( + `SELECT count(*) FROM information_schema.columns + WHERE table_name='llm_records' AND column_name=$1`, col).Scan(&n); err != nil { + t.Fatalf("inspect %s: %v", col, err) + } + if n != 1 { + t.Errorf("column %s missing after migrate", col) + } + } + + // Re-running must stay a no-op (the migration runs on every startup). + if _, err := tx.Exec(llmRecordsMigrate); err != nil { + t.Fatalf("migrate is not idempotent: %v", err) + } + + // An insert carrying raw bodies must round-trip through the migrated table. + var got string + if err := tx.QueryRow( + `INSERT INTO llm_records(model, raw_request, raw_response) VALUES ('m','{"a":1}','data: x') + RETURNING raw_response`).Scan(&got); err != nil { + t.Fatalf("insert into migrated table: %v", err) + } + if got != "data: x" { + t.Errorf("raw_response=%q want %q", got, "data: x") + } +} diff --git a/db/schema.sql b/db/schema.sql index 5c6356d..fe6924e 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -291,7 +291,7 @@ CREATE TABLE IF NOT EXISTS settings ( CREATE TABLE IF NOT EXISTS llm_profiles ( id BIGSERIAL PRIMARY KEY, name TEXT NOT NULL UNIQUE, - format TEXT NOT NULL CHECK (format IN ('openai','anthropic')), + format TEXT NOT NULL CHECK (format IN ('openai','anthropic','openai-responses')), base_url TEXT, proxy TEXT, model TEXT NOT NULL, @@ -310,6 +310,8 @@ CREATE TABLE IF NOT EXISTS llm_profiles ( -- pool_exclude true=不作为故障转移目标(仍可被 agent/任务显式绑定使用)。 priority INTEGER NOT NULL DEFAULT 0, pool_exclude BOOLEAN NOT NULL DEFAULT false, + -- streaming=true(默认)走流式 SSE；false 走真·非流式(stream:false，一次性 JSON)。 + streaming BOOLEAN NOT NULL DEFAULT true, created_at TIMESTAMPTZ NOT NULL DEFAULT now(), updated_at TIMESTAMPTZ NOT NULL DEFAULT now() ); @@ -320,6 +322,13 @@ CREATE TRIGGER trg_llm_upd BEFORE UPDATE ON llm_profiles -- 轮询顺位/排除标记；补旧库。默认 0 / false = 全部配置都参与轮询。 ALTER TABLE llm_profiles ADD COLUMN IF NOT EXISTS priority INTEGER NOT NULL DEFAULT 0; ALTER TABLE llm_profiles ADD COLUMN IF NOT EXISTS pool_exclude BOOLEAN NOT NULL DEFAULT false; +-- 流式开关；补旧库。默认 true = 保持既有的流式行为，旧配置无感升级。 +ALTER TABLE llm_profiles ADD COLUMN IF NOT EXISTS streaming BOOLEAN NOT NULL DEFAULT true; +-- 放开 format 约束以容纳 openai-responses(OpenAI Responses API)；补旧库。 +-- 每次启动执行,幂等:先删旧 CHECK 再建含三值的新 CHECK。 +ALTER TABLE llm_profiles DROP CONSTRAINT IF EXISTS llm_profiles_format_check; +ALTER TABLE llm_profiles ADD CONSTRAINT llm_profiles_format_check + CHECK (format IN ('openai','anthropic','openai-responses')); -- 思考开关字段 thinking_type，从旧的单一 reasoning_effort 语义一次性拆分而来。 -- schema.sql 每次启动都执行，故迁移必须只跑一次：仅当该列尚不存在时才回填， diff --git a/db/task_archives.go b/db/task_archives.go index 2093975..e38a4cd 100644 --- a/db/task_archives.go +++ b/db/task_archives.go @@ -1,6 +1,7 @@ package db import ( + "bytes" "context" "database/sql" "encoding/json" @@ -436,13 +437,6 @@ func (d *DB) FailTaskArchiveJob(id int64, activeState string, cause error) error } func queryArchiveRows(q interface { -<<<<<<< Updated upstream - QueryRow(query string, args ...any) *sql.Row -}, inner string, args ...any) (json.RawMessage, error) { - var raw []byte - err := q.QueryRow(`SELECT COALESCE(jsonb_agg(to_jsonb(row_data)), '[]'::jsonb) FROM (`+inner+`) row_data`, args...).Scan(&raw) - return json.RawMessage(raw), err -======= Query(query string, args ...any) (*sql.Rows, error) }, inner string, args ...any) (json.RawMessage, int64, error) { // Do not aggregate the result in PostgreSQL. A jsonb array has a hard limit @@ -481,7 +475,6 @@ func encodeArchiveRows(rows interface { } output.WriteByte(']') return json.RawMessage(output.Bytes()), count, nil ->>>>>>> Stashed changes } func writeArchiveRows(rows interface { @@ -604,9 +597,6 @@ func (d *DB) snapshotTaskArchive(taskID int64, llmRecords io.Writer) (*TaskArchi counts := make(map[string]int64, len(queries)) streamedTables := map[string]string{} for _, query := range queries { -<<<<<<< Updated upstream - raw, err := queryArchiveRows(tx, query.query, query.args...) -======= if query.name == "llm_records" && llmRecords != nil { count, err := streamArchiveRows(tx, llmRecords, query.query, query.args...) if err != nil { @@ -618,12 +608,11 @@ func (d *DB) snapshotTaskArchive(taskID int64, llmRecords io.Writer) (*TaskArchi continue } raw, count, err := queryArchiveRows(tx, query.query, query.args...) ->>>>>>> Stashed changes if err != nil { return nil, fmt.Errorf("snapshot %s: %w", query.name, err) } tables[query.name] = raw - counts[query.name] = rawRowCount(raw) + counts[query.name] = count } assetIDs, exclusiveAssetIDs, hosts, exclusiveHosts, err := archiveAssetMetadata(tx, taskID, expID, tables["assets"]) diff --git a/db/task_archives_test.go b/db/task_archives_test.go index d2496a3..498fa97 100644 --- a/db/task_archives_test.go +++ b/db/task_archives_test.go @@ -10,8 +10,6 @@ import ( "time" ) -<<<<<<< Updated upstream -======= type archiveJSONTestRows struct { values [][]byte next int @@ -122,7 +120,6 @@ func TestTaskArchiveFormatCompatibility(t *testing.T) { } } ->>>>>>> Stashed changes func TestTaskArchiveDatabaseRoundTrip(t *testing.T) { d, err := Open(testDSN(t)) if err != nil { diff --git a/go.mod b/go.mod index d56340a..b047c4b 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,10 @@ module github.com/Autumn-27/artex go 1.26.3 require ( - github.com/Autumn-27/norma v0.2.9 + github.com/Autumn-27/norma v0.3.2 github.com/golang-jwt/jwt/v5 v5.3.1 github.com/jackc/pgx/v5 v5.10.0 + github.com/klauspost/compress v1.18.6 github.com/lqqyt2423/go-mitmproxy v1.9.2 github.com/miekg/dns v1.1.72 github.com/projectdiscovery/dnsx v1.2.3 @@ -29,7 +30,6 @@ require ( github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect - github.com/klauspost/compress v1.18.6 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/microcosm-cc/bluemonday v1.0.27 // indirect diff --git a/go.sum b/go.sum index 835017c..845ee26 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -github.com/Autumn-27/norma v0.2.9 h1:VTt8Oe7XoiV85Dc+QOPu+zQ7IykQRm2aDTEbLEzbBe4= -github.com/Autumn-27/norma v0.2.9/go.mod h1:BudWrf7iHl96FgHqFr/HwN3mKYmGjJDt+xL2ZHXfFZw= +github.com/Autumn-27/norma v0.3.2 h1:3gW84SbSrz8luW8kbi2tv3O7CLdRMQvyylCHj1QUdNk= +github.com/Autumn-27/norma v0.3.2/go.mod h1:BudWrf7iHl96FgHqFr/HwN3mKYmGjJDt+xL2ZHXfFZw= github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro= github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= diff --git a/llmpool/pool.go b/llmpool/pool.go index 30b3665..be0ddff 100644 --- a/llmpool/pool.go +++ b/llmpool/pool.go @@ -122,6 +122,44 @@ func (p *Pool) Stream(ctx context.Context, req llm.CompletionRequest) iter.Seq2[ } } +// Complete implements llm.Provider with failover for non-streaming calls. A +// non-streaming request is atomic — it never delivers partial output — so every +// failover-eligible failure lands in the safe window and the next member can be +// tried without risk of duplicated output. Mirrors Stream's health-tripping and +// chain-exhaustion behavior. +func (p *Pool) Complete(ctx context.Context, req llm.CompletionRequest) (llm.Message, string, llm.Usage, error) { + order := p.order(req) + var lastErr error + for i, m := range order { + msg, sr, usage, err := m.Prov.Complete(ctx, req) + if err == nil { + p.health.Pass(m.ID) + return msg, sr, usage, nil + } + if !shouldFailover(ctx, err) { + // Non-failover error (e.g. ctx cancel, deterministic 4xx): surface as-is + // without tripping health, matching Stream's non-failover path. + p.health.Pass(m.ID) + return llm.Message{}, "", llm.Usage{}, err + } + lastErr = err + hard := isHardFailure(err) + if p.health.Trip(m.ID, trimErr(err), hard) { + log.Printf("[llmpool] 配置 %q(%s) 已熔断：%s", m.Name, m.Model, trimErr(err)) + } + if i+1 < len(order) { + n := order[i+1] + log.Printf("[llmpool] LLM 故障转移：%q(%s) → %q(%s)，原因：%s", + m.Name, m.Model, n.Name, n.Model, trimErr(err)) + } + } + if lastErr == nil { + lastErr = ErrExhausted + } + log.Printf("[llmpool] 轮询链已耗尽(%d 个配置全部失败)，最后错误：%s", len(order), trimErr(lastErr)) + return llm.Message{}, "", llm.Usage{}, fmt.Errorf("%w：%v", ErrExhausted, lastErr) +} + // order picks the members to try, in order: skip those in a cooling-off window // and those whose context window can't hold this request, then rotate within each // equal-priority group so same-priority profiles share the load. Never returns an diff --git a/llmpool/pool_test.go b/llmpool/pool_test.go index b1e69c8..21b7568 100644 --- a/llmpool/pool_test.go +++ b/llmpool/pool_test.go @@ -55,6 +55,17 @@ func (f *fakeProv) Stream(ctx context.Context, req llm.CompletionRequest) iter.S } } +func (f *fakeProv) Complete(ctx context.Context, req llm.CompletionRequest) (llm.Message, string, llm.Usage, error) { + acc := llm.NewAccumulator() + for ev, err := range f.Stream(ctx, req) { + if err != nil { + return llm.Message{}, "", llm.Usage{}, err + } + acc.Add(ev) + } + return acc.Message(), acc.StopReason, acc.Usage, nil +} + // ok builds a provider that always succeeds with one text delta. func okProv(name, text string) *fakeProv { return &fakeProv{name: name, events: [][]llm.StreamEvent{{{Type: llm.SETextDelta, Text: text}}}} diff --git a/llmrec/capture.go b/llmrec/capture.go new file mode 100644 index 0000000..d0fea40 --- /dev/null +++ b/llmrec/capture.go @@ -0,0 +1,140 @@ +package llmrec + +import ( + "context" + "fmt" + "io" + "strings" + "sync" +) + +type captureContextKey struct{} + +// Capture collects the untouched wire bodies of one logical LLM call. The +// Recorder creates it and puts it on the context; the HTTP transport that norma +// dials through (agent.quotaAwareTransport) finds it there and fills it in. +// +// This exists because everything the Recorder itself sees is already normalized: +// llm.CompletionRequest is re-serialized rather than the body buildBody() sent, +// and the response arrives as decoded StreamEvents, not the SSE frames. For +// debugging a live provider, the bytes on the wire are the only ground truth. +// +// norma's doStream retries the request-establishment phase, so one Stream can +// issue several HTTP attempts. Each attempt is kept: the discarded ones (see +// norma/llm/retry.go, which closes non-final bodies unread) are exactly what +// makes rate-limit and gateway failures diagnosable. +// +// The transport writes from norma's stream-reading goroutine while the Recorder +// snapshots at stream end, so all state is mutex-guarded. +type Capture struct { + mu sync.Mutex + request string + attempts []*attempt +} + +type attempt struct { + status int + body strings.Builder +} + +// NewCapture returns a context carrying a fresh Capture, plus the Capture itself. +func NewCapture(ctx context.Context) (context.Context, *Capture) { + c := &Capture{} + return context.WithValue(ctx, captureContextKey{}, c), c +} + +// CaptureFrom returns the Capture attached to ctx, or nil when raw capture is +// off. Callers must tolerate nil — recording is a toggle, and non-recorded +// providers dial through the same transport. +func CaptureFrom(ctx context.Context) *Capture { + if ctx == nil { + return nil + } + c, _ := ctx.Value(captureContextKey{}).(*Capture) + return c +} + +// SetRequest stores the outgoing request body. Retries re-send identical bytes, +// so only the first attempt's body is kept. +func (c *Capture) SetRequest(body string) { + if c == nil || body == "" { + return + } + c.mu.Lock() + defer c.mu.Unlock() + if c.request == "" { + c.request = body + } +} + +// TeeResponse opens a new attempt and wraps rc so everything read from it is +// mirrored into that attempt. It tees rather than reads because a successful +// response is an SSE stream that must keep streaming to the caller. +func (c *Capture) TeeResponse(status int, rc io.ReadCloser) io.ReadCloser { + if c == nil || rc == nil { + return rc + } + a := &attempt{status: status} + c.mu.Lock() + c.attempts = append(c.attempts, a) + c.mu.Unlock() + return &teeBody{rc: rc, c: c, a: a} +} + +// RawRequest returns the request body as sent, or "" if nothing was captured. +func (c *Capture) RawRequest() string { + if c == nil { + return "" + } + c.mu.Lock() + defer c.mu.Unlock() + return c.request +} + +// RawResponse returns the response bytes as received. A single attempt yields +// the untouched original (copy-pasteable straight into a replay); multiple +// attempts are concatenated behind per-attempt header lines so a retry sequence +// stays readable. +func (c *Capture) RawResponse() string { + if c == nil { + return "" + } + c.mu.Lock() + defer c.mu.Unlock() + switch len(c.attempts) { + case 0: + return "" + case 1: + return c.attempts[0].body.String() + } + var b strings.Builder + for i, a := range c.attempts { + fmt.Fprintf(&b, "===== attempt %d/%d — HTTP %d =====\n", i+1, len(c.attempts), a.status) + body := a.body.String() + b.WriteString(body) + if !strings.HasSuffix(body, "\n") { + b.WriteByte('\n') + } + } + return b.String() +} + +// teeBody mirrors reads into a Capture attempt, guarded by the Capture's mutex +// so a snapshot taken mid-stream never races the writer. +type teeBody struct { + rc io.ReadCloser + c *Capture + a *attempt +} + +func (t *teeBody) Read(p []byte) (int, error) { + n, err := t.rc.Read(p) + if n > 0 { + t.c.mu.Lock() + t.a.body.Write(p[:n]) + t.c.mu.Unlock() + } + return n, err +} + +func (t *teeBody) Close() error { return t.rc.Close() } diff --git a/llmrec/capture_test.go b/llmrec/capture_test.go new file mode 100644 index 0000000..3386202 --- /dev/null +++ b/llmrec/capture_test.go @@ -0,0 +1,103 @@ +package llmrec + +import ( + "context" + "io" + "strings" + "testing" +) + +func TestCaptureFromNilAndMissing(t *testing.T) { + var nilCtx context.Context // the transport reaches here on any unrecorded call + if CaptureFrom(nilCtx) != nil { + t.Fatal("nil context should yield no capture") + } + if CaptureFrom(context.Background()) != nil { + t.Fatal("bare context should yield no capture") + } +} + +// A nil *Capture is the "recording off" path: the transport calls these on every +// request, so they must all be no-ops rather than panics. +func TestNilCaptureIsInert(t *testing.T) { + var c *Capture + c.SetRequest("body") + if got := c.RawRequest(); got != "" { + t.Fatalf("RawRequest()=%q want empty", got) + } + if got := c.RawResponse(); got != "" { + t.Fatalf("RawResponse()=%q want empty", got) + } + rc := io.NopCloser(strings.NewReader("x")) + if c.TeeResponse(200, rc) != rc { + t.Fatal("nil capture must pass the body through untouched") + } +} + +func TestCaptureSingleAttemptKeepsBytesVerbatim(t *testing.T) { + ctx, c := NewCapture(context.Background()) + if CaptureFrom(ctx) != c { + t.Fatal("capture not retrievable from its own context") + } + c.SetRequest(`{"model":"x"}`) + // Retries re-send identical bytes; only the first is kept. + c.SetRequest(`{"model":"ignored"}`) + + const sse = "event: message_start\ndata: {\"type\":\"message_start\"}\n\nevent: message_stop\ndata: {}\n\n" + body := c.TeeResponse(200, io.NopCloser(strings.NewReader(sse))) + got, err := io.ReadAll(body) + if err != nil { + t.Fatalf("read: %v", err) + } + if string(got) != sse { + t.Fatal("tee altered the stream delivered to the caller") + } + if err := body.Close(); err != nil { + t.Fatalf("close: %v", err) + } + + if c.RawRequest() != `{"model":"x"}` { + t.Fatalf("RawRequest()=%q", c.RawRequest()) + } + // A lone attempt must stay byte-identical — no headers, no framing added, so + // it can be replayed as-is. + if c.RawResponse() != sse { + t.Fatalf("RawResponse()=%q want verbatim SSE", c.RawResponse()) + } +} + +// norma's doStream discards the bodies of retried attempts (retry.go closes them +// unread), so the capture is the only place a 429 body survives. +func TestCaptureRetriedAttemptsAreAllKept(t *testing.T) { + _, c := NewCapture(context.Background()) + + first := c.TeeResponse(429, io.NopCloser(strings.NewReader(`{"error":"rate_limit"}`))) + if _, err := io.ReadAll(first); err != nil { + t.Fatalf("read first: %v", err) + } + second := c.TeeResponse(200, io.NopCloser(strings.NewReader("data: ok\n"))) + if _, err := io.ReadAll(second); err != nil { + t.Fatalf("read second: %v", err) + } + + raw := c.RawResponse() + if !strings.Contains(raw, `{"error":"rate_limit"}`) { + t.Fatalf("dropped the retried attempt body: %q", raw) + } + if !strings.Contains(raw, "data: ok") { + t.Fatalf("dropped the final attempt body: %q", raw) + } + if !strings.Contains(raw, "attempt 1/2 — HTTP 429") || !strings.Contains(raw, "attempt 2/2 — HTTP 200") { + t.Fatalf("attempts not delimited: %q", raw) + } + if strings.Index(raw, "rate_limit") > strings.Index(raw, "data: ok") { + t.Fatal("attempts stored out of order") + } +} + +func TestCaptureEmptyWhenNoHTTPHappened(t *testing.T) { + _, c := NewCapture(context.Background()) + if c.RawRequest() != "" || c.RawResponse() != "" { + t.Fatal("unused capture should be empty") + } +} diff --git a/llmrec/llmrec.go b/llmrec/llmrec.go index 1bbd55c..b2b71f1 100644 --- a/llmrec/llmrec.go +++ b/llmrec/llmrec.go @@ -39,10 +39,10 @@ func TaskIDFrom(ctx context.Context) string { // Recorder wraps an llm.Provider and records every completion call. type Recorder struct { - inner llm.Provider - pg *db.DB - model string // model name (from config, not in CompletionRequest) - prof string // LLM profile name (from llm_profiles) + inner llm.Provider + pg *db.DB + model string // model name (from config, not in CompletionRequest) + prof string // LLM profile name (from llm_profiles) // thinkingType / reasoningEffort 是配置级思考参数(思考开关 / 思考强度)。它们在 // norma 的 buildBody() 里从 provider 配置注入真正的 HTTP body,不出现在 // CompletionRequest 上,故 Recorder 需在此单独带一份,序列化时写进录制。 @@ -110,9 +110,15 @@ func (r *Recorder) Stream(ctx context.Context, req llm.CompletionRequest) iter.S } // Serialize the request only when storing bodies (this is the expensive part). + // Alongside it, attach a Capture so the HTTP transport can hand back the + // untouched wire bodies — the normalized view below cannot reconstruct them + // (tool schemas are dropped, tool_use blocks never reach this layer, and the + // SSE framing is already decoded). See capture.go. reqBody := "" + var capt *Capture if recordBodies { reqBody = r.serializeRequest(req) + ctx, capt = NewCapture(ctx) } return func(yield func(llm.StreamEvent, error) bool) { @@ -133,7 +139,7 @@ func (r *Recorder) Stream(ctx context.Context, req llm.CompletionRequest) iter.S r.recordUsage(taskID, expID, worker, usage, int(time.Since(start).Milliseconds()), status) // Heavy trace row — only when body recording is on. if recordBodies { - r.record(req, session, taskID, worker, reqBody, start, textBuf.String(), thinkingBuf.String(), usage, stopReason, err) + r.record(req, session, taskID, worker, reqBody, capt, start, textBuf.String(), thinkingBuf.String(), usage, stopReason, err) } } @@ -179,6 +185,48 @@ func (r *Recorder) Stream(ctx context.Context, req llm.CompletionRequest) iter.S } } +// Complete implements the atomic completion path while preserving the same +// usage metering, normalized body recording and raw wire capture as Stream. +func (r *Recorder) Complete(ctx context.Context, req llm.CompletionRequest) (llm.Message, string, llm.Usage, error) { + recordBodies := r.enabled == nil || r.enabled() + start := time.Now() + session := transcript.SessionIDFrom(ctx) + parsedID, worker := parseSession(session) + expID := db.ParseExpID(parsedID) + taskID := TaskIDFrom(ctx) + if taskID == "" { + taskID = parsedID + } + + reqBody := "" + var capt *Capture + if recordBodies { + reqBody = r.serializeRequest(req) + ctx, capt = NewCapture(ctx) + } + + msg, stopReason, usage, err := r.inner.Complete(ctx, req) + status := "ok" + if err != nil { + status = "error" + } + r.recordUsage(taskID, expID, worker, usage, int(time.Since(start).Milliseconds()), status) + if recordBodies { + r.record(req, session, taskID, worker, reqBody, capt, start, msg.Text(), thinkingText(msg), usage, stopReason, err) + } + return msg, stopReason, usage, err +} + +func thinkingText(msg llm.Message) string { + var b strings.Builder + for _, block := range msg.Content { + if block.Type == llm.BlockThinking && block.Thinking != "" { + b.WriteString(block.Thinking) + } + } + return b.String() +} + // recordUsage appends one lightweight metering row to llm_usage (no bodies). Skips // zero-token calls with no model, which carry nothing worth metering. func (r *Recorder) recordUsage(taskID string, expID int64, worker string, usage llm.Usage, latencyMs int, status string) { @@ -210,7 +258,7 @@ func (r *Recorder) recordUsage(taskID string, expID int64, worker string, usage // record persists one LLM call to PostgreSQL before the provider stream returns. // Keeping the write inside the owning task operation means task deletion can // drain calls and then remove records without a late async insert recreating one. -func (r *Recorder) record(req llm.CompletionRequest, session, taskID, worker, reqBody string, start time.Time, text, thinking string, usage llm.Usage, stopReason string, streamErr error) { +func (r *Recorder) record(req llm.CompletionRequest, session, taskID, worker, reqBody string, capt *Capture, start time.Time, text, thinking string, usage llm.Usage, stopReason string, streamErr error) { latency := int(time.Since(start).Milliseconds()) status := "ok" errMsg := "" @@ -256,6 +304,11 @@ func (r *Recorder) record(req llm.CompletionRequest, session, taskID, worker, re Error: errMsg, RequestBody: reqBody, ResponseBody: string(respBody), + // Raw wire bodies; empty when the transport did not fill the Capture + // (e.g. a provider dialing through a client without the capture hook, or + // a call that failed before any HTTP request went out). + RawRequest: capt.RawRequest(), + RawResponse: capt.RawResponse(), } if err := r.pg.InsertLLMRecord(rec); err != nil { diff --git a/llmrec/llmrec_test.go b/llmrec/llmrec_test.go index e998684..19c7d4b 100644 --- a/llmrec/llmrec_test.go +++ b/llmrec/llmrec_test.go @@ -2,9 +2,28 @@ package llmrec import ( "context" + "iter" "testing" + + "github.com/Autumn-27/norma/llm" ) +type completeProvider struct{} + +func (completeProvider) Stream(context.Context, llm.CompletionRequest) iter.Seq2[llm.StreamEvent, error] { + return func(func(llm.StreamEvent, error) bool) {} +} + +func (completeProvider) Complete(context.Context, llm.CompletionRequest) (llm.Message, string, llm.Usage, error) { + return llm.Message{ + Role: llm.RoleAssistant, + Content: []llm.ContentBlock{ + {Type: llm.BlockThinking, Thinking: "reasoning"}, + llm.TextBlock("answer"), + }, + }, "stop", llm.Usage{InputTokens: 7, OutputTokens: 3}, nil +} + func TestTaskIDContextUsesExplicitRegistryID(t *testing.T) { ctx := WithTaskID(context.Background(), " 42 ") if got := TaskIDFrom(ctx); got != "42" { @@ -27,3 +46,20 @@ func TestParseSessionFallbackIsExplorationScoped(t *testing.T) { t.Fatalf("unexpected non-session parse: (%q,%q)", taskID, worker) } } + +func TestCompleteForwardsAtomicResponse(t *testing.T) { + recorder := Wrap(completeProvider{}, nil, "model", "profile", "", "", func() bool { return false }) + msg, stopReason, usage, err := recorder.Complete(context.Background(), llm.CompletionRequest{}) + if err != nil { + t.Fatalf("Complete() error: %v", err) + } + if got := msg.Text(); got != "answer" { + t.Fatalf("Complete() text=%q want answer", got) + } + if stopReason != "stop" { + t.Fatalf("Complete() stop reason=%q want stop", stopReason) + } + if usage.InputTokens != 7 || usage.OutputTokens != 3 { + t.Fatalf("Complete() usage=%+v", usage) + } +} diff --git a/reset-password.sh b/reset-password.sh new file mode 100755 index 0000000..d48f3b2 --- /dev/null +++ b/reset-password.sh @@ -0,0 +1,235 @@ +#!/usr/bin/env bash +# ============================================================================= +# ARTEX 管理员密码重置脚本 +# +# 登录用户名固定为 ARTEX；密码以 bcrypt 哈希存放在数据库 settings 表的 +# auth.password_hash 键。本脚本连上数据库后，用 pgcrypto 在库内生成 bcrypt 哈希 +# 并写回该键——与后端登录校验（golang.org/x/crypto/bcrypt）完全兼容。 +# +# 两种部署： +# local （默认）—— 宿主机直接用 psql 连数据库。连接信息按以下优先级获取： +# 命令行参数 > --dsn/$ARTEX_PG_DSN > config.json 的 database.* +# docker —— 通过 `docker compose exec`（或 `docker exec`）在 postgres +# 容器内执行 psql（compose 默认不对宿主暴露 5432，故走容器内）。 +# +# 用法示例： +# ./reset-password.sh # 本地，自动读 config.json/环境，交互输入新密码 +# ./reset-password.sh -p 'NewPass!' # 本地，直接给定新密码 +# ./reset-password.sh --dsn postgres://u:p@h:5432/artex +# ./reset-password.sh -H 127.0.0.1 -P 5433 -U autopentest -W pass -d artex +# ./reset-password.sh -m docker # docker 部署（读 .env 的 POSTGRES_*） +# ./reset-password.sh -m docker -c pg容器名 --exec docker +# +# 安全：新密码经环境变量 + psql \getenv 传入（不进入进程 argv），并用 :'var' +# 自动转义（防 SQL 注入）；数据库密码经 PGPASSWORD 传递，同样不进 argv。 +# ============================================================================= +set -euo pipefail + +PASS_KEY="auth.password_hash" +BCRYPT_COST=10 + +MODE="" # local | docker（空=自动判定） +DSN="" +HOST="" PORT="" USER="" DBPASS="" DBNAME="" SSLMODE="" +CONFIG="" +CONTAINER="" # docker 模式的 postgres 服务/容器名（默认 postgres） +EXEC_KIND="" # compose | docker（docker 模式下用哪种 exec；空=自动） +NEWPASS="" +ASSUME_YES=0 + +die() { echo "错误：$*" >&2; exit 1; } +info() { echo "· $*" >&2; } + +usage() { sed -n '2,40p' "$0" | sed 's/^# \{0,1\}//'; exit 0; } + +# ---- 参数解析 ------------------------------------------------------------- +while [[ $# -gt 0 ]]; do + case "$1" in + -m|--mode) MODE="${2:-}"; shift 2 ;; + --dsn) DSN="${2:-}"; shift 2 ;; + -H|--host) HOST="${2:-}"; shift 2 ;; + -P|--port) PORT="${2:-}"; shift 2 ;; + -U|--user) USER="${2:-}"; shift 2 ;; + -W|--db-password) DBPASS="${2:-}"; shift 2 ;; + -d|--dbname) DBNAME="${2:-}"; shift 2 ;; + --sslmode) SSLMODE="${2:-}"; shift 2 ;; + --config) CONFIG="${2:-}"; shift 2 ;; + -c|--container) CONTAINER="${2:-}"; shift 2 ;; + --exec) EXEC_KIND="${2:-}"; shift 2 ;; + -p|--new-password) NEWPASS="${2:-}"; shift 2 ;; + -y|--yes) ASSUME_YES=1; shift ;; + -h|--help) usage ;; + *) die "未知参数：$1（-h 查看用法）" ;; + esac +done + +# ---- 从 config.json 读取 database.*（仅 local 模式、且未显式给出连接时）----- +# 优先用 python3 解析（健壮）；缺 python3 时退回 grep（config.json 为规整分字段）。 +read_config_json() { + local path="$1" + [[ -f "$path" ]] || return 1 + if command -v python3 >/dev/null 2>&1; then + python3 - "$path" <<'PY' +import json, sys +try: + d = json.load(open(sys.argv[1])).get("database", {}) +except Exception: + sys.exit(1) +# 支持直接给 dsn，或分字段 +if d.get("dsn"): + print("DSN\t" + d["dsn"]); sys.exit(0) +for k in ("host","port","user","password","dbname","sslmode"): + if d.get(k) is not None: + print(k.upper() + "\t" + str(d[k])) +PY + else + # 极简后备：逐键 grep（值为字符串或数字） + local k + for k in host port user password dbname sslmode; do + local v + v=$(grep -oE "\"$k\"[[:space:]]*:[[:space:]]*(\"[^\"]*\"|[0-9]+)" "$path" 2>/dev/null \ + | head -1 | sed -E "s/.*:[[:space:]]*//; s/^\"//; s/\"$//") || true + [[ -n "$v" ]] && echo -e "${k^^}\t$v" + done + fi +} + +apply_config_fields() { + local line key val + while IFS=$'\t' read -r key val; do + [[ -z "$key" ]] && continue + case "$key" in + DSN) [[ -z "$DSN" ]] && DSN="$val" ;; + HOST) [[ -z "$HOST" ]] && HOST="$val" ;; + PORT) [[ -z "$PORT" ]] && PORT="$val" ;; + USER) [[ -z "$USER" ]] && USER="$val" ;; + PASSWORD) [[ -z "$DBPASS" ]] && DBPASS="$val" ;; + DBNAME) [[ -z "$DBNAME" ]] && DBNAME="$val" ;; + SSLMODE) [[ -z "$SSLMODE" ]] && SSLMODE="$val" ;; + esac + done +} + +# ---- 自动判定模式 --------------------------------------------------------- +if [[ -z "$MODE" ]]; then + if [[ -n "$DSN$HOST$USER$DBNAME" || -n "${ARTEX_PG_DSN:-}" || -f "${CONFIG:-config.json}" ]]; then + MODE="local" + elif command -v docker >/dev/null 2>&1 && [[ -f docker-compose.yml ]]; then + MODE="docker" + else + MODE="local" + fi +fi +info "部署模式：$MODE" + +# ---- 采集新密码 ----------------------------------------------------------- +if [[ -z "$NEWPASS" ]]; then + read -r -s -p "输入新密码（用户名固定为 ARTEX）：" NEWPASS; echo >&2 + [[ -n "$NEWPASS" ]] || die "密码不能为空" + read -r -s -p "再次输入以确认：" NEWPASS2; echo >&2 + [[ "$NEWPASS" == "$NEWPASS2" ]] || die "两次输入不一致" +fi +[[ -n "$NEWPASS" ]] || die "密码不能为空" + +# 通过环境变量把密码交给 psql（\getenv 读取，不进入 argv/ps） +export ARTEX_RESET_NEWPASS="$NEWPASS" + +# 库内生成 bcrypt 并 upsert；密码用 :'newpw' 自动转义。CREATE EXTENSION 幂等， +# 若数据库角色无建扩展权限会在此报错（提示见下方 run 的失败分支）。 +SQL=$(cat < --dsn/$ARTEX_PG_DSN > config.json + if [[ -z "$DSN" && -z "$HOST$USER$DBNAME" ]]; then + [[ -n "${ARTEX_PG_DSN:-}" ]] && DSN="$ARTEX_PG_DSN" + fi + if [[ -z "$DSN" && -z "$HOST$USER$DBNAME" ]]; then + cfg="${CONFIG:-config.json}" + if [[ -f "$cfg" ]]; then + info "从 $cfg 读取数据库配置" + apply_config_fields < <(read_config_json "$cfg") + fi + fi + + command -v psql >/dev/null 2>&1 || die "本机未找到 psql（请安装 postgresql-client，或改用 -m docker）" + + declare -a PSQL_ARGS=() + if [[ -n "$DSN" ]]; then + PSQL_ARGS=("$DSN") + target="$DSN" + else + [[ -n "$USER" ]] || die "缺少数据库用户（-U）或有效的 config.json/DSN" + [[ -n "$DBNAME" ]] || die "缺少数据库名（-d）或有效的 config.json/DSN" + HOST="${HOST:-127.0.0.1}"; PORT="${PORT:-5432}"; SSLMODE="${SSLMODE:-disable}" + PSQL_ARGS=(-h "$HOST" -p "$PORT" -U "$USER" -d "$DBNAME") + [[ -n "$SSLMODE" ]] && export PGSSLMODE="$SSLMODE" + [[ -n "$DBPASS" ]] && export PGPASSWORD="$DBPASS" + target="$USER@$HOST:$PORT/$DBNAME" + fi + + info "目标数据库：$target" + if [[ "$ASSUME_YES" -ne 1 ]]; then + read -r -p "确认在该库重置 ARTEX 密码？[y/N] " ans + [[ "$ans" == "y" || "$ans" == "Y" ]] || die "已取消" + fi + + if ! printf '%s\n' "$SQL" | psql "${PSQL_ARGS[@]}" -v ON_ERROR_STOP=1 -q >/dev/null; then + die "写入失败。若报 pgcrypto 权限/缺失，请用具备建扩展权限的角色，或先手动执行 CREATE EXTENSION pgcrypto。" + fi + +else + # ---- docker ---- + command -v docker >/dev/null 2>&1 || die "未找到 docker" + CONTAINER="${CONTAINER:-postgres}" + + # 选择 exec 方式：优先 docker compose exec（服务名），否则 docker exec（容器名） + if [[ -z "$EXEC_KIND" ]]; then + if docker compose version >/dev/null 2>&1 && [[ -f docker-compose.yml ]]; then + EXEC_KIND="compose" + else + EXEC_KIND="docker" + fi + fi + + # 容器内的 psql 凭据：优先命令行，其次 .env 的 POSTGRES_*，再退回 compose 默认(artex) + if [[ -f .env ]]; then + # shellcheck disable=SC1091 + set -a; . ./.env; set +a + fi + DUSER="${USER:-${POSTGRES_USER:-artex}}" + DNAME="${DBNAME:-${POSTGRES_DB:-artex}}" + [[ -n "$DBPASS" ]] && export PGPASSWORD="$DBPASS" + [[ -z "${PGPASSWORD:-}" && -n "${POSTGRES_PASSWORD:-}" ]] && export PGPASSWORD="$POSTGRES_PASSWORD" + + info "目标：容器 $CONTAINER 内 psql -U $DUSER -d $DNAME（exec=$EXEC_KIND）" + if [[ "$ASSUME_YES" -ne 1 ]]; then + read -r -p "确认在该容器数据库重置 ARTEX 密码？[y/N] " ans + [[ "$ans" == "y" || "$ans" == "Y" ]] || die "已取消" + fi + + # -e 只带名字不带值 → 从当前环境继承，密码不出现在 docker 命令 argv 里。 + declare -a EXEC_CMD + if [[ "$EXEC_KIND" == "compose" ]]; then + EXEC_CMD=(docker compose exec -T -e ARTEX_RESET_NEWPASS -e PGPASSWORD "$CONTAINER" + psql -U "$DUSER" -d "$DNAME" -v ON_ERROR_STOP=1 -q) + else + EXEC_CMD=(docker exec -i -e ARTEX_RESET_NEWPASS -e PGPASSWORD "$CONTAINER" + psql -U "$DUSER" -d "$DNAME" -v ON_ERROR_STOP=1 -q) + fi + + if ! printf '%s\n' "$SQL" | "${EXEC_CMD[@]}" >/dev/null; then + die "写入失败。请确认容器名（-c）、数据库账号（.env 的 POSTGRES_*），以及角色有 pgcrypto 权限。" + fi +fi + +unset ARTEX_RESET_NEWPASS +echo "✓ 已重置 ARTEX 管理员密码。请用用户名 ARTEX + 新密码登录（无需重启服务）。" diff --git a/server/commands.go b/server/commands.go index 12904a7..aa05427 100644 --- a/server/commands.go +++ b/server/commands.go @@ -15,14 +15,7 @@ func (s *Server) pgListCommands(w http.ResponseWriter, r *http.Request) { size := atoiDefault(q.Get("size"), 50) keyword := q.Get("q") - var expID *int64 - if tv := q.Get("task"); tv != "" { - if n, err := strconv.ParseInt(tv, 10, 64); err == nil { - expID = &n - } - } - - records, total, err := s.m.PG().ListCommands(expID, keyword, page, size) + records, total, err := s.m.PG().ListCommands(commandTaskFilter(q.Get("task")), keyword, page, size) if err != nil { writeErr(w, 500, err.Error()) return @@ -30,6 +23,37 @@ func (s *Server) pgListCommands(w http.ResponseWriter, r *http.Request) { writeJSON(w, 200, map[string]any{"commands": records, "total": total}) } +// pgToolStats returns per-tool call counts for the tool-execution history, under +// the same task/keyword filters the list takes — the summary describes the whole +// filtered set, not the page on screen. +func (s *Server) pgToolStats(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + pg := s.m.PG() + if pg == nil { + writeJSON(w, 200, map[string]any{"stats": []db.ToolStat{}}) + return + } + stats, err := pg.ToolStats(commandTaskFilter(q.Get("task")), q.Get("q")) + if err != nil { + writeErr(w, 500, err.Error()) + return + } + writeJSON(w, 200, map[string]any{"stats": stats}) +} + +// commandTaskFilter parses the ?task= exploration id. Absent or unparsable → nil +// (no filter), matching the list endpoint's long-standing lenient behaviour. +func commandTaskFilter(v string) *int64 { + if v == "" { + return nil + } + n, err := strconv.ParseInt(v, 10, 64) + if err != nil { + return nil + } + return &n +} + // pgListLLMRecords returns paginated LLM call records. func (s *Server) pgListLLMRecords(w http.ResponseWriter, r *http.Request) { q := r.URL.Query() diff --git a/server/dto.go b/server/dto.go index cf7133d..c681bcf 100644 --- a/server/dto.go +++ b/server/dto.go @@ -580,6 +580,9 @@ type LLMProfileDTO struct { // pool_exclude=true 则不作为故障转移目标，但仍可被 agent/任务显式绑定。 Priority int `json:"priority"` PoolExclude bool `json:"pool_exclude"` + // 收发模式：true=流式(SSE) | false=非流式。没有 omitempty —— false 必须出现在 + // 响应里，否则前端读不到「非流式」，开关会回落成默认的流式。 + Streaming bool `json:"streaming"` } func llmProfileDTO(p *db.LLMProfile) LLMProfileDTO { @@ -599,6 +602,7 @@ func llmProfileDTO(p *db.LLMProfile) LLMProfileDTO { IsDefault: p.IsDefault, Priority: p.Priority, PoolExclude: p.PoolExclude, + Streaming: p.Streaming, } } diff --git a/server/engine.go b/server/engine.go index 2754c09..165972d 100644 --- a/server/engine.go +++ b/server/engine.go @@ -687,7 +687,6 @@ func (e *Engine) interveneWork(ctx context.Context, t *Task, intentID int64, req } if !duplicate { e.publishStoredActivity(t, activity) - t.NotifyWorkerMessage(intentID, activity.ID, message) } e.readyDirectedClaim(intentID) t.NotifyWorker() @@ -752,10 +751,6 @@ func (e *Engine) recoverWorkerMessages(ctx context.Context, t *Task, allowAdmiss log.Printf("[worker chat] task %s 恢复意图 #%d 待处理消息失败: %v", t.ID, item.IntentID, recoverErr) } case db.IntentInterventionAccepted: - // Reconstruct the Planner-facing event while the Worker hand-off is still - // pending. The durable graph_overview view remains authoritative even if - // this notification was already consumed before a later restart. - t.NotifyWorkerMessage(item.IntentID, item.ActivityID, item.Message) if item.IntentState == "paused" { if changed, setErr := t.Store.CompareAndSetIntentState(item.IntentID, "paused", "open"); setErr != nil { return setErr diff --git a/server/goals.go b/server/goals.go index a4e8a9b..05f2c7e 100644 --- a/server/goals.go +++ b/server/goals.go @@ -390,7 +390,8 @@ func (s *Server) createGoals(ctx context.Context, t *Task, emit func(db.Activity } taskID, _ := strconv.ParseInt(t.ID, 10, 64) s.engine.BeginLLMCall(t.ID) - decomposed := agent.DecomposeGoalsWithProvider(ctx, s.agentsForTask(t).runtime, s.m.dir, t.Goal, t.Description, as, t.Store, taskID, emit) + goalRuntime := s.agentsForTask(t).runtime + decomposed := agent.DecomposeGoalsWithProvider(ctx, goalRuntime, s.m.dir, t.Goal, t.Description, as, t.Store, taskID, goalRuntime.nonStreaming(), emit) s.engine.EndLLMCall(t.ID) for _, g := range decomposed { if strings.TrimSpace(g.Text) != "" { diff --git a/server/llmrec_raw_test.go b/server/llmrec_raw_test.go new file mode 100644 index 0000000..673ce1a --- /dev/null +++ b/server/llmrec_raw_test.go @@ -0,0 +1,133 @@ +package server + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/Autumn-27/artex/agent" + "github.com/Autumn-27/artex/llmrec" + "github.com/Autumn-27/norma/llm" + "github.com/Autumn-27/norma/transcript" +) + +// Full chain against a real PG: Recorder → norma provider → capturing transport +// → llm_records. The unit tests cover each hop; this proves the raw bodies +// actually survive all the way into the row an operator opens when debugging. +func TestRecorderPersistsRawWireBodies(t *testing.T) { + m, err := NewManager(t.TempDir(), "") + if err != nil { + t.Skipf("postgres unavailable (%v) — skipping", err) + } + defer m.Close() + pg := m.PG() + if err := pg.EnsureLLMRecordsTable(); err != nil { + t.Fatalf("ensure table: %v", err) + } + + sse := strings.Join([]string{ + `event: message_start`, + `data: {"type":"message_start","message":{"id":"msg_raw","usage":{"input_tokens":5,"output_tokens":1}}}`, + ``, + `event: content_block_delta`, + `data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"probing"}}`, + ``, + // A tool_use block: the normalized response body drops these entirely, so + // its presence in raw_response is the whole point of the feature. + `event: content_block_start`, + `data: {"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_raw","name":"bash"}}`, + ``, + `event: content_block_delta`, + `data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\"command\":\"id\"}"}}`, + ``, + `event: message_delta`, + `data: {"type":"message_delta","delta":{"stop_reason":"tool_use"},"usage":{"output_tokens":9}}`, + ``, + `event: message_stop`, + `data: {"type":"message_stop"}`, + ``, + }, "\n") + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("content-type", "text/event-stream") + _, _ = io.WriteString(w, sse) + })) + defer srv.Close() + + cfg := agent.Config{Format: llm.FormatAnthropic, BaseURL: srv.URL, APIKey: "k", Model: "claude-raw-test"} + inner, err := cfg.NewProvider() + if err != nil { + t.Fatalf("NewProvider: %v", err) + } + const session = "exp999999-rawtest" + rec := llmrec.Wrap(inner, pg, cfg.Model, "raw-test-profile", "", "", func() bool { return true }) + + // Clean up whatever this test writes, whether or not it passes. Must be a + // defer registered after `defer m.Close()` (LIFO puts it first) — a + // t.Cleanup would run after the manager already closed the pool. + defer func() { + if _, err := pg.Exec(`DELETE FROM llm_records WHERE session_id = $1`, session); err != nil { + t.Logf("cleanup: %v", err) + } + }() + + ctx := transcript.WithSessionID(context.Background(), session) + for _, err := range rec.Stream(ctx, llm.CompletionRequest{ + System: []string{"system prompt"}, + Messages: []llm.Message{{Role: "user", Content: []llm.ContentBlock{{Type: "text", Text: "go"}}}}, + Tools: []llm.ToolSchema{{ + Name: "bash", + Description: "run a shell command", + InputSchema: map[string]any{"type": "object"}, + }}, + MaxTokens: 512, + }) { + if err != nil { + t.Fatalf("stream: %v", err) + } + } + + var id int64 + if err := pg.QueryRow(`SELECT id FROM llm_records WHERE session_id=$1 ORDER BY id DESC LIMIT 1`, session).Scan(&id); err != nil { + t.Fatalf("no record written: %v", err) + } + got, err := pg.GetLLMRecord(id) + if err != nil { + t.Fatalf("GetLLMRecord: %v", err) + } + + if got.RawResponse != sse { + t.Errorf("raw_response is not the wire bytes:\n got: %q\nwant: %q", got.RawResponse, sse) + } + // The tool call is absent from the normalized body but present in the raw one + // — the specific gap this feature closes. + if strings.Contains(got.ResponseBody, "toolu_raw") { + t.Error("normalized body unexpectedly contains the tool_use block; test no longer proves the gap") + } + if !strings.Contains(got.RawResponse, "toolu_raw") { + t.Error("raw_response lost the tool_use block") + } + + var body map[string]any + if err := json.Unmarshal([]byte(got.RawRequest), &body); err != nil { + t.Fatalf("raw_request is not valid JSON: %v\n%s", err, got.RawRequest) + } + if body["model"] != "claude-raw-test" { + t.Errorf("raw_request model=%v", body["model"]) + } + tools, _ := body["tools"].([]any) + if len(tools) != 1 { + t.Fatalf("raw_request tools=%v", body["tools"]) + } + if tool, _ := tools[0].(map[string]any); tool["input_schema"] == nil { + t.Errorf("raw_request lost the tool schema: %v", tool) + } + // The normalized request keeps tool names only, so the schema is unique to raw. + if strings.Contains(got.RequestBody, "input_schema") { + t.Error("normalized request unexpectedly carries schemas; test no longer proves the gap") + } +} diff --git a/server/manager.go b/server/manager.go index d5c20e8..6fca4dc 100644 --- a/server/manager.go +++ b/server/manager.go @@ -1639,10 +1639,10 @@ func (t *Task) Notify() { } } -// NotifyWorker wakes one idle execution slot without itself scheduling a Planner -// round. Worker-message acceptance separately records a persistent planning event -// and calls NotifyWorkerMessage; keeping execution wake-up independent prevents -// Planner debounce from delaying the directed Worker claim. +// NotifyWorker wakes one idle execution slot without also scheduling a Planner +// round. The channel is separate from notify because human intervention needs +// the reopened intent reclaimed immediately, while graph-change notifications +// are Planner-facing and debounced. A nil channel is tolerated by test fixtures. func (t *Task) NotifyWorker() { if t == nil || t.workerWake == nil { return @@ -1653,28 +1653,6 @@ func (t *Task) NotifyWorker() { } } -// NotifyWorkerMessage immediately schedules a Planner round for an accepted, -// persistent Worker direction. ActivityID identifies the durable kind=user row -// and prevents retries in the same debounce window from duplicating the prompt. -func (t *Task) NotifyWorkerMessage(intentID, activityID int64, message string) { - message = strings.TrimSpace(message) - if intentID <= 0 || activityID <= 0 || message == "" { - return - } - t.trigMu.Lock() - for _, event := range t.pendingTriggers { - if event.Kind == agent.TriggerWorkerMessage && event.ActivityID == activityID { - t.trigMu.Unlock() - return - } - } - t.pendingTriggers = append(t.pendingTriggers, agent.TriggerEvent{ - Kind: agent.TriggerWorkerMessage, ActivityID: activityID, IntentID: intentID, Detail: message, - }) - t.trigMu.Unlock() - t.Notify() -} - // NotifyDone is Notify plus a hint: a worker just finished intentID and that is // what triggered this wake-up. The planner reads the accumulated triggers next // round so it can spell out which intent finished (+ its output). Events pile up diff --git a/server/orchestration.go b/server/orchestration.go index 9057ea7..191bde2 100644 --- a/server/orchestration.go +++ b/server/orchestration.go @@ -450,9 +450,11 @@ func (s *Server) seedOrchestrationTools() { s.seedPlannerDefaultBindings() s.seedAutoReportFindingBinding() s.unbindGoalMetDefault() - s.reseedGoalsPrompt() // goals 提示词加入「抽操作约束」步 → 旧库追加一版新默认(一次性) + s.reseedGoalsPrompt() // goals 提示词加入「抽操作约束」步 → 旧库追加一版新默认(一次性) s.reseedMainAgentPrompt() // mainagent 提示词加入「目标达成后 add_intent 反问是否建目标」(一次性) - s.seedReporterAgent() // 预置「报告撰写」agent + 工具绑定 + finding 触发器(一次性) + s.reseedPlannerPrompt() // planner 提示词:重写「0 意图」正当理由 + 加量化验收核对(一次性) + s.reseedWorkerPrompt() // worker 提示词:加否定结论证据门槛(一次性) + s.seedReporterAgent() // 预置「报告撰写」agent + 工具绑定 + finding 触发器(一次性) // 注：pentest 的默认工具绑定无需迁移——BuiltinToolSeeds 在全新初始化时就把 // list_assets/insert_assets/report_finding/list_findings/list_companies 连同 // pentest 一起 seed 好了（项目尚无旧库，不做迁移）。 @@ -464,7 +466,7 @@ func (s *Server) seedOrchestrationTools() { // reaches an old DB otherwise. Preserves each tool's agent binding + enabled flag. // Bump the flag whenever these tools' schemas/descriptions change in code. func (s *Server) refreshBuiltinToolSchemas() { - const flag = "tool_schema_refresh_v6_insert_assets_related" + const flag = "tool_schema_refresh_v7_list_facts_paging" if v, _, _ := s.m.pg.GetSetting(flag); v == "true" { return } @@ -480,7 +482,9 @@ func (s *Server) refreshBuiltinToolSchemas() { // “结束空轮”的手段、刚开跑就误判整个任务完成。 // - insert_assets：新增 related 入参(标记资产是否与当前任务相关、决定是否入覆盖度)， // SeedTool 首插入only，旧库已 seed 的 schema 否则收不到这个新参数。 - refreshBuiltin := map[string]bool{"goal_met": true, "insert_assets": true} + // - list_facts：改为分页，新增 limit/before/q 入参；旧库已 seed 的空 schema 否则 + // 在工具管理页显示「无参数」，模型也拿不到这几个参数说明。 + refreshBuiltin := map[string]bool{"goal_met": true, "insert_assets": true, "list_facts": true} for _, sd := range agent.BuiltinToolSeeds() { if !refreshBuiltin[sd.Key] { continue @@ -571,6 +575,66 @@ func (s *Server) reseedMainAgentPrompt() { log.Printf("[prompts] mainagent 提示词已追加新默认版本(加入目标达成后反问建目标,一次性)") } +// reseedPlannerPrompt 把 planner 提示词刷成【当前代码默认】——默认正文重写了「0 意图」的正当理由 +// (从笼统"最常见"改为"仅在已覆盖/等待在跑 work 依赖时才 0 意图")、并新增「量化验收核对」(覆盖度等 +// 可量化目标未达标禁止 prove_goal)。SeedPromptIfEmpty 首插入only,旧库已有版本收不到,故用版本管理 +// 【追加一个新版本】并切过去(ResetPromptToDefault),旧版本仍保留在历史里,用户若自定义过可从版本记录 +// 找回。settings flag 守卫 → 只做一次。全新库无需处理(SeedPromptIfEmpty 已 seed 最新默认)。与 +// reseedGoalsPrompt 完全同构。 +func (s *Server) reseedPlannerPrompt() { + const flag = "planner_prompt_zerointent_acceptance_v1" + if v, _, _ := s.m.pg.GetSetting(flag); v == "true" { + return + } + defer func() { _ = s.m.pg.SetSetting(flag, "true") }() // 无论成功与否只尝试一次 + a, err := s.m.pg.GetAgentByKey("planner") + if err != nil || a == nil { + return // 全新库尚未建 agent 行时,seedPrompts 会直接 seed 最新默认,无需此迁移 + } + tmpl := agent.BuiltinPromptSeeds()["planner"] + if tmpl == "" { + return + } + // 全新库 seedPrompts 已 seed 最新默认 → 当前版本已等于代码默认,不必再追加重复版本。 + if cur, err := s.m.pg.CurrentPrompt(a.ID); err == nil && cur == tmpl { + return + } + if _, err := s.m.pg.ResetPromptToDefault(a.ID, tmpl); err != nil { + log.Printf("[prompts] planner 提示词重刷为新默认失败: %v", err) + return + } + log.Printf("[prompts] planner 提示词已追加新默认版本(重写0意图理由+加量化验收核对,一次性)") +} + +// reseedWorkerPrompt 把 worker 提示词刷成【当前代码默认】——默认正文强化了「否定结论的证据门槛」 +// (未穷尽手段/证据弱一律标 inferred,别用轻率 observed 否定焊死路线)。SeedPromptIfEmpty 首插入only, +// 旧库已有版本收不到,故用版本管理【追加一个新版本】并切过去,旧版本仍保留在历史里可找回。settings flag +// 守卫 → 只做一次。全新库无需处理。与 reseedGoalsPrompt 完全同构。 +func (s *Server) reseedWorkerPrompt() { + const flag = "worker_prompt_negfact_threshold_v1" + if v, _, _ := s.m.pg.GetSetting(flag); v == "true" { + return + } + defer func() { _ = s.m.pg.SetSetting(flag, "true") }() // 无论成功与否只尝试一次 + a, err := s.m.pg.GetAgentByKey("worker") + if err != nil || a == nil { + return // 全新库尚未建 agent 行时,seedPrompts 会直接 seed 最新默认,无需此迁移 + } + tmpl := agent.BuiltinPromptSeeds()["worker"] + if tmpl == "" { + return + } + // 全新库 seedPrompts 已 seed 最新默认 → 当前版本已等于代码默认,不必再追加重复版本。 + if cur, err := s.m.pg.CurrentPrompt(a.ID); err == nil && cur == tmpl { + return + } + if _, err := s.m.pg.ResetPromptToDefault(a.ID, tmpl); err != nil { + log.Printf("[prompts] worker 提示词重刷为新默认失败: %v", err) + return + } + log.Printf("[prompts] worker 提示词已追加新默认版本(加否定结论证据门槛,一次性)") +} + // seedReporterAgent 预置一个「报告撰写」自定义 agent(builtin=false，可在 UI 编辑/删除)： // 绑定 update_finding_report + 任务查询工具，并挂一个「report_finding 被调用即触发」的 // 触发器 —— 每登记一个漏洞就唤起它写详细报告。一次性(settings flag 守卫)：用户删掉后不再重建。 diff --git a/server/server.go b/server/server.go index fe798b8..b6a4182 100644 --- a/server/server.go +++ b/server/server.go @@ -314,6 +314,7 @@ func (s *Server) loadLLMConfig() (agent.Config, bool) { cfg.ContextWindowK = p.ContextWindowK cfg.ThinkingType = p.ThinkingType cfg.ReasoningEffort = p.ReasoningEffort + cfg.Stream = p.Streaming if cfg.APIKey == "" { return cfg, false } @@ -325,20 +326,20 @@ func (s *Server) loadLLMConfig() (agent.Config, bool) { // saveLLMConfig persists the LLM config as the active "default" profile in PG. func (s *Server) saveLLMConfig(cfg agent.Config) error { + // cfg.Provider() already returns one of the three valid format strings + // (anthropic / openai / openai-responses), matching the DB CHECK constraint. format := cfg.Provider() - if format != "anthropic" { - format = "openai" - } var id int64 // agent.Config carries no failover fields, so carry the stored ones forward — - // otherwise this legacy endpoint would silently reset the profile's priority - // and pool_exclude to zero values on every save. + // otherwise this legacy endpoint would silently reset the profile's priority, + // pool_exclude, and streaming to zero values on every save. var priority int var poolExclude bool + streaming := true // 旧库/新建默认流式 if profs, _ := s.m.pg.ListProfiles(); profs != nil { for _, p := range profs { if p.Name == "default" { - id, priority, poolExclude = p.ID, p.Priority, p.PoolExclude + id, priority, poolExclude, streaming = p.ID, p.Priority, p.PoolExclude, p.Streaming break } } @@ -347,7 +348,7 @@ func (s *Server) saveLLMConfig(cfg agent.Config) error { ID: id, Name: "default", Format: format, Model: cfg.Model, BaseURL: cfg.BaseURL, Proxy: cfg.Proxy, APIKey: cfg.APIKey, RatePerSecond: cfg.RatePerSecond, RatePerMinute: cfg.RatePerMinute, ContextWindowK: cfg.ContextWindowK, ThinkingType: cfg.ThinkingType, ReasoningEffort: cfg.ReasoningEffort, IsDefault: true, - Priority: priority, PoolExclude: poolExclude, + Priority: priority, PoolExclude: poolExclude, Streaming: streaming, }) if err != nil { return err @@ -397,6 +398,7 @@ func (s *Server) buildPlannerWorker(pinID *int64, gProv llm.Provider, gCfg agent wk.SetMemory(memory.NewStore(filepath.Join(s.m.dir, "memory"))) wk.SetWebSearch(s.webSearchFor("worker")) wk.SetConstraintInject(s.constraintInjectWorker) // 操作约束注入 worker(可配置,默认开;每轮读) + wk.SetNonStreaming(nonStreamingResolver(wCfg)) // 该 profile 选非流式时走 Provider.Complete pProv, pCfg := s.providerForAgent("planner", pinID, gProv, gCfg) pl := agent.NewPlanner(pProv, pCfg.Model, s.m.dir, tx, pCfg.CompactionWindow(), s.agentMaxTurns("planner")) pl.SetKillWork(s.engine.KillWork) // planner kill_work → terminate a running work @@ -404,9 +406,18 @@ func (s *Server) buildPlannerWorker(pinID *int64, gProv llm.Provider, gCfg agent pl.SetProxy(s.m.ProxyAddr(), s.m.ProxyCACert()) // WebFetch through the recording proxy pl.SetWebSearch(s.webSearchFor("planner")) pl.SetConstraintInject(s.constraintInjectPlanner) // 操作约束注入 planner(可配置,默认开;每轮读) + pl.SetNonStreaming(nonStreamingResolver(pCfg)) // 该 profile 选非流式时走 Provider.Complete return pl, wk } +// nonStreamingResolver returns a resolver capturing a profile's streaming choice. +// The agents read it per run; a profile change rebuilds the agents (applyLLM), +// so the captured value is always the one in effect for this build. +func nonStreamingResolver(cfg agent.Config) func() bool { + nonStreaming := !cfg.Stream + return func() bool { return nonStreaming } +} + // applyLLM (re)builds the planner/worker/main-agent from cfg and installs them on // the running engine as the GLOBAL active pair. Safe to call at runtime (UI configures LLM). func (s *Server) applyLLM(cfg agent.Config) error { @@ -450,12 +461,14 @@ func (s *Server) applyLLM(cfg agent.Config) error { s.mainAgent.SetProxy(s.m.ProxyAddr(), s.m.ProxyCACert()) // WebFetch through the recording proxy s.mainAgent.SetWebSearch(s.webSearchFor("mainagent")) s.mainAgent.SetSteerWork(s.engine.SteerWork) // steer_work：人对运行中 work 实时纠偏 + s.mainAgent.SetNonStreaming(nonStreamingResolver(mCfg)) // chat agent serves MANY custom agents by key → it holds the GLOBAL opts // (backend/key) and gates Enabled per-conversation-agent at Chat time. 对话始终用激活配置。 s.chatAgent = agent.NewChatAgent(prov, cfg.Model, s.m.dir, tx, win) // chat page runner s.chatAgent.SetProxy(s.m.ProxyAddr(), s.m.ProxyCACert()) s.chatAgent.SetWebSearch(s.m.WebSearchOpts()) s.chatAgent.SetGuard(s.chatGuard()) + s.chatAgent.SetNonStreaming(nonStreamingResolver(cfg)) s.llmProv = prov s.llmCfg = cfg s.llmOn = true @@ -486,6 +499,7 @@ func (s *Server) loadProfileConfig(id int64) (agent.Config, bool) { cfg.ContextWindowK = p.ContextWindowK cfg.ThinkingType = p.ThinkingType cfg.ReasoningEffort = p.ReasoningEffort + cfg.Stream = p.Streaming if cfg.APIKey == "" { return cfg, false } @@ -637,6 +651,7 @@ func (s *Server) chatAgentForProfile(id int64) *agent.ChatAgent { ca.SetProxy(s.m.ProxyAddr(), s.m.ProxyCACert()) ca.SetWebSearch(s.m.WebSearchOpts()) ca.SetGuard(s.chatGuard()) + ca.SetNonStreaming(nonStreamingResolver(cfg)) s.profMu.Lock() if ex := s.profChatAgents[id]; ex != nil { // lost the race → keep the winner ca = ex @@ -795,6 +810,7 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("GET /api/traffic/exchange", s.getTrafficExchange) mux.HandleFunc("GET /api/traffic/blob", s.getTrafficBlob) mux.HandleFunc("GET /api/commands", s.pgListCommands) + mux.HandleFunc("GET /api/commands/stats", s.pgToolStats) // 按工具聚合调用次数 mux.HandleFunc("GET /api/llm/records", s.pgListLLMRecords) mux.HandleFunc("DELETE /api/llm/records", s.pgDeleteLLMRecords) mux.HandleFunc("GET /api/llm/records/tasks", s.pgLLMTasks) @@ -1344,6 +1360,7 @@ func (s *Server) testLLM(w http.ResponseWriter, r *http.Request) { ThinkingType string `json:"thinking_type"` ReasoningEffort string `json:"reasoning_effort"` ProfileID *int64 `json:"profile_id"` // 测已存 profile 时传入：api_key 为空则用它存的 key + Streaming *bool `json:"streaming"` // 省略=流式，与保存 profile 时同一套默认 } if err := json.NewDecoder(r.Body).Decode(&req); err != nil { writeErr(w, 400, err.Error()) @@ -1354,6 +1371,11 @@ func (s *Server) testLLM(w http.ResponseWriter, r *http.Request) { // reasoning_effort/thinking field fails the test too (no false "test ok, run 400"). cfg.ThinkingType = req.ThinkingType cfg.ReasoningEffort = req.ReasoningEffort + // 同理，收发模式也照该 profile 的选择来：只支持其中一种通道的端点必须在这里就 + // 暴露，而不是等会话里才发现"测试通过的配置根本跑不动"。 + if req.Streaming != nil { + cfg.Stream = *req.Streaming + } // API Key 解析优先级：表单输入 > 指定 profile 存的 key > 全局配置的 key。 // 已存 profile 的 key 不回传浏览器，所以测试已存配置时表单为空，需从 DB 取。 if cfg.APIKey == "" && req.ProfileID != nil { @@ -1370,12 +1392,27 @@ func (s *Server) testLLM(w http.ResponseWriter, r *http.Request) { writeJSON(w, 200, map[string]any{"ok": false, "error": "未提供 API Key"}) return } - lat, err := agent.TestConnection(r.Context(), cfg) + lat, reply, err := agent.TestConnection(r.Context(), cfg) if err != nil { writeJSON(w, 200, map[string]any{"ok": false, "error": err.Error()}) return } - writeJSON(w, 200, map[string]any{"ok": true, "latency_ms": lat.Milliseconds(), "model": cfg.Model}) + // 回传模型实际回复，让"测试通过"有据可查：看得见它确实说了话，而不只是 HTTP 200。 + writeJSON(w, 200, map[string]any{ + "ok": true, "latency_ms": lat.Milliseconds(), "model": cfg.Model, "reply": truncateReply(reply), + }) +} + +// truncateReply clips a connection-test reply for display. A model told to answer +// "OK" can still ramble (or think out loud); the UI only needs enough to show it +// really said something. Rune-based so multibyte text never splits mid-character. +func truncateReply(s string) string { + const max = 200 + r := []rune(s) + if len(r) <= max { + return s + } + return string(r[:max]) + "…" } type createTaskReq struct { diff --git a/server/server_mgmt.go b/server/server_mgmt.go index 3b2fad7..8eced23 100644 --- a/server/server_mgmt.go +++ b/server/server_mgmt.go @@ -1736,10 +1736,14 @@ func (s *Server) pgSaveProfile(w http.ResponseWriter, r *http.Request) { return } // APIKey has json:"-" on db.LLMProfile so it is never leaked to the UI on - // read; accept it here via a sibling field for create/update. + // read; accept it here via a sibling field for create/update. Streaming is a + // *bool shadowing the embedded json:"streaming": an absent field must default + // to streaming (true), which a plain bool zero value (false = non-streaming) + // would get wrong — legacy/partial clients that never send it stay streaming. var body struct { db.LLMProfile - APIKey string `json:"api_key"` + APIKey string `json:"api_key"` + Streaming *bool `json:"streaming"` } if err := decode(r, &body); err != nil { writeErr(w, 400, err.Error()) @@ -1747,6 +1751,7 @@ func (s *Server) pgSaveProfile(w http.ResponseWriter, r *http.Request) { } p := body.LLMProfile p.APIKey = body.APIKey + p.Streaming = body.Streaming == nil || *body.Streaming id, err := pg.SaveProfile(&p) if err != nil { writeErr(w, 500, err.Error()) @@ -1921,11 +1926,13 @@ func (s *Server) pgListModels(w http.ResponseWriter, r *http.Request) { var candidates []candidate switch provider { - case "openai": + case "openai", "openai-responses": + // Responses API shares the OpenAI model list at /v1/models; tolerate a full + // endpoint URL of either format. if baseURL == "" { baseURL = "https://api.openai.com/v1" } - b := strings.TrimRight(strings.TrimSuffix(baseURL, "/chat/completions"), "/") + b := strings.TrimRight(strings.TrimSuffix(strings.TrimSuffix(baseURL, "/chat/completions"), "/responses"), "/") candidates = append(candidates, candidate{b + "/models", bearerHdr()}) default: // anthropic if baseURL == "" { diff --git a/server/task_archive_package.go b/server/task_archive_package.go index bc55617..a07631f 100644 --- a/server/task_archive_package.go +++ b/server/task_archive_package.go @@ -96,6 +96,17 @@ func stageTaskArchiveFiles(dataDir string, archiveID int64, taskID string, explo if err := json.Unmarshal(raw, &stage.journal); err != nil { return nil, fmt.Errorf("read task archive staging journal: %w", err) } + // The journal is written before the first rename, so its presence does not mean + // the payload is complete: a previous round may have failed mid-loop with a + // rollback that itself errored, which leaves the root in place. Replay the moves + // rather than packaging a payload that is missing transcripts or workspace files. + if err := os.MkdirAll(filepath.Join(stage.payload, "files"), archiveDirMode); err != nil { + return nil, err + } + if err := stage.applyMoves(); err != nil { + _ = stage.rollback() + return nil, err + } return stage, nil } else if !os.IsNotExist(err) { return nil, err @@ -133,20 +144,42 @@ func stageTaskArchiveFiles(dataDir string, archiveID int64, taskID string, explo _ = stage.rollback() return nil, err } - for _, move := range targets { - destination := filepath.Join(stage.payload, move.Relative) - if err := os.MkdirAll(filepath.Dir(destination), archiveDirMode); err != nil { - _ = stage.rollback() - return nil, err - } - if err := os.Rename(move.Source, destination); err != nil { - _ = stage.rollback() - return nil, fmt.Errorf("stage task archive path %s: %w", move.Source, err) - } + if err := stage.applyMoves(); err != nil { + _ = stage.rollback() + return nil, err } return stage, nil } +// applyMoves performs the renames recorded in the journal. Entries already sitting +// in the payload are skipped, so an interrupted staging round can be resumed in +// place without moving anything twice. +func (s *taskArchiveFileStage) applyMoves() error { + for _, move := range s.journal.Moves { + destination := filepath.Join(s.payload, move.Relative) + if _, err := os.Lstat(destination); err == nil { + continue + } else if !os.IsNotExist(err) { + return err + } + if _, err := os.Lstat(move.Source); err != nil { + if os.IsNotExist(err) { + // Neither side exists: the path was removed outside the archive flow + // after the journal was written. Nothing can be staged for it. + continue + } + return err + } + if err := os.MkdirAll(filepath.Dir(destination), archiveDirMode); err != nil { + return err + } + if err := os.Rename(move.Source, destination); err != nil { + return fmt.Errorf("stage task archive path %s: %w", move.Source, err) + } + } + return nil +} + func (s *taskArchiveFileStage) rollback() error { if s == nil || s.done { return nil @@ -368,6 +401,18 @@ func recoverTaskArchiveStages(dataDir string, pg *pgdb.DB) error { } root := filepath.Join(parent, entry.Name()) raw, err := os.ReadFile(filepath.Join(root, "journal.json")) + if os.IsNotExist(err) { + // Either staging died between creating the directory tree and writing the + // journal — no rename had run, so payload holds nothing — or commit/rollback + // failed part-way through removing the root, in which case payload only holds + // copies already inside the package. Without a journal there is nothing to + // roll back, and reporting an error here keeps the archive worker from ever + // starting, so discard the directory instead. + if err := os.RemoveAll(root); err != nil { + errs = append(errs, err) + } + continue + } if err != nil { errs = append(errs, err) continue diff --git a/server/task_llm.go b/server/task_llm.go index 01a94b7..5a22600 100644 --- a/server/task_llm.go +++ b/server/task_llm.go @@ -139,6 +139,36 @@ func (r *taskLLMRuntime) current() (*Task, int64, int64, llm.Provider, error) { return t, 0, pt.LLMChainRevision, prov, nil } +// nonStreaming reports whether the task's currently-active LLM source is set to +// non-streaming. It mirrors current()'s source precedence (agent binding → +// active chain profile → global) but reads only the streaming flag. Read-only +// and best-effort: any resolution error falls back to streaming (false), the +// safe default. If failover switches to a profile with a different streaming +// setting, the change takes effect on the next agent run (a fresh Session is +// built per run in captureRun). +func (r *taskLLMRuntime) nonStreaming() bool { + taskNum, err := parseTaskID(r.taskID) + if err != nil { + return false + } + pt, err := r.s.m.pg.GetTask(taskNum) + if err != nil || pt == nil { + return false + } + if _, cfg, ok := r.s.agentBindingProvider(r.agentKey); ok { + return !cfg.Stream + } + if len(pt.LLMProfileIDs) > 0 && pt.ActiveLLMProfileID != nil { + if _, cfg, ok := r.s.providerForProfile(*pt.ActiveLLMProfileID); ok { + return !cfg.Stream + } + } + if _, cfg, ok := r.s.globalProvider(); ok { + return !cfg.Stream + } + return false +} + func parseTaskID(id string) (int64, error) { n, err := strconv.ParseInt(id, 10, 64) if err != nil || n <= 0 { @@ -147,9 +177,11 @@ func parseTaskID(id string) (int64, error) { return n, nil } -func (r *taskLLMRuntime) Stream(ctx context.Context, req llm.CompletionRequest) iter.Seq2[llm.StreamEvent, error] { - ctx = llmrec.WithTaskID(ctx, r.taskID) - hooks := taskLLMStreamHooks{ +// streamHooks builds the failover/exhaustion callbacks shared by Stream and +// Complete: how to read the current profile selection, how to mark it quota +// exhausted, and how to emit a failover transition. +func (r *taskLLMRuntime) streamHooks() taskLLMStreamHooks { + return taskLLMStreamHooks{ current: func() (taskLLMSelection, error) { task, profileID, revision, provider, err := r.current() return taskLLMSelection{task: task, profileID: profileID, revision: revision, provider: provider}, err @@ -169,7 +201,70 @@ func (r *taskLLMRuntime) Stream(ctx context.Context, req llm.CompletionRequest) r.s.emitTaskLLMTransition(selection.task, transition, cause) }, } - return streamTaskLLM(ctx, r.taskID, req, hooks) +} + +func (r *taskLLMRuntime) Stream(ctx context.Context, req llm.CompletionRequest) iter.Seq2[llm.StreamEvent, error] { + ctx = llmrec.WithTaskID(ctx, r.taskID) + return streamTaskLLM(ctx, r.taskID, req, r.streamHooks()) +} + +// Complete is the non-streaming counterpart of Stream. A non-streaming call is +// atomic — it never delivers partial output — so every failure is safe to retry +// on the same provider or fail over to the next profile without risking +// duplicated model output or tool execution (the "committed" bookkeeping the +// streaming path needs is unnecessary here). +func (r *taskLLMRuntime) Complete(ctx context.Context, req llm.CompletionRequest) (llm.Message, string, llm.Usage, error) { + ctx = llmrec.WithTaskID(ctx, r.taskID) + return completeTaskLLM(ctx, r.taskID, req, r.streamHooks()) +} + +func completeTaskLLM(ctx context.Context, taskID string, req llm.CompletionRequest, hooks taskLLMStreamHooks) (llm.Message, string, llm.Usage, error) { + for { + selection, err := hooks.current() + if err != nil { + return llm.Message{}, "", llm.Usage{}, err + } + var ( + msg llm.Message + sr string + usage llm.Usage + callErr error + ) + // 同 provider 安全窗口重试:非流式调用要么整体成功、要么整体失败,没有 + // 中途已交付输出的问题,所以任何瞬时失败都可原样重试。 + for attempt := 0; ; attempt++ { + msg, sr, usage, callErr = selection.provider.Complete(ctx, req) + if callErr != nil && ctx.Err() == nil && + attempt < sameProviderStreamRetries && isRetryableStreamError(callErr) { + backoff := sameProviderRetryBackoff(attempt) + log.Printf("[task-llm] task %s 非流式调用失败,%v 后同 provider 重试 (%d/%d): %v", + taskID, backoff, attempt+1, sameProviderStreamRetries, callErr) + if sleepCtx(ctx, backoff) { + break // 退避期间 ctx 取消 → 停止重试 + } + continue + } + break + } + if callErr == nil { + return msg, sr, usage, nil + } + // profileID=0: 显式链已被清空;非额度错误:透传。二者都不改任务 failover 状态。 + if selection.profileID == 0 || !isQuotaExhaustedError(callErr) { + return llm.Message{}, "", llm.Usage{}, callErr + } + transition, markErr := hooks.exhaust(selection, callErr) + if markErr != nil { + return llm.Message{}, "", llm.Usage{}, fmt.Errorf("mark profile quota exhausted after %v: %w", callErr, markErr) + } + if transition.Advanced && !transition.Stale && hooks.transition != nil { + hooks.transition(selection, transition, callErr) + } + if !transition.Stale && transition.NextProfileID == nil { + return llm.Message{}, "", llm.Usage{}, &taskLLMError{taskID: taskID, chainExhausted: transition.ChainExhausted, cause: callErr} + } + // 没有任何输出交付给调用方,换下一个 profile 重放同一逻辑请求是安全的。 + } } func streamTaskLLM(ctx context.Context, taskID string, req llm.CompletionRequest, hooks taskLLMStreamHooks) iter.Seq2[llm.StreamEvent, error] { @@ -435,6 +530,7 @@ func (s *Server) agentsForTask(t *Task) *taskAgentBundle { window := workerRuntime.CompactionWindow() wk := agent.NewWorker(workerRuntime, "task-router", s.m.dir, tx, window, s.agentMaxTurns("worker")) wk.SetCompactionWindowResolver(workerRuntime.CompactionWindow) + wk.SetNonStreaming(workerRuntime.nonStreaming) // 按任务当前激活 profile 的流式开关(每轮读) wk.SetRunTimeout(time.Duration(s.agentRunSeconds("worker")) * time.Second) wk.SetProxy(s.m.ProxyAddr(), s.m.ProxyCACert()) wk.SetMemory(memory.NewStore(filepath.Join(s.m.dir, "memory"))) @@ -442,6 +538,7 @@ func (s *Server) agentsForTask(t *Task) *taskAgentBundle { wk.SetConstraintInject(s.constraintInjectWorker) // 操作约束注入 worker(可配置,默认开) pl := agent.NewPlanner(plannerRuntime, "task-router", s.m.dir, tx, plannerRuntime.CompactionWindow(), s.agentMaxTurns("planner")) pl.SetCompactionWindowResolver(plannerRuntime.CompactionWindow) + pl.SetNonStreaming(plannerRuntime.nonStreaming) pl.SetKillWork(s.engine.KillWork) pl.SetSteerWork(s.engine.SteerWork) pl.SetProxy(s.m.ProxyAddr(), s.m.ProxyCACert()) @@ -449,6 +546,7 @@ func (s *Server) agentsForTask(t *Task) *taskAgentBundle { pl.SetConstraintInject(s.constraintInjectPlanner) // 操作约束注入 planner(可配置,默认开) main := agent.NewMainAgent(mainRuntime, "task-router", s.m.dir, tx, mainRuntime.CompactionWindow(), s.agentMaxTurns("mainagent")) main.SetCompactionWindowResolver(mainRuntime.CompactionWindow) + main.SetNonStreaming(mainRuntime.nonStreaming) main.SetProxy(s.m.ProxyAddr(), s.m.ProxyCACert()) main.SetWebSearch(s.webSearchFor("mainagent")) main.SetSteerWork(s.engine.SteerWork) // steer_work：人对运行中 work 实时纠偏 diff --git a/server/task_llm_test.go b/server/task_llm_test.go index 48da7d3..5316b9b 100644 --- a/server/task_llm_test.go +++ b/server/task_llm_test.go @@ -37,6 +37,23 @@ func (p *scriptedLLMProvider) Stream(context.Context, llm.CompletionRequest) ite } } +func (p *scriptedLLMProvider) Complete(ctx context.Context, req llm.CompletionRequest) (llm.Message, string, llm.Usage, error) { + return accumulateStreamForTest(ctx, p.Stream, req) +} + +// accumulateStreamForTest drains a mock's Stream to satisfy the non-streaming +// Complete method. +func accumulateStreamForTest(ctx context.Context, stream func(context.Context, llm.CompletionRequest) iter.Seq2[llm.StreamEvent, error], req llm.CompletionRequest) (llm.Message, string, llm.Usage, error) { + acc := llm.NewAccumulator() + for ev, err := range stream(ctx, req) { + if err != nil { + return llm.Message{}, "", llm.Usage{}, err + } + acc.Add(ev) + } + return acc.Message(), acc.StopReason, acc.Usage, nil +} + // withZeroRetryBackoff sets the same-provider retry backoff to zero for the // duration of a (serial) test and returns a restore func for defer. func withZeroRetryBackoff() func() { @@ -73,6 +90,10 @@ func (p *flakyThenOKProvider) Stream(context.Context, llm.CompletionRequest) ite } } +func (p *flakyThenOKProvider) Complete(ctx context.Context, req llm.CompletionRequest) (llm.Message, string, llm.Usage, error) { + return accumulateStreamForTest(ctx, p.Stream, req) +} + func collectTaskLLMStream(seq iter.Seq2[llm.StreamEvent, error]) ([]llm.StreamEvent, error) { var events []llm.StreamEvent var streamErr error diff --git a/traffic/traffic.go b/traffic/traffic.go index 6dfbce3..d7a2b7f 100644 --- a/traffic/traffic.go +++ b/traffic/traffic.go @@ -911,7 +911,12 @@ func (t *Traffic) stageTreesForArchive(dirs, hosts []string, archiveID, taskID i } planned = append(planned, stagedTrafficPath{source: source}) } - if len(planned) == 0 { + // 归档路径即使一个历史 host 目录都没有(新装机的流量只落在 SQLite + _blobs) + // 也必须留下 journal：崩溃点若落在 PostgreSQL 提交与 SQLite 提交之间，重启后 + // SQLite 事务被回滚，只有这份 journal 能让恢复流程补做 host 行的删除。少了它， + // 已转冷任务的独占流量会永久留在热库里。 + uniqueHosts := uniqueArchiveHosts(hosts) + if len(planned) == 0 && len(uniqueHosts) == 0 { return "", nil, nil } parent := filepath.Join(t.dir, "_delete_staging") @@ -921,7 +926,7 @@ func (t *Traffic) stageTreesForArchive(dirs, hosts []string, archiveID, taskID i if stageDir, err = os.MkdirTemp(parent, "hosts-"); err != nil { return "", nil, fmt.Errorf("创建流量暂存目录: %w", err) } - journal := hostDeleteStageJournal{Version: 1, ArchiveID: archiveID, TaskID: taskID, Hosts: uniqueArchiveHosts(hosts)} + journal := hostDeleteStageJournal{Version: 1, ArchiveID: archiveID, TaskID: taskID, Hosts: uniqueHosts} for i := range planned { planned[i].staged = filepath.Join(stageDir, fmt.Sprintf("%d-%s", i, filepath.Base(planned[i].source))) journal.Moves = append(journal.Moves, hostDeleteStageMove{Source: planned[i].source, Staged: planned[i].staged}) diff --git a/web/src/app/(main)/chat/page.tsx b/web/src/app/(main)/chat/page.tsx index 57f6f0d..90ba8a0 100644 --- a/web/src/app/(main)/chat/page.tsx +++ b/web/src/app/(main)/chat/page.tsx @@ -6,6 +6,7 @@ import { ArrowUpIcon, Bot, ChevronDownIcon, + ListChecksIcon, Loader2Icon, MoreHorizontalIcon, PaperclipIcon, @@ -763,6 +764,7 @@ const ConversationItem = React.memo(function ConversationItem({ onCancelRename, onTogglePinned, onDelete, + selectionMode, selectedForDelete, onSelectedForDeleteChange, }: { @@ -778,6 +780,7 @@ const ConversationItem = React.memo(function ConversationItem({ onCancelRename: () => void; onTogglePinned: (conversation: Conversation) => void; onDelete: (id: number) => void; + selectionMode: boolean; selectedForDelete: boolean; onSelectedForDeleteChange: (id: number, checked: boolean) => void; }) { @@ -800,12 +803,14 @@ const ConversationItem = React.memo(function ConversationItem({ active ? "bg-accent text-accent-foreground" : "hover:bg-accent/50", )} > - onSelectedForDeleteChange(conv.id, checked === true)} - aria-label={`选择对话「${conv.title || "新对话"}」`} - className="ml-1 shrink-0" - /> + {selectionMode && ( + onSelectedForDeleteChange(conv.id, checked === true)} + aria-label={`选择对话「${conv.title || "新对话"}」`} + className="ml-1 shrink-0" + /> + )} {renaming ? ( (null); const [renameText, setRenameText] = React.useState(""); const [selectedConversationIds, setSelectedConversationIds] = React.useState>(() => new Set()); + // selectionMode gates the multi-select UI: off by default (clean list, no + // checkboxes); the header "多选" button turns it on, "完成" turns it off and + // clears the selection. + const [selectionMode, setSelectionMode] = React.useState(false); const [bulkDeleteOpen, setBulkDeleteOpen] = React.useState(false); const [bulkDeleting, setBulkDeleting] = React.useState(false); const [visibleConversationCount, setVisibleConversationCount] = React.useState(CONVERSATION_LIST_PAGE); @@ -998,6 +1007,11 @@ export default function ChatPage() { setSelectedConversationIds(checked ? new Set(convs.map((conversation) => conversation.id)) : new Set()); } + function exitSelectionMode() { + setSelectionMode(false); + setSelectedConversationIds(new Set()); + } + const del = React.useCallback( async (id: number) => { try { @@ -1046,6 +1060,9 @@ export default function ChatPage() { toast.error(`${failed.length} 个对话删除失败：${details}${failed.length > 3 ? " 等" : ""}`); } setBulkDeleteOpen(false); + // Fully successful → return to the clean list; keep selection mode on if + // some failed so the user can retry the remaining ones. + if (failed.length === 0) setSelectionMode(false); reloadConvs(); } catch (error) { toast.error(`批量删除失败：${(error as Error).message}`); @@ -1100,29 +1117,48 @@ export default function ChatPage() { - {convs.length > 0 && ( -
- toggleAllConversations(checked === true)} - aria-label="选择全部对话" - /> - - {selectedConversationCount > 0 ? `已选 ${selectedConversationCount} 个` : `共 ${convs.length} 个`} - - {selectedConversationCount > 0 && ( + {convs.length > 0 && + (selectionMode ? ( +
+ toggleAllConversations(checked === true)} + aria-label="选择全部对话" + /> + + {selectedConversationCount > 0 ? `已选 ${selectedConversationCount} 个` : `共 ${convs.length} 个`} + + {selectedConversationCount > 0 && ( + + )} + +
+ ) : ( +
+ + 共 {convs.length} 个 + - )} -
- )} +
+ ))}
@@ -1142,6 +1178,7 @@ export default function ChatPage() { onCancelRename={cancelRename} onTogglePinned={togglePinned} onDelete={del} + selectionMode={selectionMode} selectedForDelete={selectedConversationIds.has(c.id)} onSelectedForDeleteChange={toggleConversationSelected} /> diff --git a/web/src/app/(main)/function/commands/page.tsx b/web/src/app/(main)/function/commands/page.tsx index 7d58b48..6a5954e 100644 --- a/web/src/app/(main)/function/commands/page.tsx +++ b/web/src/app/(main)/function/commands/page.tsx @@ -2,17 +2,18 @@ import * as React from "react"; -import { ChevronLeftIcon, ChevronRightIcon, Loader2Icon, SearchIcon, TerminalIcon } from "lucide-react"; +import { BarChart3Icon, ChevronLeftIcon, ChevronRightIcon, Loader2Icon, SearchIcon, TerminalIcon } from "lucide-react"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { Card } from "@/components/ui/card"; +import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle } from "@/components/ui/dialog"; import { Input } from "@/components/ui/input"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { Sheet, SheetContent, SheetDescription, SheetHeader, SheetTitle } from "@/components/ui/sheet"; import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; import { api } from "@/lib/api"; -import type { CommandRecord } from "@/lib/types"; +import type { CommandRecord, ToolStat } from "@/lib/types"; import { cn } from "@/lib/utils"; function fmtTime(ts: string) { @@ -59,6 +60,11 @@ export default function CommandsPage() { const [total, setTotal] = React.useState(0); const [loading, setLoading] = React.useState(false); + // 各工具调用次数。弹窗打开时才拉取（多一次聚合查询，不必每次翻页都付）。 + const [statsOpen, setStatsOpen] = React.useState(false); + const [stats, setStats] = React.useState([]); + const [statsLoading, setStatsLoading] = React.useState(false); + // Selected execution is rendered in a right-side detail sheet. const [selected, setSelected] = React.useState(null); @@ -96,6 +102,24 @@ export default function CommandsPage() { }; }, [page, size, queryQ, taskFilter]); + // 统计跟随筛选条件走，和表格描述的是同一批记录（但不分页）。 + React.useEffect(() => { + if (!statsOpen) return; + let alive = true; + setStatsLoading(true); + api + .commandStats({ task: taskFilter || undefined, q: queryQ || undefined }) + .then((r) => alive && setStats(r.stats ?? [])) + .catch(() => alive && setStats([])) + .finally(() => alive && setStatsLoading(false)); + return () => { + alive = false; + }; + }, [statsOpen, queryQ, taskFilter]); + + const statsTotal = stats.reduce((n, s) => n + s.total, 0); + const statsErrors = stats.reduce((n, s) => n + s.errors, 0); + const statsMax = stats.reduce((n, s) => Math.max(n, s.total), 0); const totalPages = Math.max(1, Math.ceil(total / size)); const rangeStart = total === 0 ? 0 : page * size + 1; const rangeEnd = page * size + commands.length; @@ -141,6 +165,11 @@ export default function CommandsPage() { + +
{rangeStart}–{rangeEnd} / {total} @@ -243,6 +272,66 @@ export default function CommandsPage() {
+ {/* 工具调用统计：与表格同一批记录（同筛选、不分页） */} + + + + 工具调用统计 + + {taskFilter || queryQ ? "当前筛选条件下的全部记录" : "全部工具执行记录"} + {stats.length > 0 && ( + <> + {" · "} + {stats.length} 个工具 ·{" "} + {statsTotal} 次调用 + {statsErrors > 0 && ( + <> + {" · 失败 "} + {statsErrors} + + )} + + )} + + + + {statsLoading && stats.length === 0 ? ( +
+ +
+ ) : stats.length === 0 ? ( +
暂无统计数据
+ ) : ( +
+ {stats.map((s) => ( +
+
+
+ {s.tool} + {s.errors > 0 && ( + 失败 {s.errors} + )} +
+
+
0 ? (s.total / statsMax) * 100 : 0}%` }} + /> +
+
+
+
{s.total}
+
+ {statsTotal > 0 ? ((s.total / statsTotal) * 100).toFixed(1) : "0.0"}% +
+
+
+ ))} +
+ )} + +
+ !open && setSelected(null)}> {selected && ( diff --git a/web/src/app/(main)/function/llm-records/page.tsx b/web/src/app/(main)/function/llm-records/page.tsx index cb70c44..5f05618 100644 --- a/web/src/app/(main)/function/llm-records/page.tsx +++ b/web/src/app/(main)/function/llm-records/page.tsx @@ -9,6 +9,8 @@ import { Loader2Icon, XIcon, Trash2Icon, + CopyIcon, + CheckIcon, } from "lucide-react"; import { @@ -74,6 +76,47 @@ function tryFormatJSON(s: string): string { } } +// 复制当前框内文本的小按钮。复制成功后短暂显示对勾。text 为空/仅占位符时禁用。 +function CopyButton({ text }: { text: string }) { + const [copied, setCopied] = React.useState(false); + const timer = React.useRef | null>(null); + React.useEffect(() => () => { if (timer.current) clearTimeout(timer.current); }, []); + + const disabled = !text; + const copy = async () => { + if (disabled) return; + try { + await navigator.clipboard.writeText(text); + } catch { + // navigator.clipboard 在非安全上下文(如 http 局域网)不可用，回退到 execCommand。 + const ta = document.createElement("textarea"); + ta.value = text; + ta.style.position = "fixed"; + ta.style.opacity = "0"; + document.body.appendChild(ta); + ta.select(); + try { document.execCommand("copy"); } catch { /* 忽略：不支持则静默 */ } + document.body.removeChild(ta); + } + setCopied(true); + if (timer.current) clearTimeout(timer.current); + timer.current = setTimeout(() => setCopied(false), 1500); + }; + + return ( + + ); +} + const PAGE_SIZES = [25, 50, 100]; export default function LLMRecordsPage() { @@ -96,6 +139,21 @@ export default function LLMRecordsPage() { const [selected, setSelected] = React.useState(null); const [detail, setDetail] = React.useState(null); const [detailLoading, setDetailLoading] = React.useState(false); + // 归一化视图 / HTTP 原文视图。原文是排查 provider 侧问题的唯一依据：归一化视图 + // 不含工具 schema，响应里也没有 tool_use 块。 + const [rawView, setRawView] = React.useState(false); + + const hasRaw = !!(detail?.raw_request || detail?.raw_response); + // 开关保持用户选择，但切到一条无原文的旧记录时自动落回解析视图，而不是显示空白。 + const showRaw = rawView && hasRaw; + // 原文请求体是 JSON，pretty-print 只改排版不改语义，便于阅读；原文响应是 SSE + // 帧，tryFormatJSON 解析失败会原样返回，故两边共用一个函数即可。 + const reqText = showRaw + ? detail?.raw_request && tryFormatJSON(detail.raw_request) + : detail?.request_body && tryFormatJSON(detail.request_body); + const respText = showRaw + ? detail?.raw_response + : detail?.response_body && tryFormatJSON(detail.response_body); // Per-task delete (task picker + confirm dialog) const [tasks, setTasks] = React.useState([]); @@ -427,10 +485,22 @@ export default function LLMRecordsPage() { ) : ( Error )} + {/* 原文视图开关。旧记录没有原文，此时禁用而非静默回退，避免看着像 + 「原文与解析一致」。 */} + {!MOCK && ( )}
- + div]:block!", + !listOpen && "max-lg:hidden", + )} + >
{(["mainagent", "planner", "system", "worker"] as const).map((role) => { const items = grouped[role]; @@ -1460,6 +1511,7 @@ export function SessionsTab({ taskId }: { taskId: string }) { unread={store[keyForSession(s)]?.unread} onClick={() => { setActiveId(s.id); + setListOpen(false); // 手机端选完即收起，把高度还给会话记录 setWorkerMessage(""); setWorkerMessageRequestId(""); }} @@ -1498,7 +1550,7 @@ export function SessionsTab({ taskId }: { taskId: string }) { {/* Right: transcript */}
-
+
{(() => { const isWorker = active.role === "worker"; const meta = isWorker ? sessionMeta.get(active.id) : undefined; @@ -1557,8 +1609,24 @@ export function SessionsTab({ taskId }: { taskId: string }) {
{tokenTotal.any && ( + {/* 手机端用短标签（入/缓/出）：长标签会把这一行撑成两行，进一步压缩记录区。 */} - + + + + 输入 {activeTokens.input_tokens.toLocaleString()} · 输出{" "} diff --git a/web/src/app/(main)/system/llm/page.tsx b/web/src/app/(main)/system/llm/page.tsx index b08c6ef..213a07f 100644 --- a/web/src/app/(main)/system/llm/page.tsx +++ b/web/src/app/(main)/system/llm/page.tsx @@ -296,7 +296,7 @@ function ProfileSheet({ }) { const isNew = !profile; const [name, setName] = React.useState(""); - const [format, setFormat] = React.useState<"anthropic" | "openai">("anthropic"); + const [format, setFormat] = React.useState<"anthropic" | "openai" | "openai-responses">("anthropic"); const [model, setModel] = React.useState(""); const [baseUrl, setBaseUrl] = React.useState(""); const [proxy, setProxy] = React.useState(""); @@ -309,6 +309,7 @@ function ProfileSheet({ const [effort, setEffort] = React.useState(NONE); const [priority, setPriority] = React.useState("0"); // 轮询顺位;越大越先 const [poolExclude, setPoolExclude] = React.useState(false); + const [streaming, setStreaming] = React.useState(true); // true=流式(默认);false=非流式 const [testing, setTesting] = React.useState(false); const [saving, setSaving] = React.useState(false); const [models, setModels] = React.useState([]); @@ -320,7 +321,9 @@ function ProfileSheet({ React.useEffect(() => { if (!open) return; setName(profile?.name ?? ""); - setFormat(profile?.format === "openai" ? "openai" : "anthropic"); + setFormat( + profile?.format === "openai" || profile?.format === "openai-responses" ? profile.format : "anthropic", + ); setModel(profile?.model ?? ""); setBaseUrl(profile?.base_url ?? ""); setProxy(profile?.proxy ?? ""); @@ -331,6 +334,7 @@ function ProfileSheet({ setEffort(fromStore(profile?.reasoning_effort)); setPriority(String(profile?.priority ?? 0)); setPoolExclude(profile?.pool_exclude ?? false); + setStreaming(profile?.streaming ?? true); setApiKey(""); setKeyHint(profile?.api_key_hint ?? ""); setModels([]); @@ -365,8 +369,22 @@ function ProfileSheet({ try { // 用配置实际会跑的思考参数来测，这样不支持该字段的模型在这里就失败， // 而不是等到跑任务时才炸。传 profile id：Key 输入框留空时用已存的 Key。 - const r = await api.testLLM(format, model, baseUrl, apiKey, proxy, toStore(thinkingType), toStore(effort), profileId); - if (r.ok) toast.success(`连接成功 · ${r.latency_ms ?? "?"}ms · ${r.model ?? model}`); + const r = await api.testLLM( + format, + model, + baseUrl, + apiKey, + proxy, + toStore(thinkingType), + toStore(effort), + profileId, + streaming, + ); + // 回复内容一并展示：看得见模型确实说了话，才算和会话里跑通是一回事。 + if (r.ok) + toast.success(`连接成功 · ${r.latency_ms ?? "?"}ms · ${r.model ?? model}`, { + description: r.reply ? `回复：${r.reply}` : undefined, + }); else toast.error(`连接失败：${r.error ?? "未知"}`); } catch (e) { toast.error(`测试出错：${(e as Error).message}`); @@ -398,6 +416,7 @@ function ProfileSheet({ reasoning_effort: toStore(effort), priority: Number(priority) || 0, pool_exclude: poolExclude, + streaming, }); if (isNew) toast.success(`已新建：${name.trim()}（在卡片上「设为激活」以启用）`); else toast.success(profile?.is_default ? "已保存，激活配置即时生效，无需重启" : "已保存"); @@ -445,13 +464,14 @@ function ProfileSheet({
- setFormat(v as "anthropic" | "openai" | "openai-responses")}> Anthropic - OpenAI + OpenAI (Chat Completions) + OpenAI (Responses API)
@@ -525,7 +545,8 @@ function ProfileSheet({ onChange={(e) => setProxy(e.target.value)} />

- 仅 LLM 出站请求走此代理，支持 http/https/socks5，可带账号密码（如 socks5://user:pass@host:port，密码含特殊字符需 URL 编码）；留空表示不使用代理（直连）。 + 仅 LLM 出站请求走此代理，支持 http/https/socks5，可带账号密码（如 + socks5://user:pass@host:port，密码含特殊字符需 URL 编码）；留空表示不使用代理（直连）。

@@ -595,6 +616,17 @@ function ProfileSheet({
+
+
+ +

+ 开启（默认）走流式 SSE，有运行中实时进度与实时 token 计数。 关闭则走真·非流式（stream:false， + 一次性返回完整响应）——可绕开部分网关糟糕的 SSE 实现（空帧 / 思考字段丢帧）， + 代价是失去运行中的实时进度。 +

+
+ +
@@ -602,8 +634,8 @@ function ProfileSheet({

- 控制是否发送 thinking 字段。不发送=不带该字段（兼容 MiniMax 等不支持 - 的模型）；关闭=发 disabled；开启=发 enabled。与下面的强度互相独立。 + 控制是否发送 thinking 字段。不发送=不带该字段（兼容 MiniMax 等不支持 的模型）；关闭=发 + disabled；开启=发 enabled。与下面的强度互相独立。

diff --git a/web/src/lib/api.ts b/web/src/lib/api.ts index 6041256..4516702 100644 --- a/web/src/lib/api.ts +++ b/web/src/lib/api.ts @@ -74,6 +74,7 @@ import type { TokenTotal, TokenUsage, Tool, + ToolStat, TrafficDetail, TrafficHost, TrafficResp, @@ -685,8 +686,10 @@ export const api = { thinking_type = "", reasoning_effort = "", profile_id?: number, + streaming = true, // 用该配置真实的收发模式来测，别让"流式能通、非流式不通"漏到会话里 ) => - post<{ ok: boolean; error?: string; latency_ms?: number; model?: string }>("/llm/test", { + // reply = 模型实际回复(已截断);一个字都不回的配置后端直接判失败 + post<{ ok: boolean; error?: string; latency_ms?: number; model?: string; reply?: string }>("/llm/test", { provider, model, base_url, @@ -695,6 +698,7 @@ export const api = { thinking_type, reasoning_effort, profile_id, + streaming, }), llmProfiles: () => get<{ profiles: LLMProfile[] }>("/llm/profiles").then((r) => arr(r.profiles)), saveLLMProfile: (p: { @@ -712,6 +716,7 @@ export const api = { reasoning_effort?: string; // ""(不发送)|"low"|"medium"|"high"|"xhigh"|"max" priority?: number; // 轮询顺位，越大越先用 pool_exclude?: boolean; // true=不作为故障转移目标 + streaming?: boolean; // true(默认)=流式 | false=非流式 }) => post<{ id: number }>("/llm/profiles", p), deleteLLMProfile: (id: string) => del<{ deleted: number }>(`/llm/profiles/${id}`), activateLLMProfile: (id: string) => post<{ ok: boolean }>("/llm/profiles/active", { id: Number(id) }), @@ -980,6 +985,13 @@ export const api = { sp.set("size", String(params?.size ?? 50)); return get<{ commands: CommandRecord[]; total: number }>(`/commands?${sp}`); }, + // 各工具调用次数；沿用列表的 task/q 筛选，统计的是整个结果集而非当前页。 + commandStats: (params?: { task?: string; q?: string }) => { + const sp = new URLSearchParams(); + if (params?.task) sp.set("task", params.task); + if (params?.q) sp.set("q", params.q); + return get<{ stats: ToolStat[] }>(`/commands/stats?${sp}`); + }, // ---- LLM records ---- llmRecords: (params?: { model?: string; session?: string; task?: string; page?: number; size?: number }) => { diff --git a/web/src/lib/mock/data.ts b/web/src/lib/mock/data.ts index 51a0007..b3f64c1 100644 --- a/web/src/lib/mock/data.ts +++ b/web/src/lib/mock/data.ts @@ -1263,63 +1263,330 @@ export const frontier: TaskNode[] = explorationGraph.nodes.filter( // intent_id 的活动流。planner=worker "planner"、主 agent=worker "mainagent"。 // 每个 worker 会话含完整工具执行链（tool_use → tool_result → …→ result）。 export const activity: Activity[] = [ - // planner - { seq: 1, worker: "planner", ts: T("2026-07-24T09:12:10Z"), kind: "round", summary: "第 0 轮 · 目标拆解" }, + // planner —— 每轮完整决策链：读态势 → 拉 worker 产出对照 → 判目标 → add_intent 派方向 + // ── 第 0 轮 · 目标拆解与首批意图（2026-07-24 09:12）── + { seq: 1000, worker: "planner", ts: T("2026-07-24T09:12:10Z"), kind: "round", summary: "第 0 轮 · 目标拆解" }, { - seq: 2, + seq: 1001, worker: "planner", - ts: T("2026-07-24T09:12:30Z"), + ts: T("2026-07-24T09:12:25Z"), kind: "thinking", - summary: "acme.com 是企业站，攻击面：官网(www)、后台(admin)、API(api)、商城(shop)。先枚举资产，再挑高价值目标。", + summary: + "graph_overview：goals 已由目标层拆好——g1「获取 acme.com 后台管理权限」、g2「读取用户敏感数据」，均 open；facts=0、recent_facts 为空（图基本空白，还没有任何 worker 产出）。范围内资产只有根域 acme.com + 一个 C 段。攻击面初判：www / admin / api / shop / vpn 五个入口。图里没有 fact，我只能做【极轻量只读探测】把初始意图说具体，绝不在 plan 里把活干了。", }, { - seq: 3, + seq: 1002, + worker: "planner", + ts: T("2026-07-24T09:12:40Z"), + kind: "tool_use", + tool: "bash", + tool_use_id: "p0-probe", + summary: "curl -sI https://acme.com; curl -sI https://admin.acme.com", + detail: "开局态势几乎空白，做 ≤3 次只读探测校准初始意图方向（不做枚举/验证，那是 worker 的活）。", + }, + { + seq: 1003, + worker: "planner", + ts: T("2026-07-24T09:12:52Z"), + kind: "tool_result", + tool: "bash", + tool_use_id: "p0-probe", + summary: "acme.com → 302 → www；Server: nginx/1.24.0\nadmin.acme.com → 200，X-Powered-By 缺省，body 命中 element-ui/vue 特征", + detail: "admin 是独立后台登录页；确认五入口有效，可把初始意图描述写精准。", + }, + { + seq: 1004, worker: "planner", ts: T("2026-07-24T09:13:00Z"), - kind: "text", - summary: - "拆出两个目标：① 获取后台管理权限 ② 读取用户敏感数据。派生首批意图：子域枚举、后台口令测试、注入探测、API 越权。", + kind: "tool_use", + tool: "add_intent", + tool_use_id: "p0-ai", + summary: "add_intent × 4（批量派首批探索方向）", + detail: + "intents=[\n { summary: 'acme.com 全量子域枚举 + 存活主机端口/服务识别，摸清攻击面', asset_ids: [acme.com], priority: 6 },\n { summary: 'admin.acme.com 后台默认口令/弱口令测试（Element-UI 登录，无验证码）', asset_ids: [admin], priority: 9 },\n { summary: 'www.acme.com/search q 参数 SQL 注入探测', asset_ids: [www], priority: 8 },\n { summary: 'api.acme.com 订单接口越权(IDOR)测试', asset_ids: [api], priority: 8 },\n]\nparent_ids 均空（顶层全新方向，兜底连 origin fact）。", }, - { seq: 300, worker: "planner", ts: T("2026-07-25T14:12:00Z"), kind: "round", summary: "第 1 轮 · 攻击面加码" }, { - seq: 301, + seq: 1005, worker: "planner", - ts: T("2026-07-25T14:12:20Z"), - kind: "text", - summary: - ".git 泄露出源码+硬编码 DB 口令，shop 指纹疑似 Fastjson。新增目标③「突破 DMZ 建立内网立足点」，派生指纹关联、反序列化 RCE 两条意图。", + ts: T("2026-07-24T09:13:02Z"), + kind: "tool_result", + tool: "add_intent", + tool_use_id: "p0-ai", + summary: "4 intents created → ids=[i1, i2, i4, i5]，已连入 frontier", + detail: "刻意保持路线多样：i1 侦察 + i2（g1 认证路线）+ i4/i5（g2 数据路线），2 条独立机理并行，不过早收敛。", }, { - seq: 302, + seq: 1006, + worker: "planner", + ts: T("2026-07-24T09:13:05Z"), + kind: "text", + summary: + "本轮派生首批 4 条意图：i1 资产侦察、i2 后台口令、i4 注入、i5 API 越权。g1 走 i2、g2 走 i4/i5，双目标各有独立入口在推进。等 worker 产出事实后再据血缘加码。", + }, + + // ── 第 1 轮 · 攻击面加码（2026-07-25 14:12）── + { seq: 1100, worker: "planner", ts: T("2026-07-25T14:12:00Z"), kind: "round", summary: "第 1 轮 · 攻击面加码" }, + { + seq: 1101, + worker: "planner", + ts: T("2026-07-25T14:12:15Z"), + kind: "thinking", + summary: + "对照 recent_facts：workG 交回 faGit「.git 可下载，git-dumper 还原出后端源码」与 faCreds「config.php 硬编码 sa/Acme@2021（疑似内网通用）」；work#6 指纹交回 fa3「shop 命中 Fastjson 1.2.24（存在已知反序列化 RCE）」。两条【材料性新机理】：① 硬编码 DB 口令可能横跨内网复用；② Fastjson 有成熟 RCE 打法。这打开一条与现有认证/注入路线【本质不同】的入口——从 shop 直接 RCE 突破进 DMZ。先拉 faGit 详情核实产物。", + }, + { + seq: 1102, + worker: "planner", + ts: T("2026-07-25T14:12:28Z"), + kind: "tool_use", + tool: "node_detail", + tool_use_id: "p1-nd", + summary: "node_detail(faGit) —— 核对 workG 的源码还原产物", + detail: "add_intent 前先看 worker 到底还原出什么，避免凭指纹臆测方向。", + }, + { + seq: 1103, + worker: "planner", + ts: T("2026-07-25T14:12:35Z"), + kind: "tool_result", + tool: "node_detail", + tool_use_id: "p1-nd", + summary: + "faGit 证据：git-dumper 还原出 config.php / db.php / order_api.php；config.php 内 $db=['host'=>'10.10.10.5','user'=>'sa','pass'=>'Acme@2021']", + detail: "硬编码指向内网 10.10.10.5(FS01)，佐证 faCreds 的「内网通用」推断——RCE 落地后这组口令值得复用。", + }, + { + seq: 1104, + worker: "planner", + ts: T("2026-07-25T14:12:50Z"), + kind: "tool_use", + tool: "add_intent", + tool_use_id: "p1-ai", + summary: "add_intent × 2（Fastjson 突破链，串行分步先派可执行一步）", + detail: + "intents=[\n { summary: 'shop.acme.com 组件指纹深挖 + Fastjson 1.2.24 CVE 精确关联(确认 gadget 链)', asset_ids: [shop], parent_ids: [fa3], priority: 7 },\n { summary: '构造 Fastjson 反序列化 payload 触发 shop RCE，获取反弹 shell', asset_ids: [shop], parent_ids: [fa3], priority: 9 },\n]\n注：这是强依赖串行链，已 TodoWrite 记『指纹→RCE→立足点提权』三步，本轮只派前两步（提权依赖 shell 产出，下轮再派）。", + }, + { + seq: 1105, + worker: "planner", + ts: T("2026-07-25T14:12:52Z"), + kind: "tool_result", + tool: "add_intent", + tool_use_id: "p1-ai", + summary: "2 intents created → ids=[i6, i7]，已由 fa3 连入血缘", + }, + { + seq: 1106, + worker: "planner", + ts: T("2026-07-25T14:13:00Z"), + kind: "text", + summary: + "态势升级：源码 + 硬编码内网口令 + Fastjson RCE 构成一条全新突破路线。已请目标层登记 g3「从外网突破 DMZ，建立内网立足点」，派 i6（指纹/CVE 关联）、i7（触发 RCE）。DMZ 提权一步依赖 shell，记入待办下轮派。", + }, + + // ── 第 2 轮 · 立足点已建，转内网（2026-07-25 20:25）── + { + seq: 1200, worker: "planner", ts: T("2026-07-25T20:25:00Z"), kind: "round", summary: "第 2 轮 · 立足点已建，转内网", }, { - seq: 303, + seq: 1201, worker: "planner", - ts: T("2026-07-25T20:26:00Z"), + ts: T("2026-07-25T20:25:18Z"), + kind: "thinking", + summary: + "对照血缘：i7 交回 fa4「反弹 shell www-data@dmz-web01(10.0.20.15)，位于 DMZ 段」；提权链交回 fa5「sudo -l 命中 NOPASSWD python3 → 可提权 root」、fa6「立足点第二网卡直连内网 10.10.10.0/24」。g3 的验收条件（突破 DMZ + 建立足点）已被 fa4+fa6 直接满足。先核实 fa6 的内网可达性再 prove_goal。", + }, + { + seq: 1202, + worker: "planner", + ts: T("2026-07-25T20:25:30Z"), + kind: "tool_use", + tool: "node_detail", + tool_use_id: "p2-nd", + summary: "node_detail(fa6) —— 确认双网卡与内网可达", + }, + { + seq: 1203, + worker: "planner", + ts: T("2026-07-25T20:25:38Z"), + kind: "tool_result", + tool: "node_detail", + tool_use_id: "p2-nd", + summary: "fa6 证据：ip a → eth0 10.0.20.15(DMZ) / eth1 10.10.10.15(内网)；ping 10.10.10.10 通，跨越 DMZ 边界成立", + }, + { + seq: 1204, + worker: "planner", + ts: T("2026-07-25T20:25:50Z"), + kind: "tool_use", + tool: "prove_goal", + tool_use_id: "p2-pg", + summary: "prove_goal(g3, evidence=fa6)", + detail: "reason：i7 RCE 拿到 DMZ 立足点(fa4)、fa5 提权 root、fa6 证明第二网卡直连内网并 ping 通 DC01——『突破 DMZ 建立内网立足点』验收达成。", + }, + { + seq: 1205, + worker: "planner", + ts: T("2026-07-25T20:25:52Z"), + kind: "tool_result", + tool: "prove_goal", + tool_use_id: "p2-pg", + summary: "goal g3 → met（证据 fa6 已连入）；仍有 g1/g2/g4 未达成，任务继续", + }, + { + seq: 1206, + worker: "planner", + ts: T("2026-07-25T20:26:10Z"), + kind: "tool_use", + tool: "add_intent", + tool_use_id: "p2-ai", + summary: "add_intent × 4（以立足点为 pivot 展开内网横向）", + detail: + "intents=[\n { summary: '以 dmz-web01 为 pivot 做内网 10.10.10.0/24 主机发现与端口识别', asset_ids: [dmz-web01], parent_ids: [fa6], priority: 8 },\n { summary: '内网横向：复用硬编码口令 sa/Acme@2021 做凭据复用与密码喷洒', parent_ids: [faCreds], priority: 8 },\n { summary: '内部 Jenkins(10.10.10.20) 未授权 /script Groovy 执行(RCE)', parent_ids: [fa6], priority: 9 },\n { summary: '对域服务账号做 Kerberoasting 并离线破解', parent_ids: [fa6], priority: 8 },\n]\n主机发现是前置(先派)，其余三条是并行的横向探索维度。", + }, + { + seq: 1207, + worker: "planner", + ts: T("2026-07-25T20:26:12Z"), + kind: "tool_result", + tool: "add_intent", + tool_use_id: "p2-ai", + summary: "4 intents created → ids=[i9, i10, i11, i12]", + }, + { + seq: 1208, + worker: "planner", + ts: T("2026-07-25T20:26:20Z"), kind: "text", summary: - "RCE 拿到 www-data 并提权 root，立足点双网卡直连 10.10.10.0/24（g3 达成）。新增目标④「拿下内部靶标 DC01」，派生内网主机发现、凭据复用、Jenkins 利用、Kerberoasting 四条意图。", + "g3 达成，攻击面从外网转入内网。已请登记 g4「内网横向，拿下内部靶标域控 DC01」，派 i9（主机发现）+ i10/i11/i12 三条并行横向。凭据复用直接锚在 faCreds 上——外网源码泄露的口令喂给内网横向。", }, - { seq: 304, worker: "planner", ts: T("2026-07-26T00:25:00Z"), kind: "round", summary: "第 3 轮 · 靶标达成复盘" }, + + // ── 第 3 轮 · 靶标达成复盘（2026-07-26 00:25）── + { seq: 1300, worker: "planner", ts: T("2026-07-26T00:25:00Z"), kind: "round", summary: "第 3 轮 · 靶标达成复盘" }, { - seq: 305, + seq: 1301, worker: "planner", - ts: T("2026-07-26T00:26:00Z"), + ts: T("2026-07-26T00:25:16Z"), + kind: "thinking", + summary: + "对照本轮 worker 产出：i9→fa7「内网存活 DC01/FS01/JENKINS」；i11→fa8「Jenkins /script 无鉴权 Groovy」+ fa9「从 Jenkins 凭据库导出 acme\\svc_deploy 明文口令」；i12→fa10「Kerberoast svc_sql TGS，hashcat 破出 Sql@2020」。svc_deploy 属 Domain Admins，i13 已用其 psexec 拿下 DC01 并 secretsdump。g4 验收（拿下 DC01）应已达成，先核对 finding 证据链再盖章。", + }, + { + seq: 1302, + worker: "planner", + ts: T("2026-07-26T00:25:30Z"), + kind: "tool_use", + tool: "list_findings", + tool_use_id: "p3-lf", + summary: "list_findings —— 核对 DC01 沦陷的确认证据", + }, + { + seq: 1303, + worker: "planner", + ts: T("2026-07-26T00:25:40Z"), + kind: "tool_result", + tool: "list_findings", + tool_use_id: "p3-lf", + summary: + "F-09「域控 DC01 完全控制 / secretsdump 导出全域哈希」critical（证据：psexec svc_deploy@10.10.10.10 + NTDS.dit dump）；上游链 fa9→fa10→i13 完整", + }, + { + seq: 1304, + worker: "planner", + ts: T("2026-07-26T00:25:52Z"), + kind: "tool_use", + tool: "prove_goal", + tool_use_id: "p3-pg", + summary: "prove_goal(g4, evidence=F-09)", + detail: "reason：Jenkins 泄露 svc_deploy(Domain Admin) → psexec 控制 DC01 → secretsdump 全域哈希，F-09 已确认，『拿下内部靶标域控 DC01』达成。", + }, + { + seq: 1305, + worker: "planner", + ts: T("2026-07-26T00:25:54Z"), + kind: "tool_result", + tool: "prove_goal", + tool_use_id: "p3-pg", + summary: "goal g4 → met；系统提示：g2 仍 open，非最后一个目标，任务未整体收官", + }, + { + seq: 1306, + worker: "planner", + ts: T("2026-07-26T00:26:10Z"), kind: "text", summary: - "svc_deploy 属于 Domain Admins，已 psexec 拿下 DC01 并 secretsdump 全域哈希（g4 达成）。内外网链路打通，收敛剩余 g2（后台批量用户数据）。", + "g4 收官，内外网链路完全打通（外网 RCE → DMZ → 内网 → 域控）。逐一比对未覆盖方向：g2「读取用户敏感数据」仍缺临门一脚，但现有 i3（后台用户接口）+ i5（API IDOR）已覆盖该方向、且都在 running。本轮无【本质不同的未覆盖新方向】——按克制原则 0 新意图，等 i3/i5 产出。", }, - { seq: 4, worker: "planner", ts: T("2026-07-26T03:55:00Z"), kind: "round", summary: "第 4 轮 · 复盘" }, + + // ── 第 4 轮 · 后台达成，收敛 g2（2026-07-26 03:55）── + { seq: 1400, worker: "planner", ts: T("2026-07-26T03:55:00Z"), kind: "round", summary: "第 4 轮 · 后台达成，收敛 g2" }, { - seq: 5, + seq: 1401, worker: "planner", - ts: T("2026-07-26T03:55:10Z"), + ts: T("2026-07-26T03:55:14Z"), + kind: "thinking", + summary: + "对照 recent_facts：work#2 用 faCreds 里的 Acme@2021 命中 admin 后台（i2 done），后台管理权限到手——g1 验收达成，先 prove_goal 盖章。随后做【量化验收核对】：g2「读取用户敏感数据」当前只有 i3/i5 摸到接口面、尚未真正取到批量数据，实测未达标，【禁止】prove_goal，改为提优先级补足取数意图。", + }, + { + seq: 1402, + worker: "planner", + ts: T("2026-07-26T03:55:26Z"), + kind: "tool_use", + tool: "prove_goal", + tool_use_id: "p4-pg", + summary: "prove_goal(g1, evidence=fa1)", + detail: "reason：i2 用外网泄露的硬编码口令 Acme@2021 成功登入 admin 后台（Element-UI），获取后台管理权限，g1 达成。", + }, + { + seq: 1403, + worker: "planner", + ts: T("2026-07-26T03:55:28Z"), + kind: "tool_result", + tool: "prove_goal", + tool_use_id: "p4-pg", + summary: "goal g1 → met；剩余未达成：g2", + }, + { + seq: 1404, + worker: "planner", + ts: T("2026-07-26T03:55:40Z"), + kind: "thinking", + summary: + "g2 量化核对：目标要求『读取用户敏感数据』=实际拿到批量用户记录。graph_overview 里 findings 尚无『成功导出用户数据』条目，i3/i5 均 running 未回取数结果——实测未达标。严禁以『后台已进、大体达成』提前标 met，继续派意图补足差距。", + }, + { + seq: 1405, + worker: "planner", + ts: T("2026-07-26T03:55:52Z"), + kind: "tool_use", + tool: "add_intent", + tool_use_id: "p4-ai", + summary: "add_intent × 2（收敛 g2 取数，提高优先级）", + detail: + "intents=[\n { summary: '已登入 admin 后台，枚举用户管理/导出接口并批量拉取用户敏感字段', asset_ids: [admin], parent_ids: [fa1], priority: 9 },\n { summary: 'api.acme.com 订单 IDOR 遍历 order_id 批量拉取用户订单/PII', asset_ids: [api], parent_ids: [fa2], priority: 9 },\n]\n注：i3/i5 已在 running，本轮以更精准的取数描述 + 提优先级(→9)加码，不重复派同义意图。", + }, + { + seq: 1406, + worker: "planner", + ts: T("2026-07-26T03:55:54Z"), + kind: "tool_result", + tool: "add_intent", + tool_use_id: "p4-ai", + summary: "2 intents updated → ids=[i3, i5]（优先级升至 9，描述细化为『批量取数』）", + }, + { + seq: 1407, + worker: "planner", + ts: T("2026-07-26T03:55:58Z"), kind: "text", - summary: "后台已拿下(g1 达成)。继续 g2：派生「枚举后台用户接口」「API IDOR 测试」两条意图并提高优先级。", + summary: + "g1 达成（4/4 里已 met 3 个：g1/g3/g4）。g2 严守量化验收——只进后台不算，必须拿到批量用户数据才盖章；已把 i3/i5 提到 priority 9 并细化为取数意图，等 worker 回传即可收官整个任务。", }, // work#1 · i1 子域枚举与端口扫描（done） @@ -3766,5 +4033,41 @@ export function llmRecordDetail(id: number, records = llmRecords) { null, 2, ), + // HTTP 原文：请求含被归一化视图丢弃的完整工具 schema，响应为原始 SSE 帧。 + raw_request: JSON.stringify({ + model: item.model, + max_tokens: 8192, + stream: true, + system: [{ type: "text", text: "你是一个授权渗透测试系统的「执行者」…（省略）", cache_control: { type: "ephemeral" } }], + messages: [{ role: "user", content: "开始执行 system 提示里的这条意图：只做它、只产生事实、做完即停。" }], + tools: [ + { + name: "bash", + description: "在目标环境中执行一条 shell 命令并返回其输出。", + input_schema: { type: "object", properties: { command: { type: "string" } }, required: ["command"] }, + }, + ], + thinking: { type: "enabled" }, + }), + raw_response: [ + `event: message_start`, + `data: {"type":"message_start","message":{"id":"msg_01mock","model":"${item.model}","usage":{"input_tokens":${item.input_tokens},"output_tokens":1}}}`, + ``, + `event: content_block_delta`, + `data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"对 search?q= 做注入探测。"}}`, + ``, + `event: content_block_start`, + `data: {"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_01mock","name":"bash"}}`, + ``, + `event: content_block_delta`, + `data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\\"command\\":\\"curl -s 'https://www.acme.com/search?q=1%27'\\"}"}}`, + ``, + `event: message_delta`, + `data: {"type":"message_delta","delta":{"stop_reason":"tool_use"},"usage":{"output_tokens":${item.output_tokens}}}`, + ``, + `event: message_stop`, + `data: {"type":"message_stop"}`, + ``, + ].join("\n"), }; } diff --git a/web/src/lib/mock/handler.ts b/web/src/lib/mock/handler.ts index dd61d86..98b6c44 100644 --- a/web/src/lib/mock/handler.ts +++ b/web/src/lib/mock/handler.ts @@ -1579,6 +1579,17 @@ function route(m: string, path: string, seg: string[], q: URLSearchParams, b: Re // ── 工具执行历史 ── if (path === "/commands" && m === "GET") return { commands: D.commandRecords, total: D.commandRecords.length }; + if (path === "/commands/stats" && m === "GET") { + const tally = new Map(); + for (const c of D.commandRecords) { + const tool = c.tool || "-"; + const s = tally.get(tool) ?? { tool, total: 0, errors: 0 }; + s.total++; + if (c.is_error) s.errors++; + tally.set(tool, s); + } + return { stats: [...tally.values()].sort((a, b) => b.total - a.total || a.tool.localeCompare(b.tool)) }; + } // ── LLM ── if (path === "/llm/records" && m === "GET") return { records: mockLLMRecords, total: mockLLMRecords.length }; @@ -1595,7 +1606,8 @@ function route(m: string, path: string, seg: string[], q: URLSearchParams, b: Re } if (path === "/llm" && m === "GET") return D.llmConfig; if (path === "/llm" && m === "POST") return { ok: true }; - if (path === "/llm/test") return { ok: true, latency_ms: 128, model: String(b.model ?? "claude-opus-4-8") }; + if (path === "/llm/test") + return { ok: true, latency_ms: 128, model: String(b.model ?? "claude-opus-4-8"), reply: "OK" }; if (path === "/llm/profiles" && m === "GET") return { profiles: D.llmProfiles }; if (path === "/llm/profiles" && m === "POST") return { id: Number(b.id) || 3 }; if (path === "/llm/profiles/active") return { ok: true }; diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts index 633688e..808ffad 100644 --- a/web/src/lib/types.ts +++ b/web/src/lib/types.ts @@ -852,7 +852,7 @@ export interface Settings { export interface LLMProfile { id: string; name: string; - format: "openai" | "anthropic"; + format: "openai" | "anthropic" | "openai-responses"; base_url?: string; proxy?: string; model: string; @@ -869,6 +869,8 @@ export interface LLMProfile { priority?: number; // true = 不作为故障转移目标（仍可被 agent/任务显式绑定使用）。 pool_exclude?: boolean; + // true（默认）= 流式(SSE) | false = 真·非流式(stream:false，一次性返回)。 + streaming?: boolean; } // ---- LLM 轮询（故障转移）---- @@ -1133,6 +1135,13 @@ export interface CommandRecord { created_at: string; } +// 单个工具的调用统计（/commands/stats）；errors 为其中失败的次数。 +export interface ToolStat { + tool: string; + total: number; + errors: number; +} + // ---- LLM recording ---- export interface LLMRecordItem { id: number; @@ -1154,6 +1163,11 @@ export interface LLMRecordItem { export interface LLMRecordDetail extends LLMRecordItem { request_body: string; response_body: string; + // provider 实际收发的 HTTP 原文：请求为 buildBody() 发出的完整 body（含工具 + // schema），响应为原始 SSE 帧。上面的 request_body/response_body 是归一化视图， + // 丢弃了工具 schema 与 tool_use 块。旧记录为空。 + raw_request?: string; + raw_response?: string; } // One distinct task with its LLM-record count (task picker on the records page).