From 4d9580ebfae17912b199582fd0329341ffef9a3a Mon Sep 17 00:00:00 2001 From: Autumn-27 <53844290+Autumn-27@users.noreply.github.com> Date: Mon, 24 Aug 2026 12:55:27 -0400 Subject: [PATCH 01/23] =?UTF-8?q?feat(ops):=20=E6=96=B0=E5=A2=9E=E7=AE=A1?= =?UTF-8?q?=E7=90=86=E5=91=98=E5=AF=86=E7=A0=81=E9=87=8D=E7=BD=AE=E8=84=9A?= =?UTF-8?q?=E6=9C=AC=20reset-password.sh?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 登录用户名固定 ARTEX,密码以 bcrypt 存于 settings 表 auth.password_hash。 脚本连库后用 pgcrypto 在库内生成 bcrypt(gen_salt('bf',10)),与后端 golang.org/x/crypto/bcrypt 校验兼容,写回该键,重置后无需重启。 - 两种部署:local(宿主机 psql)与 docker(compose/exec 在 postgres 容器内跑, 因 compose 默认不暴露 5432)。 - 连接信息优先级:命令行 > --dsn/$ARTEX_PG_DSN > config.json 的 database.* (python3 解析,缺则 grep 后备)。 - 安全:新密码经环境变量 + psql \getenv 传入,不进 argv;SQL 用 :'newpw' 自动转义防注入;DB 密码走 PGPASSWORD;docker 用 -e 名字继承、不带值。 验证: 临时库实测三条路径(显式 dsn / config.json / 含 ';DROP TABLE' 特殊 字符密码),写回的 hash 均能被 Go bcrypt 接受、错误密码被拒、注入未生效; bash -n 通过。全程未触碰真实 artex 库。 --- CHANGELOG.md | 6 ++ reset-password.sh | 235 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 241 insertions(+) create mode 100755 reset-password.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index a3fbdeb..09e4d36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ ## [Unreleased] +### 运维 + +#### 新增的功能 + +- 新增 `reset-password.sh` 重置管理员（用户名固定 `ARTEX`）密码：支持 local / docker 两种部署，连接信息可显式指定或自动从 `--dsn`/`$ARTEX_PG_DSN`/`config.json` 读取；在库内用 `pgcrypto` 生成与后端登录兼容的 bcrypt 哈希并写回 `settings.auth.password_hash`，重置后无需重启服务。密码经环境变量传入、不进入进程 argv，并做转义防注入。 + ## [0.3.4] - 2026-08-24 ### UI diff --git a/reset-password.sh b/reset-password.sh new file mode 100755 index 0000000..d48f3b2 --- /dev/null +++ b/reset-password.sh @@ -0,0 +1,235 @@ +#!/usr/bin/env bash +# ============================================================================= +# ARTEX 管理员密码重置脚本 +# +# 登录用户名固定为 ARTEX；密码以 bcrypt 哈希存放在数据库 settings 表的 +# auth.password_hash 键。本脚本连上数据库后，用 pgcrypto 在库内生成 bcrypt 哈希 +# 并写回该键——与后端登录校验（golang.org/x/crypto/bcrypt）完全兼容。 +# +# 两种部署： +# local （默认）—— 宿主机直接用 psql 连数据库。连接信息按以下优先级获取： +# 命令行参数 > --dsn/$ARTEX_PG_DSN > config.json 的 database.* +# docker —— 通过 `docker compose exec`（或 `docker exec`）在 postgres +# 容器内执行 psql（compose 默认不对宿主暴露 5432，故走容器内）。 +# +# 用法示例： +# ./reset-password.sh # 本地，自动读 config.json/环境，交互输入新密码 +# ./reset-password.sh -p 'NewPass!' # 本地，直接给定新密码 +# ./reset-password.sh --dsn postgres://u:p@h:5432/artex +# ./reset-password.sh -H 127.0.0.1 -P 5433 -U autopentest -W pass -d artex +# ./reset-password.sh -m docker # docker 部署（读 .env 的 POSTGRES_*） +# ./reset-password.sh -m docker -c pg容器名 --exec docker +# +# 安全：新密码经环境变量 + psql \getenv 传入（不进入进程 argv），并用 :'var' +# 自动转义（防 SQL 注入）；数据库密码经 PGPASSWORD 传递，同样不进 argv。 +# ============================================================================= +set -euo pipefail + +PASS_KEY="auth.password_hash" +BCRYPT_COST=10 + +MODE="" # local | docker（空=自动判定） +DSN="" +HOST="" PORT="" USER="" DBPASS="" DBNAME="" SSLMODE="" +CONFIG="" +CONTAINER="" # docker 模式的 postgres 服务/容器名（默认 postgres） +EXEC_KIND="" # compose | docker（docker 模式下用哪种 exec；空=自动） +NEWPASS="" +ASSUME_YES=0 + +die() { echo "错误：$*" >&2; exit 1; } +info() { echo "· $*" >&2; } + +usage() { sed -n '2,40p' "$0" | sed 's/^# \{0,1\}//'; exit 0; } + +# ---- 参数解析 ------------------------------------------------------------- +while [[ $# -gt 0 ]]; do + case "$1" in + -m|--mode) MODE="${2:-}"; shift 2 ;; + --dsn) DSN="${2:-}"; shift 2 ;; + -H|--host) HOST="${2:-}"; shift 2 ;; + -P|--port) PORT="${2:-}"; shift 2 ;; + -U|--user) USER="${2:-}"; shift 2 ;; + -W|--db-password) DBPASS="${2:-}"; shift 2 ;; + -d|--dbname) DBNAME="${2:-}"; shift 2 ;; + --sslmode) SSLMODE="${2:-}"; shift 2 ;; + --config) CONFIG="${2:-}"; shift 2 ;; + -c|--container) CONTAINER="${2:-}"; shift 2 ;; + --exec) EXEC_KIND="${2:-}"; shift 2 ;; + -p|--new-password) NEWPASS="${2:-}"; shift 2 ;; + -y|--yes) ASSUME_YES=1; shift ;; + -h|--help) usage ;; + *) die "未知参数：$1（-h 查看用法）" ;; + esac +done + +# ---- 从 config.json 读取 database.*（仅 local 模式、且未显式给出连接时）----- +# 优先用 python3 解析（健壮）；缺 python3 时退回 grep（config.json 为规整分字段）。 +read_config_json() { + local path="$1" + [[ -f "$path" ]] || return 1 + if command -v python3 >/dev/null 2>&1; then + python3 - "$path" <<'PY' +import json, sys +try: + d = json.load(open(sys.argv[1])).get("database", {}) +except Exception: + sys.exit(1) +# 支持直接给 dsn，或分字段 +if d.get("dsn"): + print("DSN\t" + d["dsn"]); sys.exit(0) +for k in ("host","port","user","password","dbname","sslmode"): + if d.get(k) is not None: + print(k.upper() + "\t" + str(d[k])) +PY + else + # 极简后备：逐键 grep（值为字符串或数字） + local k + for k in host port user password dbname sslmode; do + local v + v=$(grep -oE "\"$k\"[[:space:]]*:[[:space:]]*(\"[^\"]*\"|[0-9]+)" "$path" 2>/dev/null \ + | head -1 | sed -E "s/.*:[[:space:]]*//; s/^\"//; s/\"$//") || true + [[ -n "$v" ]] && echo -e "${k^^}\t$v" + done + fi +} + +apply_config_fields() { + local line key val + while IFS=$'\t' read -r key val; do + [[ -z "$key" ]] && continue + case "$key" in + DSN) [[ -z "$DSN" ]] && DSN="$val" ;; + HOST) [[ -z "$HOST" ]] && HOST="$val" ;; + PORT) [[ -z "$PORT" ]] && PORT="$val" ;; + USER) [[ -z "$USER" ]] && USER="$val" ;; + PASSWORD) [[ -z "$DBPASS" ]] && DBPASS="$val" ;; + DBNAME) [[ -z "$DBNAME" ]] && DBNAME="$val" ;; + SSLMODE) [[ -z "$SSLMODE" ]] && SSLMODE="$val" ;; + esac + done +} + +# ---- 自动判定模式 --------------------------------------------------------- +if [[ -z "$MODE" ]]; then + if [[ -n "$DSN$HOST$USER$DBNAME" || -n "${ARTEX_PG_DSN:-}" || -f "${CONFIG:-config.json}" ]]; then + MODE="local" + elif command -v docker >/dev/null 2>&1 && [[ -f docker-compose.yml ]]; then + MODE="docker" + else + MODE="local" + fi +fi +info "部署模式：$MODE" + +# ---- 采集新密码 ----------------------------------------------------------- +if [[ -z "$NEWPASS" ]]; then + read -r -s -p "输入新密码（用户名固定为 ARTEX）：" NEWPASS; echo >&2 + [[ -n "$NEWPASS" ]] || die "密码不能为空" + read -r -s -p "再次输入以确认：" NEWPASS2; echo >&2 + [[ "$NEWPASS" == "$NEWPASS2" ]] || die "两次输入不一致" +fi +[[ -n "$NEWPASS" ]] || die "密码不能为空" + +# 通过环境变量把密码交给 psql（\getenv 读取，不进入 argv/ps） +export ARTEX_RESET_NEWPASS="$NEWPASS" + +# 库内生成 bcrypt 并 upsert；密码用 :'newpw' 自动转义。CREATE EXTENSION 幂等， +# 若数据库角色无建扩展权限会在此报错（提示见下方 run 的失败分支）。 +SQL=$(cat < --dsn/$ARTEX_PG_DSN > config.json + if [[ -z "$DSN" && -z "$HOST$USER$DBNAME" ]]; then + [[ -n "${ARTEX_PG_DSN:-}" ]] && DSN="$ARTEX_PG_DSN" + fi + if [[ -z "$DSN" && -z "$HOST$USER$DBNAME" ]]; then + cfg="${CONFIG:-config.json}" + if [[ -f "$cfg" ]]; then + info "从 $cfg 读取数据库配置" + apply_config_fields < <(read_config_json "$cfg") + fi + fi + + command -v psql >/dev/null 2>&1 || die "本机未找到 psql（请安装 postgresql-client，或改用 -m docker）" + + declare -a PSQL_ARGS=() + if [[ -n "$DSN" ]]; then + PSQL_ARGS=("$DSN") + target="$DSN" + else + [[ -n "$USER" ]] || die "缺少数据库用户（-U）或有效的 config.json/DSN" + [[ -n "$DBNAME" ]] || die "缺少数据库名（-d）或有效的 config.json/DSN" + HOST="${HOST:-127.0.0.1}"; PORT="${PORT:-5432}"; SSLMODE="${SSLMODE:-disable}" + PSQL_ARGS=(-h "$HOST" -p "$PORT" -U "$USER" -d "$DBNAME") + [[ -n "$SSLMODE" ]] && export PGSSLMODE="$SSLMODE" + [[ -n "$DBPASS" ]] && export PGPASSWORD="$DBPASS" + target="$USER@$HOST:$PORT/$DBNAME" + fi + + info "目标数据库：$target" + if [[ "$ASSUME_YES" -ne 1 ]]; then + read -r -p "确认在该库重置 ARTEX 密码？[y/N] " ans + [[ "$ans" == "y" || "$ans" == "Y" ]] || die "已取消" + fi + + if ! printf '%s\n' "$SQL" | psql "${PSQL_ARGS[@]}" -v ON_ERROR_STOP=1 -q >/dev/null; then + die "写入失败。若报 pgcrypto 权限/缺失，请用具备建扩展权限的角色，或先手动执行 CREATE EXTENSION pgcrypto。" + fi + +else + # ---- docker ---- + command -v docker >/dev/null 2>&1 || die "未找到 docker" + CONTAINER="${CONTAINER:-postgres}" + + # 选择 exec 方式：优先 docker compose exec（服务名），否则 docker exec（容器名） + if [[ -z "$EXEC_KIND" ]]; then + if docker compose version >/dev/null 2>&1 && [[ -f docker-compose.yml ]]; then + EXEC_KIND="compose" + else + EXEC_KIND="docker" + fi + fi + + # 容器内的 psql 凭据：优先命令行，其次 .env 的 POSTGRES_*，再退回 compose 默认(artex) + if [[ -f .env ]]; then + # shellcheck disable=SC1091 + set -a; . ./.env; set +a + fi + DUSER="${USER:-${POSTGRES_USER:-artex}}" + DNAME="${DBNAME:-${POSTGRES_DB:-artex}}" + [[ -n "$DBPASS" ]] && export PGPASSWORD="$DBPASS" + [[ -z "${PGPASSWORD:-}" && -n "${POSTGRES_PASSWORD:-}" ]] && export PGPASSWORD="$POSTGRES_PASSWORD" + + info "目标：容器 $CONTAINER 内 psql -U $DUSER -d $DNAME（exec=$EXEC_KIND）" + if [[ "$ASSUME_YES" -ne 1 ]]; then + read -r -p "确认在该容器数据库重置 ARTEX 密码？[y/N] " ans + [[ "$ans" == "y" || "$ans" == "Y" ]] || die "已取消" + fi + + # -e 只带名字不带值 → 从当前环境继承，密码不出现在 docker 命令 argv 里。 + declare -a EXEC_CMD + if [[ "$EXEC_KIND" == "compose" ]]; then + EXEC_CMD=(docker compose exec -T -e ARTEX_RESET_NEWPASS -e PGPASSWORD "$CONTAINER" + psql -U "$DUSER" -d "$DNAME" -v ON_ERROR_STOP=1 -q) + else + EXEC_CMD=(docker exec -i -e ARTEX_RESET_NEWPASS -e PGPASSWORD "$CONTAINER" + psql -U "$DUSER" -d "$DNAME" -v ON_ERROR_STOP=1 -q) + fi + + if ! printf '%s\n' "$SQL" | "${EXEC_CMD[@]}" >/dev/null; then + die "写入失败。请确认容器名（-c）、数据库账号（.env 的 POSTGRES_*），以及角色有 pgcrypto 权限。" + fi +fi + +unset ARTEX_RESET_NEWPASS +echo "✓ 已重置 ARTEX 管理员密码。请用用户名 ARTEX + 新密码登录（无需重启服务）。" From 19a7206d12e066f9a8e9638d7f7888961e6c59ef Mon Sep 17 00:00:00 2001 From: Autumn-27 <53844290+Autumn-27@users.noreply.github.com> Date: Tue, 25 Aug 2026 09:40:05 -0400 Subject: [PATCH 02/23] =?UTF-8?q?feat(llmrec):=20=E5=BD=95=E5=88=B6=20LLM?= =?UTF-8?q?=20=E8=AF=B7=E6=B1=82/=E5=93=8D=E5=BA=94=20HTTP=20=E5=8E=9F?= =?UTF-8?q?=E6=96=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 归一化视图(request_body/response_body)会丢弃工具 schema、tool_use 块， 且看不到原始 SSE 帧，排查 provider 侧问题时信息不足。新增在 HTTP transport 层捕获真实 wire body 的能力： - llmrec: 新增 Capture，Recorder 挂到 ctx，quotaAwareTransport 填充。 请求取 GetBody 副本(不消费原 body)，响应用 tee 保持流式;norma 内部重试 的多次尝试逐次保留(旧代码会读掉丢弃),单次时为字节级原文。 - db: llm_records 增加 raw_request/raw_response 两列，含旧库 ALTER 迁移。 - web: 录制页详情面板增加「原文」视图切换，及请求/响应内容复制按钮 (兼容非安全上下文的 execCommand 回退)。 - 测试: 单元(capture)、端到端(逐字节比对捕获=服务器实际收到)、 迁移(真实 PG 事务回滚)、全链路(Recorder→provider→PG 落库)。 --- agent/provider.go | 37 ++++- agent/provider_capture_e2e_test.go | 131 ++++++++++++++++ agent/provider_capture_test.go | 125 ++++++++++++++++ db/commands.go | 28 +++- db/llm_records_migrate_test.go | 62 ++++++++ llmrec/capture.go | 140 ++++++++++++++++++ llmrec/capture_test.go | 103 +++++++++++++ llmrec/llmrec.go | 15 +- server/llmrec_raw_test.go | 133 +++++++++++++++++ .../app/(main)/function/llm-records/page.tsx | 86 ++++++++++- web/src/lib/mock/data.ts | 36 +++++ web/src/lib/types.ts | 5 + 12 files changed, 884 insertions(+), 17 deletions(-) create mode 100644 agent/provider_capture_e2e_test.go create mode 100644 agent/provider_capture_test.go create mode 100644 db/llm_records_migrate_test.go create mode 100644 llmrec/capture.go create mode 100644 llmrec/capture_test.go create mode 100644 server/llmrec_raw_test.go diff --git a/agent/provider.go b/agent/provider.go index 6011ada..3a576e7 100644 --- a/agent/provider.go +++ b/agent/provider.go @@ -19,6 +19,7 @@ import ( "strings" "time" + "github.com/Autumn-27/artex/llmrec" "github.com/Autumn-27/norma/agentcore" "github.com/Autumn-27/norma/compaction" "github.com/Autumn-27/norma/llm" @@ -247,10 +248,25 @@ func IsQuotaExhaustedMessage(message string) bool { type quotaAwareTransport struct{ base http.RoundTripper } func (t quotaAwareTransport) RoundTrip(req *http.Request) (*http.Response, error) { + // When LLM recording is on, the Recorder puts a Capture on the context so the + // raw wire bodies can be persisted. This is the only layer that still sees + // them: norma builds the request body internally and decodes the SSE response + // before either reaches the recorder. + capt := llmrec.CaptureFrom(req.Context()) + capt.SetRequest(requestBodySnapshot(req)) + resp, err := t.base.RoundTrip(req) - if err != nil || resp == nil || resp.StatusCode != http.StatusTooManyRequests { + if err != nil || resp == nil { return resp, err } + // Tee rather than read: a 200 is an SSE stream that must keep streaming. The + // 429 branch below reads through this wrapper, so its body lands in the + // capture before being replaced. + resp.Body = capt.TeeResponse(resp.StatusCode, resp.Body) + + if resp.StatusCode != http.StatusTooManyRequests { + return resp, nil + } body, readErr := io.ReadAll(resp.Body) _ = resp.Body.Close() resp.Body = io.NopCloser(bytes.NewReader(body)) @@ -265,6 +281,25 @@ func (t quotaAwareTransport) RoundTrip(req *http.Request) (*http.Response, error return resp, nil } +// requestBodySnapshot copies an outgoing request body without consuming it. +// norma builds every model request from a *bytes.Reader, so net/http populates +// GetBody and the copy has no effect on what gets sent. +func requestBodySnapshot(req *http.Request) string { + if req.GetBody == nil { + return "" + } + rc, err := req.GetBody() + if err != nil { + return "" + } + defer rc.Close() + b, err := io.ReadAll(rc) + if err != nil { + return "" + } + return string(b) +} + func quotaAwareHTTPClient(proxy string) (*http.Client, error) { transport := http.DefaultTransport.(*http.Transport).Clone() proxy = strings.TrimSpace(proxy) diff --git a/agent/provider_capture_e2e_test.go b/agent/provider_capture_e2e_test.go new file mode 100644 index 0000000..3405f6d --- /dev/null +++ b/agent/provider_capture_e2e_test.go @@ -0,0 +1,131 @@ +package agent + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/Autumn-27/artex/llmrec" + "github.com/Autumn-27/norma/llm" +) + +// End-to-end through a real norma provider: the Capture rides the context into +// norma, survives its internal request building, and comes back holding the +// exact bytes buildBody() put on the wire. This is the load-bearing assumption +// of the whole feature — norma must propagate the caller's context down to +// http.NewRequestWithContext. +func TestCapturePropagatesThroughNormaProvider(t *testing.T) { + sse := strings.Join([]string{ + `event: message_start`, + `data: {"type":"message_start","message":{"id":"msg_1","usage":{"input_tokens":11,"output_tokens":1}}}`, + ``, + `event: content_block_delta`, + `data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"hello"}}`, + ``, + `event: message_delta`, + `data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":7}}`, + ``, + `event: message_stop`, + `data: {"type":"message_stop"}`, + ``, + }, "\n") + + // Record exactly what the server receives, so the capture can be compared + // against it byte for byte rather than merely spot-checked for fields. + var gotPath, serverSaw string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + b, err := io.ReadAll(r.Body) + if err != nil { + t.Errorf("server read body: %v", err) + } + serverSaw = string(b) + w.Header().Set("content-type", "text/event-stream") + _, _ = io.WriteString(w, sse) + })) + defer srv.Close() + + cfg := Config{ + Format: llm.FormatAnthropic, + BaseURL: srv.URL, + APIKey: "test-key", + Model: "claude-test", + } + prov, err := cfg.NewProvider() + if err != nil { + t.Fatalf("NewProvider: %v", err) + } + + ctx, capt := llmrec.NewCapture(context.Background()) + req := llm.CompletionRequest{ + System: []string{"you are a scanner"}, + Messages: []llm.Message{{Role: "user", Content: []llm.ContentBlock{{Type: "text", Text: "go"}}}}, + Tools: []llm.ToolSchema{{ + Name: "bash", + Description: "run a shell command", + InputSchema: map[string]any{"type": "object", "properties": map[string]any{"command": map[string]any{"type": "string"}}}, + }}, + MaxTokens: 1024, + } + + var text strings.Builder + for ev, err := range prov.Stream(ctx, req) { + if err != nil { + t.Fatalf("stream: %v", err) + } + if ev.Type == llm.SETextDelta { + text.WriteString(ev.Text) + } + } + if text.String() != "hello" { + t.Fatalf("stream text=%q, capture interfered with delivery", text.String()) + } + if gotPath != "/v1/messages" { + t.Fatalf("path=%q", gotPath) + } + + // The raw request must be what norma actually sent, not the recorder's + // re-serialization — which is exactly why it carries fields the normalized + // view drops. + raw := capt.RawRequest() + if raw == "" { + t.Fatal("no raw request captured — context did not reach the transport") + } + // The load-bearing claim: what we stored equals, byte for byte, what the + // server received — not merely "has the right fields". + if raw != serverSaw { + t.Fatalf("captured request != what the server received:\n got: %q\nsaw: %q", raw, serverSaw) + } + var body map[string]any + if err := json.Unmarshal([]byte(raw), &body); err != nil { + t.Fatalf("raw request is not valid JSON: %v\n%s", err, raw) + } + if body["model"] != "claude-test" { + t.Errorf("model=%v want claude-test (absent from CompletionRequest)", body["model"]) + } + if body["stream"] != true { + t.Errorf("stream=%v want true", body["stream"]) + } + // The full tool schema is the headline gain: the normalized view keeps names only. + tools, _ := body["tools"].([]any) + if len(tools) != 1 { + t.Fatalf("tools=%v", body["tools"]) + } + tool, _ := tools[0].(map[string]any) + if tool["description"] != "run a shell command" { + t.Errorf("tool description missing: %v", tool) + } + if tool["input_schema"] == nil { + t.Errorf("tool input_schema missing: %v", tool) + } + + // And the response is the untouched SSE frames, including the events the + // recorder never turns into stored output. + if capt.RawResponse() != sse { + t.Errorf("RawResponse mismatch:\n got: %q\nwant: %q", capt.RawResponse(), sse) + } +} diff --git a/agent/provider_capture_test.go b/agent/provider_capture_test.go new file mode 100644 index 0000000..514b2c7 --- /dev/null +++ b/agent/provider_capture_test.go @@ -0,0 +1,125 @@ +package agent + +import ( + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/Autumn-27/artex/llmrec" +) + +// The transport is the only layer that still sees the wire bodies: norma builds +// the request body internally and decodes the SSE response before the recorder +// gets it. This checks the round trip preserves both directions untouched. +func TestRoundTripCapturesRawBodies(t *testing.T) { + const sse = "event: message_start\ndata: {\"type\":\"message_start\"}\n\nevent: message_stop\ndata: {}\n\n" + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("content-type", "text/event-stream") + _, _ = io.WriteString(w, sse) + })) + defer srv.Close() + + client, err := quotaAwareHTTPClient("") + if err != nil { + t.Fatalf("client: %v", err) + } + + const reqBody = `{"model":"claude","messages":[{"role":"user","content":"hi"}],"tools":[{"name":"t","input_schema":{}}]}` + req, err := http.NewRequest(http.MethodPost, srv.URL, strings.NewReader(reqBody)) + if err != nil { + t.Fatalf("request: %v", err) + } + ctx, capt := llmrec.NewCapture(req.Context()) + req = req.WithContext(ctx) + + resp, err := client.Do(req) + if err != nil { + t.Fatalf("do: %v", err) + } + got, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read: %v", err) + } + _ = resp.Body.Close() + + if string(got) != sse { + t.Fatal("capture altered the response delivered to norma") + } + if capt.RawRequest() != reqBody { + t.Fatalf("RawRequest()=%q want %q", capt.RawRequest(), reqBody) + } + if capt.RawResponse() != sse { + t.Fatalf("RawResponse()=%q want the raw SSE frames", capt.RawResponse()) + } +} + +// A 429 body is read and replaced in-place by the quota check. Capturing must +// still see it, and the replacement body must remain readable downstream. +func TestRoundTripCaptures429BodyAlongsideQuotaRewrite(t *testing.T) { + const body = `{"error":{"message":"insufficient_quota"}}` + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusTooManyRequests) + _, _ = io.WriteString(w, body) + })) + defer srv.Close() + + client, err := quotaAwareHTTPClient("") + if err != nil { + t.Fatalf("client: %v", err) + } + req, err := http.NewRequest(http.MethodPost, srv.URL, strings.NewReader("{}")) + if err != nil { + t.Fatalf("request: %v", err) + } + ctx, capt := llmrec.NewCapture(req.Context()) + req = req.WithContext(ctx) + + resp, err := client.Do(req) + if err != nil { + t.Fatalf("do: %v", err) + } + defer resp.Body.Close() + + // Quota exhaustion is normalized to 402 so the router fails over. + if resp.StatusCode != http.StatusPaymentRequired { + t.Fatalf("status=%d want 402", resp.StatusCode) + } + if capt.RawResponse() != body { + t.Fatalf("RawResponse()=%q want %q", capt.RawResponse(), body) + } + rest, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read replaced body: %v", err) + } + if string(rest) != body { + t.Fatalf("replaced body=%q want it still readable", rest) + } +} + +// Recording off = no Capture on the context. The transport must behave exactly +// as before, including the quota rewrite. +func TestRoundTripWithoutCaptureIsUnchanged(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, "ok") + })) + defer srv.Close() + + client, err := quotaAwareHTTPClient("") + if err != nil { + t.Fatalf("client: %v", err) + } + resp, err := client.Post(srv.URL, "application/json", strings.NewReader("{}")) + if err != nil { + t.Fatalf("post: %v", err) + } + defer resp.Body.Close() + got, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read: %v", err) + } + if string(got) != "ok" { + t.Fatalf("body=%q", got) + } +} diff --git a/db/commands.go b/db/commands.go index 27172da..9a696d0 100644 --- a/db/commands.go +++ b/db/commands.go @@ -99,6 +99,13 @@ type LLMRecord struct { Error string `json:"error,omitempty"` RequestBody string `json:"request_body,omitempty"` ResponseBody string `json:"response_body,omitempty"` + // RawRequest / RawResponse are the untouched HTTP bodies exchanged with the + // provider — the request as buildBody() sent it (full tool schemas included) + // and the raw SSE frames. RequestBody/ResponseBody above are the normalized + // view, which drops tool schemas and tool_use blocks entirely. Empty for + // records written before this was added, or when the call never reached HTTP. + RawRequest string `json:"raw_request,omitempty"` + RawResponse string `json:"raw_response,omitempty"` } const llmRecordsSchema = ` @@ -118,7 +125,9 @@ CREATE TABLE IF NOT EXISTS llm_records ( status TEXT, error TEXT, request_body TEXT, - response_body TEXT + response_body TEXT, + raw_request TEXT, + raw_response TEXT ); CREATE INDEX IF NOT EXISTS idx_llm_records_ts ON llm_records(ts); CREATE INDEX IF NOT EXISTS idx_llm_records_session ON llm_records(session_id); @@ -129,6 +138,8 @@ const llmRecordsMigrate = ` ALTER TABLE llm_records ADD COLUMN IF NOT EXISTS task_id TEXT; ALTER TABLE llm_records ADD COLUMN IF NOT EXISTS worker TEXT; ALTER TABLE llm_records ADD COLUMN IF NOT EXISTS profile_name TEXT; +ALTER TABLE llm_records ADD COLUMN IF NOT EXISTS raw_request TEXT; +ALTER TABLE llm_records ADD COLUMN IF NOT EXISTS raw_response TEXT; ` // EnsureLLMRecordsTable creates the llm_records table if it does not exist. @@ -143,11 +154,12 @@ func (d *DB) EnsureLLMRecordsTable() error { // InsertLLMRecord stores one LLM call record. func (d *DB) InsertLLMRecord(r *LLMRecord) error { _, err := d.Exec(` -INSERT INTO llm_records(model, profile_name, session_id, task_id, worker, latency_ms, input_tokens, output_tokens, cache_read, cache_write, status, error, request_body, response_body) -VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14)`, +INSERT INTO llm_records(model, profile_name, session_id, task_id, worker, latency_ms, input_tokens, output_tokens, cache_read, cache_write, status, error, request_body, response_body, raw_request, raw_response) +VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16)`, r.Model, nullIfEmpty(r.ProfileName), r.SessionID, nullIfEmpty(r.TaskID), nullIfEmpty(r.Worker), r.LatencyMs, r.InputTokens, r.OutputTokens, r.CacheRead, r.CacheWrite, - r.Status, nullIfEmpty(r.Error), nullIfEmpty(r.RequestBody), nullIfEmpty(r.ResponseBody)) + r.Status, nullIfEmpty(r.Error), nullIfEmpty(r.RequestBody), nullIfEmpty(r.ResponseBody), + nullIfEmpty(r.RawRequest), nullIfEmpty(r.RawResponse)) return err } @@ -260,19 +272,21 @@ func (d *DB) DeleteLLMRecords(task string) (int64, error) { // GetLLMRecord returns a single LLM record with full request/response bodies. func (d *DB) GetLLMRecord(id int64) (*LLMRecord, error) { var r LLMRecord - var reqBody, respBody sql.NullString + var reqBody, respBody, rawReq, rawResp sql.NullString err := d.QueryRow(`SELECT id, ts, COALESCE(model,''), COALESCE(profile_name,''), COALESCE(session_id,''), COALESCE(task_id,''), COALESCE(worker,''), COALESCE(latency_ms,0), COALESCE(input_tokens,0), COALESCE(output_tokens,0), COALESCE(cache_read,0), COALESCE(cache_write,0), - COALESCE(status,''), COALESCE(error,''), request_body, response_body + COALESCE(status,''), COALESCE(error,''), request_body, response_body, raw_request, raw_response FROM llm_records WHERE id=$1`, id). Scan(&r.ID, &r.Ts, &r.Model, &r.ProfileName, &r.SessionID, &r.TaskID, &r.Worker, &r.LatencyMs, &r.InputTokens, &r.OutputTokens, &r.CacheRead, &r.CacheWrite, &r.Status, &r.Error, - &reqBody, &respBody) + &reqBody, &respBody, &rawReq, &rawResp) if err != nil { return nil, err } r.RequestBody = reqBody.String r.ResponseBody = respBody.String + r.RawRequest = rawReq.String + r.RawResponse = rawResp.String return &r, nil } diff --git a/db/llm_records_migrate_test.go b/db/llm_records_migrate_test.go new file mode 100644 index 0000000..135f06f --- /dev/null +++ b/db/llm_records_migrate_test.go @@ -0,0 +1,62 @@ +package db + +import ( + "testing" +) + +// The raw-body columns were added after release, so existing installs only get +// them through llmRecordsMigrate. This runs the real migration against the dev +// PG on a table stripped back to its pre-upgrade shape, inside a transaction +// that is always rolled back — PG does transactional DDL, so nothing persists. +func TestLLMRecordsMigrateAddsRawColumnsToOldTable(t *testing.T) { + d, err := Open(testDSN(t)) + if err != nil { + t.Skipf("postgres unavailable (%v) — skipping", err) + } + defer d.Close() + if err := d.EnsureLLMRecordsTable(); err != nil { + t.Fatalf("ensure: %v", err) + } + + tx, err := d.Begin() + if err != nil { + t.Fatalf("begin: %v", err) + } + defer tx.Rollback() //nolint:errcheck // the test never commits + + // Roll the table back to how a pre-upgrade install looks. + if _, err := tx.Exec(`ALTER TABLE llm_records DROP COLUMN IF EXISTS raw_request, DROP COLUMN IF EXISTS raw_response`); err != nil { + t.Fatalf("simulate old table: %v", err) + } + if _, err := tx.Exec(llmRecordsMigrate); err != nil { + t.Fatalf("migrate: %v", err) + } + + for _, col := range []string{"raw_request", "raw_response"} { + var n int + if err := tx.QueryRow( + `SELECT count(*) FROM information_schema.columns + WHERE table_name='llm_records' AND column_name=$1`, col).Scan(&n); err != nil { + t.Fatalf("inspect %s: %v", col, err) + } + if n != 1 { + t.Errorf("column %s missing after migrate", col) + } + } + + // Re-running must stay a no-op (the migration runs on every startup). + if _, err := tx.Exec(llmRecordsMigrate); err != nil { + t.Fatalf("migrate is not idempotent: %v", err) + } + + // An insert carrying raw bodies must round-trip through the migrated table. + var got string + if err := tx.QueryRow( + `INSERT INTO llm_records(model, raw_request, raw_response) VALUES ('m','{"a":1}','data: x') + RETURNING raw_response`).Scan(&got); err != nil { + t.Fatalf("insert into migrated table: %v", err) + } + if got != "data: x" { + t.Errorf("raw_response=%q want %q", got, "data: x") + } +} diff --git a/llmrec/capture.go b/llmrec/capture.go new file mode 100644 index 0000000..d0fea40 --- /dev/null +++ b/llmrec/capture.go @@ -0,0 +1,140 @@ +package llmrec + +import ( + "context" + "fmt" + "io" + "strings" + "sync" +) + +type captureContextKey struct{} + +// Capture collects the untouched wire bodies of one logical LLM call. The +// Recorder creates it and puts it on the context; the HTTP transport that norma +// dials through (agent.quotaAwareTransport) finds it there and fills it in. +// +// This exists because everything the Recorder itself sees is already normalized: +// llm.CompletionRequest is re-serialized rather than the body buildBody() sent, +// and the response arrives as decoded StreamEvents, not the SSE frames. For +// debugging a live provider, the bytes on the wire are the only ground truth. +// +// norma's doStream retries the request-establishment phase, so one Stream can +// issue several HTTP attempts. Each attempt is kept: the discarded ones (see +// norma/llm/retry.go, which closes non-final bodies unread) are exactly what +// makes rate-limit and gateway failures diagnosable. +// +// The transport writes from norma's stream-reading goroutine while the Recorder +// snapshots at stream end, so all state is mutex-guarded. +type Capture struct { + mu sync.Mutex + request string + attempts []*attempt +} + +type attempt struct { + status int + body strings.Builder +} + +// NewCapture returns a context carrying a fresh Capture, plus the Capture itself. +func NewCapture(ctx context.Context) (context.Context, *Capture) { + c := &Capture{} + return context.WithValue(ctx, captureContextKey{}, c), c +} + +// CaptureFrom returns the Capture attached to ctx, or nil when raw capture is +// off. Callers must tolerate nil — recording is a toggle, and non-recorded +// providers dial through the same transport. +func CaptureFrom(ctx context.Context) *Capture { + if ctx == nil { + return nil + } + c, _ := ctx.Value(captureContextKey{}).(*Capture) + return c +} + +// SetRequest stores the outgoing request body. Retries re-send identical bytes, +// so only the first attempt's body is kept. +func (c *Capture) SetRequest(body string) { + if c == nil || body == "" { + return + } + c.mu.Lock() + defer c.mu.Unlock() + if c.request == "" { + c.request = body + } +} + +// TeeResponse opens a new attempt and wraps rc so everything read from it is +// mirrored into that attempt. It tees rather than reads because a successful +// response is an SSE stream that must keep streaming to the caller. +func (c *Capture) TeeResponse(status int, rc io.ReadCloser) io.ReadCloser { + if c == nil || rc == nil { + return rc + } + a := &attempt{status: status} + c.mu.Lock() + c.attempts = append(c.attempts, a) + c.mu.Unlock() + return &teeBody{rc: rc, c: c, a: a} +} + +// RawRequest returns the request body as sent, or "" if nothing was captured. +func (c *Capture) RawRequest() string { + if c == nil { + return "" + } + c.mu.Lock() + defer c.mu.Unlock() + return c.request +} + +// RawResponse returns the response bytes as received. A single attempt yields +// the untouched original (copy-pasteable straight into a replay); multiple +// attempts are concatenated behind per-attempt header lines so a retry sequence +// stays readable. +func (c *Capture) RawResponse() string { + if c == nil { + return "" + } + c.mu.Lock() + defer c.mu.Unlock() + switch len(c.attempts) { + case 0: + return "" + case 1: + return c.attempts[0].body.String() + } + var b strings.Builder + for i, a := range c.attempts { + fmt.Fprintf(&b, "===== attempt %d/%d — HTTP %d =====\n", i+1, len(c.attempts), a.status) + body := a.body.String() + b.WriteString(body) + if !strings.HasSuffix(body, "\n") { + b.WriteByte('\n') + } + } + return b.String() +} + +// teeBody mirrors reads into a Capture attempt, guarded by the Capture's mutex +// so a snapshot taken mid-stream never races the writer. +type teeBody struct { + rc io.ReadCloser + c *Capture + a *attempt +} + +func (t *teeBody) Read(p []byte) (int, error) { + n, err := t.rc.Read(p) + if n > 0 { + t.c.mu.Lock() + t.a.body.Write(p[:n]) + t.c.mu.Unlock() + } + return n, err +} + +func (t *teeBody) Close() error { return t.rc.Close() } diff --git a/llmrec/capture_test.go b/llmrec/capture_test.go new file mode 100644 index 0000000..3386202 --- /dev/null +++ b/llmrec/capture_test.go @@ -0,0 +1,103 @@ +package llmrec + +import ( + "context" + "io" + "strings" + "testing" +) + +func TestCaptureFromNilAndMissing(t *testing.T) { + var nilCtx context.Context // the transport reaches here on any unrecorded call + if CaptureFrom(nilCtx) != nil { + t.Fatal("nil context should yield no capture") + } + if CaptureFrom(context.Background()) != nil { + t.Fatal("bare context should yield no capture") + } +} + +// A nil *Capture is the "recording off" path: the transport calls these on every +// request, so they must all be no-ops rather than panics. +func TestNilCaptureIsInert(t *testing.T) { + var c *Capture + c.SetRequest("body") + if got := c.RawRequest(); got != "" { + t.Fatalf("RawRequest()=%q want empty", got) + } + if got := c.RawResponse(); got != "" { + t.Fatalf("RawResponse()=%q want empty", got) + } + rc := io.NopCloser(strings.NewReader("x")) + if c.TeeResponse(200, rc) != rc { + t.Fatal("nil capture must pass the body through untouched") + } +} + +func TestCaptureSingleAttemptKeepsBytesVerbatim(t *testing.T) { + ctx, c := NewCapture(context.Background()) + if CaptureFrom(ctx) != c { + t.Fatal("capture not retrievable from its own context") + } + c.SetRequest(`{"model":"x"}`) + // Retries re-send identical bytes; only the first is kept. + c.SetRequest(`{"model":"ignored"}`) + + const sse = "event: message_start\ndata: {\"type\":\"message_start\"}\n\nevent: message_stop\ndata: {}\n\n" + body := c.TeeResponse(200, io.NopCloser(strings.NewReader(sse))) + got, err := io.ReadAll(body) + if err != nil { + t.Fatalf("read: %v", err) + } + if string(got) != sse { + t.Fatal("tee altered the stream delivered to the caller") + } + if err := body.Close(); err != nil { + t.Fatalf("close: %v", err) + } + + if c.RawRequest() != `{"model":"x"}` { + t.Fatalf("RawRequest()=%q", c.RawRequest()) + } + // A lone attempt must stay byte-identical — no headers, no framing added, so + // it can be replayed as-is. + if c.RawResponse() != sse { + t.Fatalf("RawResponse()=%q want verbatim SSE", c.RawResponse()) + } +} + +// norma's doStream discards the bodies of retried attempts (retry.go closes them +// unread), so the capture is the only place a 429 body survives. +func TestCaptureRetriedAttemptsAreAllKept(t *testing.T) { + _, c := NewCapture(context.Background()) + + first := c.TeeResponse(429, io.NopCloser(strings.NewReader(`{"error":"rate_limit"}`))) + if _, err := io.ReadAll(first); err != nil { + t.Fatalf("read first: %v", err) + } + second := c.TeeResponse(200, io.NopCloser(strings.NewReader("data: ok\n"))) + if _, err := io.ReadAll(second); err != nil { + t.Fatalf("read second: %v", err) + } + + raw := c.RawResponse() + if !strings.Contains(raw, `{"error":"rate_limit"}`) { + t.Fatalf("dropped the retried attempt body: %q", raw) + } + if !strings.Contains(raw, "data: ok") { + t.Fatalf("dropped the final attempt body: %q", raw) + } + if !strings.Contains(raw, "attempt 1/2 — HTTP 429") || !strings.Contains(raw, "attempt 2/2 — HTTP 200") { + t.Fatalf("attempts not delimited: %q", raw) + } + if strings.Index(raw, "rate_limit") > strings.Index(raw, "data: ok") { + t.Fatal("attempts stored out of order") + } +} + +func TestCaptureEmptyWhenNoHTTPHappened(t *testing.T) { + _, c := NewCapture(context.Background()) + if c.RawRequest() != "" || c.RawResponse() != "" { + t.Fatal("unused capture should be empty") + } +} diff --git a/llmrec/llmrec.go b/llmrec/llmrec.go index 1bbd55c..5fc77d1 100644 --- a/llmrec/llmrec.go +++ b/llmrec/llmrec.go @@ -110,9 +110,15 @@ func (r *Recorder) Stream(ctx context.Context, req llm.CompletionRequest) iter.S } // Serialize the request only when storing bodies (this is the expensive part). + // Alongside it, attach a Capture so the HTTP transport can hand back the + // untouched wire bodies — the normalized view below cannot reconstruct them + // (tool schemas are dropped, tool_use blocks never reach this layer, and the + // SSE framing is already decoded). See capture.go. reqBody := "" + var capt *Capture if recordBodies { reqBody = r.serializeRequest(req) + ctx, capt = NewCapture(ctx) } return func(yield func(llm.StreamEvent, error) bool) { @@ -133,7 +139,7 @@ func (r *Recorder) Stream(ctx context.Context, req llm.CompletionRequest) iter.S r.recordUsage(taskID, expID, worker, usage, int(time.Since(start).Milliseconds()), status) // Heavy trace row — only when body recording is on. if recordBodies { - r.record(req, session, taskID, worker, reqBody, start, textBuf.String(), thinkingBuf.String(), usage, stopReason, err) + r.record(req, session, taskID, worker, reqBody, capt, start, textBuf.String(), thinkingBuf.String(), usage, stopReason, err) } } @@ -210,7 +216,7 @@ func (r *Recorder) recordUsage(taskID string, expID int64, worker string, usage // record persists one LLM call to PostgreSQL before the provider stream returns. // Keeping the write inside the owning task operation means task deletion can // drain calls and then remove records without a late async insert recreating one. -func (r *Recorder) record(req llm.CompletionRequest, session, taskID, worker, reqBody string, start time.Time, text, thinking string, usage llm.Usage, stopReason string, streamErr error) { +func (r *Recorder) record(req llm.CompletionRequest, session, taskID, worker, reqBody string, capt *Capture, start time.Time, text, thinking string, usage llm.Usage, stopReason string, streamErr error) { latency := int(time.Since(start).Milliseconds()) status := "ok" errMsg := "" @@ -256,6 +262,11 @@ func (r *Recorder) record(req llm.CompletionRequest, session, taskID, worker, re Error: errMsg, RequestBody: reqBody, ResponseBody: string(respBody), + // Raw wire bodies; empty when the transport did not fill the Capture + // (e.g. a provider dialing through a client without the capture hook, or + // a call that failed before any HTTP request went out). + RawRequest: capt.RawRequest(), + RawResponse: capt.RawResponse(), } if err := r.pg.InsertLLMRecord(rec); err != nil { diff --git a/server/llmrec_raw_test.go b/server/llmrec_raw_test.go new file mode 100644 index 0000000..673ce1a --- /dev/null +++ b/server/llmrec_raw_test.go @@ -0,0 +1,133 @@ +package server + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/Autumn-27/artex/agent" + "github.com/Autumn-27/artex/llmrec" + "github.com/Autumn-27/norma/llm" + "github.com/Autumn-27/norma/transcript" +) + +// Full chain against a real PG: Recorder → norma provider → capturing transport +// → llm_records. The unit tests cover each hop; this proves the raw bodies +// actually survive all the way into the row an operator opens when debugging. +func TestRecorderPersistsRawWireBodies(t *testing.T) { + m, err := NewManager(t.TempDir(), "") + if err != nil { + t.Skipf("postgres unavailable (%v) — skipping", err) + } + defer m.Close() + pg := m.PG() + if err := pg.EnsureLLMRecordsTable(); err != nil { + t.Fatalf("ensure table: %v", err) + } + + sse := strings.Join([]string{ + `event: message_start`, + `data: {"type":"message_start","message":{"id":"msg_raw","usage":{"input_tokens":5,"output_tokens":1}}}`, + ``, + `event: content_block_delta`, + `data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"probing"}}`, + ``, + // A tool_use block: the normalized response body drops these entirely, so + // its presence in raw_response is the whole point of the feature. + `event: content_block_start`, + `data: {"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_raw","name":"bash"}}`, + ``, + `event: content_block_delta`, + `data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\"command\":\"id\"}"}}`, + ``, + `event: message_delta`, + `data: {"type":"message_delta","delta":{"stop_reason":"tool_use"},"usage":{"output_tokens":9}}`, + ``, + `event: message_stop`, + `data: {"type":"message_stop"}`, + ``, + }, "\n") + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("content-type", "text/event-stream") + _, _ = io.WriteString(w, sse) + })) + defer srv.Close() + + cfg := agent.Config{Format: llm.FormatAnthropic, BaseURL: srv.URL, APIKey: "k", Model: "claude-raw-test"} + inner, err := cfg.NewProvider() + if err != nil { + t.Fatalf("NewProvider: %v", err) + } + const session = "exp999999-rawtest" + rec := llmrec.Wrap(inner, pg, cfg.Model, "raw-test-profile", "", "", func() bool { return true }) + + // Clean up whatever this test writes, whether or not it passes. Must be a + // defer registered after `defer m.Close()` (LIFO puts it first) — a + // t.Cleanup would run after the manager already closed the pool. + defer func() { + if _, err := pg.Exec(`DELETE FROM llm_records WHERE session_id = $1`, session); err != nil { + t.Logf("cleanup: %v", err) + } + }() + + ctx := transcript.WithSessionID(context.Background(), session) + for _, err := range rec.Stream(ctx, llm.CompletionRequest{ + System: []string{"system prompt"}, + Messages: []llm.Message{{Role: "user", Content: []llm.ContentBlock{{Type: "text", Text: "go"}}}}, + Tools: []llm.ToolSchema{{ + Name: "bash", + Description: "run a shell command", + InputSchema: map[string]any{"type": "object"}, + }}, + MaxTokens: 512, + }) { + if err != nil { + t.Fatalf("stream: %v", err) + } + } + + var id int64 + if err := pg.QueryRow(`SELECT id FROM llm_records WHERE session_id=$1 ORDER BY id DESC LIMIT 1`, session).Scan(&id); err != nil { + t.Fatalf("no record written: %v", err) + } + got, err := pg.GetLLMRecord(id) + if err != nil { + t.Fatalf("GetLLMRecord: %v", err) + } + + if got.RawResponse != sse { + t.Errorf("raw_response is not the wire bytes:\n got: %q\nwant: %q", got.RawResponse, sse) + } + // The tool call is absent from the normalized body but present in the raw one + // — the specific gap this feature closes. + if strings.Contains(got.ResponseBody, "toolu_raw") { + t.Error("normalized body unexpectedly contains the tool_use block; test no longer proves the gap") + } + if !strings.Contains(got.RawResponse, "toolu_raw") { + t.Error("raw_response lost the tool_use block") + } + + var body map[string]any + if err := json.Unmarshal([]byte(got.RawRequest), &body); err != nil { + t.Fatalf("raw_request is not valid JSON: %v\n%s", err, got.RawRequest) + } + if body["model"] != "claude-raw-test" { + t.Errorf("raw_request model=%v", body["model"]) + } + tools, _ := body["tools"].([]any) + if len(tools) != 1 { + t.Fatalf("raw_request tools=%v", body["tools"]) + } + if tool, _ := tools[0].(map[string]any); tool["input_schema"] == nil { + t.Errorf("raw_request lost the tool schema: %v", tool) + } + // The normalized request keeps tool names only, so the schema is unique to raw. + if strings.Contains(got.RequestBody, "input_schema") { + t.Error("normalized request unexpectedly carries schemas; test no longer proves the gap") + } +} diff --git a/web/src/app/(main)/function/llm-records/page.tsx b/web/src/app/(main)/function/llm-records/page.tsx index cb70c44..5f05618 100644 --- a/web/src/app/(main)/function/llm-records/page.tsx +++ b/web/src/app/(main)/function/llm-records/page.tsx @@ -9,6 +9,8 @@ import { Loader2Icon, XIcon, Trash2Icon, + CopyIcon, + CheckIcon, } from "lucide-react"; import { @@ -74,6 +76,47 @@ function tryFormatJSON(s: string): string { } } +// 复制当前框内文本的小按钮。复制成功后短暂显示对勾。text 为空/仅占位符时禁用。 +function CopyButton({ text }: { text: string }) { + const [copied, setCopied] = React.useState(false); + const timer = React.useRef | null>(null); + React.useEffect(() => () => { if (timer.current) clearTimeout(timer.current); }, []); + + const disabled = !text; + const copy = async () => { + if (disabled) return; + try { + await navigator.clipboard.writeText(text); + } catch { + // navigator.clipboard 在非安全上下文(如 http 局域网)不可用，回退到 execCommand。 + const ta = document.createElement("textarea"); + ta.value = text; + ta.style.position = "fixed"; + ta.style.opacity = "0"; + document.body.appendChild(ta); + ta.select(); + try { document.execCommand("copy"); } catch { /* 忽略：不支持则静默 */ } + document.body.removeChild(ta); + } + setCopied(true); + if (timer.current) clearTimeout(timer.current); + timer.current = setTimeout(() => setCopied(false), 1500); + }; + + return ( + + ); +} + const PAGE_SIZES = [25, 50, 100]; export default function LLMRecordsPage() { @@ -96,6 +139,21 @@ export default function LLMRecordsPage() { const [selected, setSelected] = React.useState(null); const [detail, setDetail] = React.useState(null); const [detailLoading, setDetailLoading] = React.useState(false); + // 归一化视图 / HTTP 原文视图。原文是排查 provider 侧问题的唯一依据：归一化视图 + // 不含工具 schema，响应里也没有 tool_use 块。 + const [rawView, setRawView] = React.useState(false); + + const hasRaw = !!(detail?.raw_request || detail?.raw_response); + // 开关保持用户选择，但切到一条无原文的旧记录时自动落回解析视图，而不是显示空白。 + const showRaw = rawView && hasRaw; + // 原文请求体是 JSON，pretty-print 只改排版不改语义，便于阅读；原文响应是 SSE + // 帧，tryFormatJSON 解析失败会原样返回，故两边共用一个函数即可。 + const reqText = showRaw + ? detail?.raw_request && tryFormatJSON(detail.raw_request) + : detail?.request_body && tryFormatJSON(detail.request_body); + const respText = showRaw + ? detail?.raw_response + : detail?.response_body && tryFormatJSON(detail.response_body); // Per-task delete (task picker + confirm dialog) const [tasks, setTasks] = React.useState([]); @@ -427,10 +485,22 @@ export default function LLMRecordsPage() { ) : ( Error )} + {/* 原文视图开关。旧记录没有原文，此时禁用而非静默回退，避免看着像 + 「原文与解析一致」。 */} + - {convs.length > 0 && ( -
- toggleAllConversations(checked === true)} - aria-label="选择全部对话" - /> - - {selectedConversationCount > 0 ? `已选 ${selectedConversationCount} 个` : `共 ${convs.length} 个`} - - {selectedConversationCount > 0 && ( + {convs.length > 0 && + (selectionMode ? ( +
+ toggleAllConversations(checked === true)} + aria-label="选择全部对话" + /> + + {selectedConversationCount > 0 ? `已选 ${selectedConversationCount} 个` : `共 ${convs.length} 个`} + + {selectedConversationCount > 0 && ( + + )} + +
+ ) : ( +
+ + 共 {convs.length} 个 + - )} -
- )} +
+ ))}
@@ -1126,6 +1162,7 @@ export default function ChatPage() { onCancelRename={() => setRenamingId(null)} onTogglePinned={() => void togglePinned(c)} onDelete={() => del(c.id)} + selectionMode={selectionMode} selectedForDelete={selectedConversationIds.has(c.id)} onSelectedForDeleteChange={(checked) => toggleConversationSelected(c.id, checked)} /> From 99ac96da2fc01585d8428948dbc024b465972855 Mon Sep 17 00:00:00 2001 From: Autumn-27 <53844290+Autumn-27@users.noreply.github.com> Date: Tue, 25 Aug 2026 12:19:04 -0400 Subject: [PATCH 06/23] =?UTF-8?q?feat(agent):=20=E4=BC=98=E5=8C=96=20plann?= =?UTF-8?q?er/worker=20=E6=8F=90=E7=A4=BA=E8=AF=8D=E5=B9=B6=E5=8A=A0?= =?UTF-8?q?=E9=87=8F=E5=8C=96=E9=AA=8C=E6=94=B6=E6=A0=B8=E5=AF=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - planner 判目标新增「量化验收核对」：覆盖度/flag 数/权限等可量化目标 未达标时禁止 prove_goal，修复覆盖度要求 100%、实测 40% 却被判完成 - 重写 planner「0 意图」判定：仅在「方向已被现有意图覆盖」或「下一步 依赖在跑 work 的产出」两种情况才 0 意图，避免目标未达成时过早收手 - 强化 worker 否定结论证据门槛：未穷尽手段/证据弱一律标 inferred， 避免轻率的 observed 否定把整条路线焊死 - 加 reseedPlannerPrompt / reseedWorkerPrompt 一次性迁移，旧库追加新 默认版本并切换，旧版保留在历史可恢复 --- CHANGELOG.md | 12 ++++++++ agent/planner.go | 6 +++- agent/worker.go | 2 +- server/orchestration.go | 62 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 80 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 09e4d36..3ec251f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,18 @@ ## [Unreleased] +### Agent + +#### 新增的功能 + +- planner 判目标时新增「量化验收核对」：目标含可量化验收条件（资产测试覆盖度达到 X%、拿到 N 个 flag、获得某权限）时，`prove_goal` 前必须核对 `graph_overview` 的实测值（`coverage.pct`、findings 计数等），实测未达标则禁止 `prove_goal`、改为派意图补足差距，不得以「主要部分已完成/大体达成」为由提前标 met。修复了目标要求覆盖度 100%、实测仅 40% 却被判完成的问题。 + +#### 修改的功能 + +- 重写 planner 提示词中「本轮 0 意图」的判定依据：原文将 0 意图描述为「最常见、最重要的原则」，会让规划者在目标未达成、范围内仍有未测面时过早收手。现改为只在两种具体情况下才应产出 0 意图——① 想到的方向都已被 open/running/recent_done 意图覆盖；② 下一步依赖当前正在运行的 work 的产出、而产出尚未出现（应等其跑完、图更新后的下一次唤醒再规划）。并补充反向约束：确有未被覆盖且不依赖在跑 work 的新方向、或目标未达成且仍有未测面时，不要因「0 意图常见」而收手。 +- 强化 worker 提示词中「否定结论的证据门槛」：对「不可注入/端口关闭/无登录入口」等可能让规划者放弃一整条方向的否定结论，要求下结论前先穷尽该意图内的合理手段（换编码/参数/路径/方法）；手段未走完或证据偏弱一律标 `confidence=inferred`，避免用轻率的 `observed` 否定把整条路线焊死（任务早期的错误否定尤其会把方向带偏且难以自愈）。 +- 上述 planner / worker 默认提示词变更通过一次性迁移（`reseedPlannerPrompt` / `reseedWorkerPrompt`，各带 settings flag 守卫）追加为新版本并切为当前版本；旧版本保留在版本历史中，自定义过提示词的用户可在版本记录里恢复。 + ### 运维 #### 新增的功能 diff --git a/agent/planner.go b/agent/planner.go index 01c1bb8..88f2013 100644 --- a/agent/planner.go +++ b/agent/planner.go @@ -242,7 +242,10 @@ const plannerDefaultTmpl = `你是一个授权渗透测试系统的"规划者" 任务目标：{{.Goal}} -⚠️ 最重要的原则：**这一轮生成 0 个意图是完全正常、而且是最常见的结果。** 你不是"每次都要产出意图"的机器。绝大多数唤醒，frontier 里已有的意图已经覆盖了所有已知方向，你应当什么都不加、直接结束。重复/换措辞地生成已经存在的意图是严重错误。 +⚠️ **生成 0 个意图是合法、正常的结果，但它有【具体的正当理由】，不是"少派更稳"的默认姿态。** 只在下列情况才该产出 0 个意图： +- **已覆盖**：你想到的方向都已被 open / running / recent_done 意图覆盖——重复或换措辞地再生成已存在的意图是严重错误；或 +- **等待依赖**：下一步依赖【当前正在运行的 work】的产出，而产出还没出来——此时应【等这些 work 跑完、图更新后的下一次唤醒再规划】，现在硬派会让下游 worker 拿不到还不存在的前置而空转。 +反过来：若确有【未被任何意图覆盖、且不依赖在跑 work】的新方向，或【目标尚未达成且范围内仍有未测面】，就【不要】因为"0 意图常见"而收手——该派就派。别把 0 意图当成偷懒的默认。 决策流程（每次唤醒）： 1. **完整态势已直接附在本提示下方（就是 graph_overview 的返回，无需再调它）**：task（**原始任务标题+目标**，即根节点）、资产计数、goals 及其状态、open/running/recent_done 意图、sites_without_endpoints、findings（**确认漏洞**数）、facts（**探索事实/结论**数，与漏洞是两类）、recent_facts（最近事实的 {id, summary, confidence?}，含"端口关闭/不可注入"等**否定结论**——据此别再为已探明的死路生成意图；但 confidence=inferred 的否定结论只是【推断】、证据弱，别当铁案，若该方向对目标很关键值得派一条复核意图）。**这里的探索节点（goals/意图/facts/findings）都只含本任务的**（绝不会有别的任务的目标）；**资产图则全局共享**（多任务同一份，资产计数是全局在范围内的数据，非本任务独有）。需要更深的细节时才按需调：list_facts 列全部事实、list_findings 列全部漏洞、**node_detail(id)** 取某条的完整证据/详情（列表/recent_facts 只给摘要）。 @@ -250,6 +253,7 @@ const plannerDefaultTmpl = `你是一个授权渗透测试系统的"规划者" - sites_without_endpoints / findings——"哪些方向【可能】需要探索"。 - 只有需要某一片的细节时，才**按需**调 list_assets（pull 模式：可用 q 关键字搜索，可叠加 type/company_id/task_id 过滤，分页 limit/offset；或用 id/ids 直接取）、asset_neighbors、list_findings。资产图全局共享，别默认拉全量。资产中可能包含非本次任务涉及到的资产，所以需要主要出现非本次任务相关的资产时忽略这些资产。 2. 判目标（核心职责）：graph_overview 的 goals 字段已含目标与状态；对已被某发现/事实证明的未达成目标，调 prove_goal(goal_id,evidence_id,reason) 标记 met。**当你用 prove_goal 标记的这一个恰好是最后一个未完成目标时，系统会自动判定整个任务完成**——收官完全由逐个 prove_goal 驱动，你无需、也没有别的“一键完成”手段。 + ⚠️ **量化验收核对（严禁提前盖章）**：若某目标含【可量化的验收条件】（如"资产测试覆盖度达到 X%"、"拿到 N 个 flag"、"获得某权限"），prove_goal **前【必须】核对本提示上方 graph_overview 里的实测值**（coverage.pct、findings 计数等）：实测【未达标】就【禁止】prove_goal，改为派意图补足差距——**【不得】以"主要部分已完成/大体达成/核心目标已拿下"为由提前标 met**。例：目标要求覆盖度 100%、而 graph_overview 实测 coverage.pct=40%，则该目标【未达成】，继续派补测意图，不许 prove_goal。 2.5. **（可选，仅限开局、极轻量）探测理解**：你具备 Bash 等执行能力，它的**唯一正当用途**是——当**图里几乎还没有事实**（recent_facts 基本为空、任务刚开始）、仅凭态势无法把初始意图描述具体时，对目标做**极少量、只读**的探测（如 1–2 次 curl 看首页/指纹），据此产出更精准的**初始意图**。 ⚠️ **牢记你的身份边界：你是"规划者"，不是"执行者"。你在这里做的一切都只为【生成/说清意图】，绝不是【在 plan 里把活干了】。** 探测的**唯一合法产物是一句更精准的意图描述**——绝不能是漏洞的发现、验证、利用，也不能是端点/目录/参数的枚举结果。任何"我顺手把这个也测了/确认了"的念头都是越界：那是 worker 在 work 阶段该做的事，你只需把它**写成一个意图派下去**。 **三条硬性边界，务必守住**： diff --git a/agent/worker.go b/agent/worker.go index b6c6e8c..c75c97b 100644 --- a/agent/worker.go +++ b/agent/worker.go @@ -169,7 +169,7 @@ const workerDefaultTmpl = `你是一个授权渗透测试系统的"执行者"(wo 可用工具： - insert_assets：登记新资产（资产图）。**你只传原始信息，key 与父子关联由代码算**：新接口→传完整 url+method（代码自动建 domain→site→endpoint、自动抽 URL 里的 query 参数，body/header 参数放 params）；指纹→type=tech,name=技术名,on_url=站点地址,props填{version,category}。多个资产放 assets 数组一次批量登记。属性写在资产自己的 props 上，探索结论不要写这里。 -- record_fact：把探索【事实/结论】写入探索图并连到意图（传 intent_id）。正向/否定结论、观察、判断用它；**一次探索的多个观察汇总成一条事实**（summary 总结一句话 + detail 放细节），不要一个属性一条。真有多条不同结论才用 facts 数组。**只写你真实看到的**：给 evidence（一行关键证据：命令+关键输出，简洁，别粘大段——细节在 detail）、标 confidence（observed 直接看到 / inferred 推断）；**否定结论**（不可注入/端口关闭等）尤其要给证据、证据弱就标 inferred，别让错的否定误导规划者放弃方向。 +- record_fact：把探索【事实/结论】写入探索图并连到意图（传 intent_id）。正向/否定结论、观察、判断用它；**一次探索的多个观察汇总成一条事实**（summary 总结一句话 + detail 放细节），不要一个属性一条。真有多条不同结论才用 facts 数组。**只写你真实看到的**：给 evidence（一行关键证据：命令+关键输出，简洁，别粘大段——细节在 detail）、标 confidence（observed 直接看到 / inferred 推断）；**否定结论的证据门槛**（不可注入/端口关闭/无登录入口等【可能让规划者放弃一整条方向】的结论）：下结论前先确认你已【穷尽这条意图内的合理手段】（换编码/换参数/换路径/换方法）；手段没走完、或证据只是"看起来像/大概率如此"，一律标 confidence=inferred。**宁可标 inferred 让规划者复核，也别用一个轻率的 observed 否定把一整条路线焊死**——尤其任务早期，一个错误的 observed 否定会把整个任务带偏、且后续很难自己扳回来。 - report_finding：确认漏洞 → 记录(含 PoC，传 intent_id=你领到的意图id)。**只有你在本次运行里真实触发过该漏洞、拿到了可复现的证据（请求/响应或命令输出）才用它。** 严禁把下列当作已确认漏洞上报：仅凭版本号/指纹匹配到某 CVE、仅凭"参数看起来可注入"、仅凭外部漏洞库/更新日志/代码 diff 推断。**不要用查 CVE 库或"对比补丁版本"替代实际触发。** 触发不了但确有嫌疑，就用 record_fact 记一条 confidence=inferred 的事实（描述嫌疑点+为何未能触发），交给规划者派后续意图，别硬记成 finding。 - list_assets（查询资产，非探索节点） / asset_neighbors / list_facts(探索事实) / list_findings(漏洞) / node_detail(探索节点 id，非资产 id)：按需查上下文。 diff --git a/server/orchestration.go b/server/orchestration.go index 9057ea7..287eaa0 100644 --- a/server/orchestration.go +++ b/server/orchestration.go @@ -452,6 +452,8 @@ func (s *Server) seedOrchestrationTools() { s.unbindGoalMetDefault() s.reseedGoalsPrompt() // goals 提示词加入「抽操作约束」步 → 旧库追加一版新默认(一次性) s.reseedMainAgentPrompt() // mainagent 提示词加入「目标达成后 add_intent 反问是否建目标」(一次性) + s.reseedPlannerPrompt() // planner 提示词:重写「0 意图」正当理由 + 加量化验收核对(一次性) + s.reseedWorkerPrompt() // worker 提示词:加否定结论证据门槛(一次性) s.seedReporterAgent() // 预置「报告撰写」agent + 工具绑定 + finding 触发器(一次性) // 注：pentest 的默认工具绑定无需迁移——BuiltinToolSeeds 在全新初始化时就把 // list_assets/insert_assets/report_finding/list_findings/list_companies 连同 @@ -571,6 +573,66 @@ func (s *Server) reseedMainAgentPrompt() { log.Printf("[prompts] mainagent 提示词已追加新默认版本(加入目标达成后反问建目标,一次性)") } +// reseedPlannerPrompt 把 planner 提示词刷成【当前代码默认】——默认正文重写了「0 意图」的正当理由 +// (从笼统"最常见"改为"仅在已覆盖/等待在跑 work 依赖时才 0 意图")、并新增「量化验收核对」(覆盖度等 +// 可量化目标未达标禁止 prove_goal)。SeedPromptIfEmpty 首插入only,旧库已有版本收不到,故用版本管理 +// 【追加一个新版本】并切过去(ResetPromptToDefault),旧版本仍保留在历史里,用户若自定义过可从版本记录 +// 找回。settings flag 守卫 → 只做一次。全新库无需处理(SeedPromptIfEmpty 已 seed 最新默认)。与 +// reseedGoalsPrompt 完全同构。 +func (s *Server) reseedPlannerPrompt() { + const flag = "planner_prompt_zerointent_acceptance_v1" + if v, _, _ := s.m.pg.GetSetting(flag); v == "true" { + return + } + defer func() { _ = s.m.pg.SetSetting(flag, "true") }() // 无论成功与否只尝试一次 + a, err := s.m.pg.GetAgentByKey("planner") + if err != nil || a == nil { + return // 全新库尚未建 agent 行时,seedPrompts 会直接 seed 最新默认,无需此迁移 + } + tmpl := agent.BuiltinPromptSeeds()["planner"] + if tmpl == "" { + return + } + // 全新库 seedPrompts 已 seed 最新默认 → 当前版本已等于代码默认,不必再追加重复版本。 + if cur, err := s.m.pg.CurrentPrompt(a.ID); err == nil && cur == tmpl { + return + } + if _, err := s.m.pg.ResetPromptToDefault(a.ID, tmpl); err != nil { + log.Printf("[prompts] planner 提示词重刷为新默认失败: %v", err) + return + } + log.Printf("[prompts] planner 提示词已追加新默认版本(重写0意图理由+加量化验收核对,一次性)") +} + +// reseedWorkerPrompt 把 worker 提示词刷成【当前代码默认】——默认正文强化了「否定结论的证据门槛」 +// (未穷尽手段/证据弱一律标 inferred,别用轻率 observed 否定焊死路线)。SeedPromptIfEmpty 首插入only, +// 旧库已有版本收不到,故用版本管理【追加一个新版本】并切过去,旧版本仍保留在历史里可找回。settings flag +// 守卫 → 只做一次。全新库无需处理。与 reseedGoalsPrompt 完全同构。 +func (s *Server) reseedWorkerPrompt() { + const flag = "worker_prompt_negfact_threshold_v1" + if v, _, _ := s.m.pg.GetSetting(flag); v == "true" { + return + } + defer func() { _ = s.m.pg.SetSetting(flag, "true") }() // 无论成功与否只尝试一次 + a, err := s.m.pg.GetAgentByKey("worker") + if err != nil || a == nil { + return // 全新库尚未建 agent 行时,seedPrompts 会直接 seed 最新默认,无需此迁移 + } + tmpl := agent.BuiltinPromptSeeds()["worker"] + if tmpl == "" { + return + } + // 全新库 seedPrompts 已 seed 最新默认 → 当前版本已等于代码默认,不必再追加重复版本。 + if cur, err := s.m.pg.CurrentPrompt(a.ID); err == nil && cur == tmpl { + return + } + if _, err := s.m.pg.ResetPromptToDefault(a.ID, tmpl); err != nil { + log.Printf("[prompts] worker 提示词重刷为新默认失败: %v", err) + return + } + log.Printf("[prompts] worker 提示词已追加新默认版本(加否定结论证据门槛,一次性)") +} + // seedReporterAgent 预置一个「报告撰写」自定义 agent(builtin=false，可在 UI 编辑/删除)： // 绑定 update_finding_report + 任务查询工具，并挂一个「report_finding 被调用即触发」的 // 触发器 —— 每登记一个漏洞就唤起它写详细报告。一次性(settings flag 守卫)：用户删掉后不再重建。 From 48f260fdde5fc04e01443adbfd2ad9d355d84229 Mon Sep 17 00:00:00 2001 From: Autumn-27 <53844290+Autumn-27@users.noreply.github.com> Date: Tue, 25 Aug 2026 12:22:06 -0400 Subject: [PATCH 07/23] =?UTF-8?q?docs(changelog):=20=E5=8F=91=E5=B8=83=200?= =?UTF-8?q?.3.5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ec251f..f6564cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ## [Unreleased] +## [0.3.5] - 2026-08-25 + ### Agent #### 新增的功能 @@ -22,6 +24,10 @@ - 新增 `reset-password.sh` 重置管理员（用户名固定 `ARTEX`）密码：支持 local / docker 两种部署，连接信息可显式指定或自动从 `--dsn`/`$ARTEX_PG_DSN`/`config.json` 读取；在库内用 `pgcrypto` 生成与后端登录兼容的 bcrypt 哈希并写回 `settings.auth.password_hash`，重置后无需重启服务。密码经环境变量传入、不进入进程 argv，并做转义防注入。 +### 贡献者 + +- [@Autumn-27](https://github.com/Autumn-27) + ## [0.3.4] - 2026-08-24 ### UI @@ -261,7 +267,8 @@ - [@Autumn-27](https://github.com/Autumn-27) - [@neouks](https://github.com/neouks) -[Unreleased]: https://github.com/Autumn-27/ARTEX/compare/v0.3.4...HEAD +[Unreleased]: https://github.com/Autumn-27/ARTEX/compare/v0.3.5...HEAD +[0.3.5]: https://github.com/Autumn-27/ARTEX/compare/v0.3.4...v0.3.5 [0.3.4]: https://github.com/Autumn-27/ARTEX/compare/v0.3.3...v0.3.4 [0.3.3]: https://github.com/Autumn-27/ARTEX/compare/v0.3.2...v0.3.3 [0.3.2]: https://github.com/Autumn-27/ARTEX/compare/v0.3.1...v0.3.2 From f6c4cbb013db9eaf700e3a39db74492af4254e29 Mon Sep 17 00:00:00 2001 From: Autumn-27 <53844290+Autumn-27@users.noreply.github.com> Date: Tue, 25 Aug 2026 12:27:38 -0400 Subject: [PATCH 08/23] =?UTF-8?q?docs(changelog):=20=E8=A1=A5=E5=85=A8=200?= =?UTF-8?q?.3.5=20=E9=81=97=E6=BC=8F=E6=9D=A1=E7=9B=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 补上 v0.3.4 以来已合入但未记录的变更：LLM 流式/非流式切换、 OpenAI Responses API 格式、LLM 请求/响应原文录制、会话列表多选模式、 以及 #57 的任务/会话/流量体验增强；0.3.5 贡献者补记 @neouks。 --- CHANGELOG.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f6564cd..f9cb758 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,30 @@ ## [0.3.5] - 2026-08-25 +### LLM + +#### 新增的功能 + +- 每个 LLM 配置支持流式/非流式切换（默认流式）：开走流式 SSE；关走真·非流式（`stream:false`、一次性返回完整 JSON），可绕开部分网关糟糕的 SSE 实现（空帧、思考字段丢帧），代价是失去运行中的实时进度与实时 Token 计数。worker / planner / mainagent / chat / goals 均按当前激活配置动态取值；`llmpool` / `llmrec` / 任务运行时三层 Provider 包装均兼容非流式；`llm_profiles` 新增 `streaming` 列并 `ALTER` 补旧库（默认 `true`，旧配置无感）。 +- 支持 OpenAI Responses API 格式的 LLM 配置：每个配置新增第三种格式 `openai-responses`（打 `POST /v1/responses`），与 Chat Completions / Anthropic 并列；`BaseURL` 归一化、默认模型 `gpt-5`；`llm_profiles` 的 `format` 约束加入 `openai-responses` 并幂等迁移补旧库；前端格式下拉新增「OpenAI (Responses API)」。依赖升级 norma v0.3.1（含 `reasoning_content` 回传修复）。 +- 录制 LLM 请求/响应 HTTP 原文：在 HTTP transport 层捕获真实 wire body，保留归一化视图看不到的工具 schema、`tool_use` 块与原始 SSE 帧；norma 内部重试的多次尝试逐次保留；`llm_records` 新增 `raw_request` / `raw_response` 两列并 `ALTER` 补旧库；录制页详情面板增加「原文」视图切换与请求/响应复制按钮（兼容非安全上下文的 `execCommand` 回退）。 + +### 对话 + +#### 修改的功能 + +- 会话列表多选改为「多选」模式开关：默认列表不再常驻每行勾选框（观感更干净），表头改为「共 N 个 + 多选」按钮；点「多选」进入选择模式（勾选框、全选、批量删除），「完成」退出并清空选择，批量删除全部成功后自动退出（有失败则留在选择模式便于重试）；单条重命名 / 置顶 / 删除仍走每行 ⋯ 菜单。 + +### UI + +#### 修改的功能 + +- 任务管理、会话操作与流量查看体验增强（#57）：任务列表列排序偏好可持久化记忆、行内重命名改用图标直接触发（不再经菜单）、抽屉（sheet）交互与流量查看细节优化。 + +#### 修复的问题 + +- Worker 资产标签只显示域名与 IP，并正确处理空标签的场景。 + ### Agent #### 新增的功能 @@ -27,6 +51,7 @@ ### 贡献者 - [@Autumn-27](https://github.com/Autumn-27) +- [@neouks](https://github.com/neouks) ## [0.3.4] - 2026-08-24 From f891f4ffb8f47ac247ee033e17bca4f9c7903e9c Mon Sep 17 00:00:00 2001 From: Autumn-27 <53844290+Autumn-27@users.noreply.github.com> Date: Thu, 27 Aug 2026 10:00:04 -0400 Subject: [PATCH 09/23] =?UTF-8?q?chore(deps):=20=E5=8D=87=E7=BA=A7=20norma?= =?UTF-8?q?=20v0.3.1=20=E2=86=92=20v0.3.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 带入本轮 norma 修复： - OpenAI 系响应里被静默丢弃的 reasoning/refusal，Responses 补 reasoning_text - reasoning 字段三名去重 - 空响应(零内容块)有界重试 - 压缩边界切断工具配对导致的网关 400（同时修复已烤进 transcript 的存量孤儿） --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index e79c704..ae76393 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/Autumn-27/artex go 1.26.3 require ( - github.com/Autumn-27/norma v0.3.1 + github.com/Autumn-27/norma v0.3.2 github.com/golang-jwt/jwt/v5 v5.3.1 github.com/jackc/pgx/v5 v5.10.0 github.com/lqqyt2423/go-mitmproxy v1.9.2 diff --git a/go.sum b/go.sum index 1ba35d1..23fac3a 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/Autumn-27/norma v0.3.1 h1:cSRojFJjivm4l3ck/KJXym7VyCnM1GLY7iq1pRYk4kk= github.com/Autumn-27/norma v0.3.1/go.mod h1:BudWrf7iHl96FgHqFr/HwN3mKYmGjJDt+xL2ZHXfFZw= +github.com/Autumn-27/norma v0.3.2 h1:3gW84SbSrz8luW8kbi2tv3O7CLdRMQvyylCHj1QUdNk= +github.com/Autumn-27/norma v0.3.2/go.mod h1:BudWrf7iHl96FgHqFr/HwN3mKYmGjJDt+xL2ZHXfFZw= github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro= github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= From 8be7631eb3bef1b121509985da13d8704612c7fc Mon Sep 17 00:00:00 2001 From: Autumn-27 <53844290+Autumn-27@users.noreply.github.com> Date: Thu, 27 Aug 2026 10:23:06 -0400 Subject: [PATCH 10/23] =?UTF-8?q?fix(llm):=20=E9=85=8D=E7=BD=AE=E5=88=97?= =?UTF-8?q?=E8=A1=A8=E8=BF=94=E5=9B=9E=20streaming=20=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit LLMProfileDTO 漏了 streaming，接口从不返回该字段，前端读到 undefined 后一律回落成默认的流式，导致「非流式」保存成功但重新打开开关仍是开的。 字段不加 omitempty：false 必须出现在响应里。 Closes #69 --- server/dto.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/dto.go b/server/dto.go index 8db7ba7..3b1f7dc 100644 --- a/server/dto.go +++ b/server/dto.go @@ -566,6 +566,9 @@ type LLMProfileDTO struct { // pool_exclude=true 则不作为故障转移目标，但仍可被 agent/任务显式绑定。 Priority int `json:"priority"` PoolExclude bool `json:"pool_exclude"` + // 收发模式：true=流式(SSE) | false=非流式。没有 omitempty —— false 必须出现在 + // 响应里，否则前端读不到「非流式」，开关会回落成默认的流式。 + Streaming bool `json:"streaming"` } func llmProfileDTO(p *db.LLMProfile) LLMProfileDTO { @@ -585,6 +588,7 @@ func llmProfileDTO(p *db.LLMProfile) LLMProfileDTO { IsDefault: p.IsDefault, Priority: p.Priority, PoolExclude: p.PoolExclude, + Streaming: p.Streaming, } } From e10f8413dcf0ae6f8c979866bfe323c9e4244511 Mon Sep 17 00:00:00 2001 From: Autumn-27 <53844290+Autumn-27@users.noreply.github.com> Date: Thu, 27 Aug 2026 10:56:11 -0400 Subject: [PATCH 11/23] =?UTF-8?q?feat(commands):=20=E5=B7=A5=E5=85=B7?= =?UTF-8?q?=E6=89=A7=E8=A1=8C=E9=A1=B5=E6=96=B0=E5=A2=9E=E8=B0=83=E7=94=A8?= =?UTF-8?q?=E6=AC=A1=E6=95=B0=E7=BB=9F=E8=AE=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 「工具执行」页工具栏加「统计」按钮，弹窗按工具展示调用次数、占比与 失败次数，按次数降序。统计沿用列表的 task/q 筛选，算的是整个结果集 而非当前页；弹窗打开时才请求，翻页不会多付一次聚合查询。 - db: 抽出 commandFilter,列表与统计共用同一份 WHERE;新增 ToolStats - server: GET /api/commands/stats - web: 统计弹窗 + demo 模式 mock --- db/commands.go | 69 +++++++++++--- server/commands.go | 40 ++++++-- server/server.go | 1 + web/src/app/(main)/function/commands/page.tsx | 93 ++++++++++++++++++- web/src/lib/api.ts | 8 ++ web/src/lib/mock/handler.ts | 11 +++ web/src/lib/types.ts | 7 ++ 7 files changed, 207 insertions(+), 22 deletions(-) diff --git a/db/commands.go b/db/commands.go index 9a696d0..d95f0b8 100644 --- a/db/commands.go +++ b/db/commands.go @@ -19,18 +19,10 @@ type CommandRecord struct { CreatedAt time.Time `json:"created_at"` } -// ListCommands returns tool executions (tool_use + paired tool_result) across all -// explorations, with optional filtering and pagination. Covers every tool, not -// just Bash; q matches the tool name or its input. -func (d *DB) ListCommands(expID *int64, q string, page, size int) ([]CommandRecord, int, error) { - if size <= 0 { - size = 50 - } - if page < 0 { - page = 0 - } - offset := page * size - +// commandFilter builds the WHERE clause shared by the tool-execution list and +// its per-tool tally, so the summary always describes exactly the rows the table +// pages through. Returns the clause, its args, and the next placeholder index. +func commandFilter(expID *int64, q string) (string, []any, int) { where := `WHERE u.kind = 'tool_use'` args := []any{} argN := 1 @@ -45,6 +37,59 @@ func (d *DB) ListCommands(expID *int64, q string, page, size int) ([]CommandReco args = append(args, "%"+q+"%") argN++ } + return where, args, argN +} + +// ToolStat is one tool's execution tally for the usage summary. +type ToolStat struct { + Tool string `json:"tool"` + Total int `json:"total"` + Errors int `json:"errors"` +} + +// ToolStats counts executions grouped by tool under the same filters +// ListCommands takes. Unpaginated on purpose: the tally describes the whole +// filtered set, not the page currently on screen. +func (d *DB) ToolStats(expID *int64, q string) ([]ToolStat, error) { + where, args, _ := commandFilter(expID, q) + + rows, err := d.Query(` +SELECT COALESCE(NULLIF(u.tool,''),'-') AS tool, COUNT(*) AS total, + COUNT(*) FILTER (WHERE COALESCE(r.is_error,false)) AS errors +FROM activity u +LEFT JOIN activity r ON r.tool_use_id = u.tool_use_id AND r.kind = 'tool_result' +`+where+` +GROUP BY 1 +ORDER BY total DESC, tool ASC`, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + out := []ToolStat{} + for rows.Next() { + var s ToolStat + if err := rows.Scan(&s.Tool, &s.Total, &s.Errors); err != nil { + return nil, err + } + out = append(out, s) + } + return out, rows.Err() +} + +// ListCommands returns tool executions (tool_use + paired tool_result) across all +// explorations, with optional filtering and pagination. Covers every tool, not +// just Bash; q matches the tool name or its input. +func (d *DB) ListCommands(expID *int64, q string, page, size int) ([]CommandRecord, int, error) { + if size <= 0 { + size = 50 + } + if page < 0 { + page = 0 + } + offset := page * size + + where, args, argN := commandFilter(expID, q) // count var total int diff --git a/server/commands.go b/server/commands.go index 12904a7..aa05427 100644 --- a/server/commands.go +++ b/server/commands.go @@ -15,14 +15,7 @@ func (s *Server) pgListCommands(w http.ResponseWriter, r *http.Request) { size := atoiDefault(q.Get("size"), 50) keyword := q.Get("q") - var expID *int64 - if tv := q.Get("task"); tv != "" { - if n, err := strconv.ParseInt(tv, 10, 64); err == nil { - expID = &n - } - } - - records, total, err := s.m.PG().ListCommands(expID, keyword, page, size) + records, total, err := s.m.PG().ListCommands(commandTaskFilter(q.Get("task")), keyword, page, size) if err != nil { writeErr(w, 500, err.Error()) return @@ -30,6 +23,37 @@ func (s *Server) pgListCommands(w http.ResponseWriter, r *http.Request) { writeJSON(w, 200, map[string]any{"commands": records, "total": total}) } +// pgToolStats returns per-tool call counts for the tool-execution history, under +// the same task/keyword filters the list takes — the summary describes the whole +// filtered set, not the page on screen. +func (s *Server) pgToolStats(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + pg := s.m.PG() + if pg == nil { + writeJSON(w, 200, map[string]any{"stats": []db.ToolStat{}}) + return + } + stats, err := pg.ToolStats(commandTaskFilter(q.Get("task")), q.Get("q")) + if err != nil { + writeErr(w, 500, err.Error()) + return + } + writeJSON(w, 200, map[string]any{"stats": stats}) +} + +// commandTaskFilter parses the ?task= exploration id. Absent or unparsable → nil +// (no filter), matching the list endpoint's long-standing lenient behaviour. +func commandTaskFilter(v string) *int64 { + if v == "" { + return nil + } + n, err := strconv.ParseInt(v, 10, 64) + if err != nil { + return nil + } + return &n +} + // pgListLLMRecords returns paginated LLM call records. func (s *Server) pgListLLMRecords(w http.ResponseWriter, r *http.Request) { q := r.URL.Query() diff --git a/server/server.go b/server/server.go index 303ef72..dca9193 100644 --- a/server/server.go +++ b/server/server.go @@ -787,6 +787,7 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("GET /api/traffic/exchange", s.getTrafficExchange) mux.HandleFunc("GET /api/traffic/blob", s.getTrafficBlob) mux.HandleFunc("GET /api/commands", s.pgListCommands) + mux.HandleFunc("GET /api/commands/stats", s.pgToolStats) // 按工具聚合调用次数 mux.HandleFunc("GET /api/llm/records", s.pgListLLMRecords) mux.HandleFunc("DELETE /api/llm/records", s.pgDeleteLLMRecords) mux.HandleFunc("GET /api/llm/records/tasks", s.pgLLMTasks) diff --git a/web/src/app/(main)/function/commands/page.tsx b/web/src/app/(main)/function/commands/page.tsx index 7d58b48..6a5954e 100644 --- a/web/src/app/(main)/function/commands/page.tsx +++ b/web/src/app/(main)/function/commands/page.tsx @@ -2,17 +2,18 @@ import * as React from "react"; -import { ChevronLeftIcon, ChevronRightIcon, Loader2Icon, SearchIcon, TerminalIcon } from "lucide-react"; +import { BarChart3Icon, ChevronLeftIcon, ChevronRightIcon, Loader2Icon, SearchIcon, TerminalIcon } from "lucide-react"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { Card } from "@/components/ui/card"; +import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle } from "@/components/ui/dialog"; import { Input } from "@/components/ui/input"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { Sheet, SheetContent, SheetDescription, SheetHeader, SheetTitle } from "@/components/ui/sheet"; import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; import { api } from "@/lib/api"; -import type { CommandRecord } from "@/lib/types"; +import type { CommandRecord, ToolStat } from "@/lib/types"; import { cn } from "@/lib/utils"; function fmtTime(ts: string) { @@ -59,6 +60,11 @@ export default function CommandsPage() { const [total, setTotal] = React.useState(0); const [loading, setLoading] = React.useState(false); + // 各工具调用次数。弹窗打开时才拉取（多一次聚合查询，不必每次翻页都付）。 + const [statsOpen, setStatsOpen] = React.useState(false); + const [stats, setStats] = React.useState([]); + const [statsLoading, setStatsLoading] = React.useState(false); + // Selected execution is rendered in a right-side detail sheet. const [selected, setSelected] = React.useState(null); @@ -96,6 +102,24 @@ export default function CommandsPage() { }; }, [page, size, queryQ, taskFilter]); + // 统计跟随筛选条件走，和表格描述的是同一批记录（但不分页）。 + React.useEffect(() => { + if (!statsOpen) return; + let alive = true; + setStatsLoading(true); + api + .commandStats({ task: taskFilter || undefined, q: queryQ || undefined }) + .then((r) => alive && setStats(r.stats ?? [])) + .catch(() => alive && setStats([])) + .finally(() => alive && setStatsLoading(false)); + return () => { + alive = false; + }; + }, [statsOpen, queryQ, taskFilter]); + + const statsTotal = stats.reduce((n, s) => n + s.total, 0); + const statsErrors = stats.reduce((n, s) => n + s.errors, 0); + const statsMax = stats.reduce((n, s) => Math.max(n, s.total), 0); const totalPages = Math.max(1, Math.ceil(total / size)); const rangeStart = total === 0 ? 0 : page * size + 1; const rangeEnd = page * size + commands.length; @@ -141,6 +165,11 @@ export default function CommandsPage() { + +
{rangeStart}–{rangeEnd} / {total} @@ -243,6 +272,66 @@ export default function CommandsPage() {
+ {/* 工具调用统计：与表格同一批记录（同筛选、不分页） */} + + + + 工具调用统计 + + {taskFilter || queryQ ? "当前筛选条件下的全部记录" : "全部工具执行记录"} + {stats.length > 0 && ( + <> + {" · "} + {stats.length} 个工具 ·{" "} + {statsTotal} 次调用 + {statsErrors > 0 && ( + <> + {" · 失败 "} + {statsErrors} + + )} + + )} + + + + {statsLoading && stats.length === 0 ? ( +
+ +
+ ) : stats.length === 0 ? ( +
暂无统计数据
+ ) : ( +
+ {stats.map((s) => ( +
+
+
+ {s.tool} + {s.errors > 0 && ( + 失败 {s.errors} + )} +
+
+
0 ? (s.total / statsMax) * 100 : 0}%` }} + /> +
+
+
+
{s.total}
+
+ {statsTotal > 0 ? ((s.total / statsTotal) * 100).toFixed(1) : "0.0"}% +
+
+
+ ))} +
+ )} + +
+ !open && setSelected(null)}> {selected && ( diff --git a/web/src/lib/api.ts b/web/src/lib/api.ts index f606f6c..6a0232e 100644 --- a/web/src/lib/api.ts +++ b/web/src/lib/api.ts @@ -71,6 +71,7 @@ import type { TokenTotal, TokenUsage, Tool, + ToolStat, TrafficDetail, TrafficHost, TrafficResp, @@ -941,6 +942,13 @@ export const api = { sp.set("size", String(params?.size ?? 50)); return get<{ commands: CommandRecord[]; total: number }>(`/commands?${sp}`); }, + // 各工具调用次数；沿用列表的 task/q 筛选，统计的是整个结果集而非当前页。 + commandStats: (params?: { task?: string; q?: string }) => { + const sp = new URLSearchParams(); + if (params?.task) sp.set("task", params.task); + if (params?.q) sp.set("q", params.q); + return get<{ stats: ToolStat[] }>(`/commands/stats?${sp}`); + }, // ---- LLM records ---- llmRecords: (params?: { model?: string; session?: string; task?: string; page?: number; size?: number }) => { diff --git a/web/src/lib/mock/handler.ts b/web/src/lib/mock/handler.ts index 6e9b4fd..9e69c14 100644 --- a/web/src/lib/mock/handler.ts +++ b/web/src/lib/mock/handler.ts @@ -1128,6 +1128,17 @@ function route(m: string, path: string, seg: string[], q: URLSearchParams, b: Re // ── 工具执行历史 ── if (path === "/commands" && m === "GET") return { commands: D.commandRecords, total: D.commandRecords.length }; + if (path === "/commands/stats" && m === "GET") { + const tally = new Map(); + for (const c of D.commandRecords) { + const tool = c.tool || "-"; + const s = tally.get(tool) ?? { tool, total: 0, errors: 0 }; + s.total++; + if (c.is_error) s.errors++; + tally.set(tool, s); + } + return { stats: [...tally.values()].sort((a, b) => b.total - a.total || a.tool.localeCompare(b.tool)) }; + } // ── LLM ── if (path === "/llm/records" && m === "GET") return { records: mockLLMRecords, total: mockLLMRecords.length }; diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts index 0569277..092871d 100644 --- a/web/src/lib/types.ts +++ b/web/src/lib/types.ts @@ -1065,6 +1065,13 @@ export interface CommandRecord { created_at: string; } +// 单个工具的调用统计（/commands/stats）；errors 为其中失败的次数。 +export interface ToolStat { + tool: string; + total: number; + errors: number; +} + // ---- LLM recording ---- export interface LLMRecordItem { id: number; From 7b1f7e141b3e5deba717094c7f4ba7718e8fd433 Mon Sep 17 00:00:00 2001 From: Autumn-27 <53844290+Autumn-27@users.noreply.github.com> Date: Thu, 27 Aug 2026 11:14:48 -0400 Subject: [PATCH 12/23] =?UTF-8?q?fix(llm):=20=E6=B5=8B=E8=AF=95=E8=BF=9E?= =?UTF-8?q?=E6=8E=A5=E6=A0=A1=E9=AA=8C=E6=A8=A1=E5=9E=8B=E7=A1=AE=E6=9C=89?= =?UTF-8?q?=E5=9B=9E=E5=A4=8D=EF=BC=8C=E5=B9=B6=E6=8C=89=E9=85=8D=E7=BD=AE?= =?UTF-8?q?=E7=9A=84=E6=94=B6=E5=8F=91=E6=A8=A1=E5=BC=8F=E6=B5=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 测试连接此前只判 err：请求通了但模型一个字不吐(思考烧光预算、正文被 安全策略吞掉、兼容层丢 content)照样报成功，而这种配置在会话里就是不 回话——正是「测试成功、对话不成功」的来源。 - TestConnection 返回回复文本，空回复直接判失败 - 回复回传前端，成功 toast 里展示，测试通过有据可查 - 测试连接改用该配置真实的收发模式：此前恒走流式(ConfigFrom 默认 Stream=true),选了非流式的配置测的是另一条通道；请求省略 streaming 时仍默认流式，老客户端不受影响 Closes #65 --- agent/provider.go | 22 +++++++++++++++++----- server/server.go | 25 +++++++++++++++++++++++-- web/src/app/(main)/system/llm/page.tsx | 18 ++++++++++++++++-- web/src/lib/api.ts | 5 ++++- web/src/lib/mock/handler.ts | 3 ++- 5 files changed, 62 insertions(+), 11 deletions(-) diff --git a/agent/provider.go b/agent/provider.go index ba12d5e..09b277b 100644 --- a/agent/provider.go +++ b/agent/provider.go @@ -357,11 +357,12 @@ func quotaAwareHTTPClient(proxy string) (*http.Client, error) { } // TestConnection makes a minimal real completion to verify the provider/model/ -// endpoint/key actually work. Returns the round-trip latency. -func TestConnection(ctx context.Context, c Config) (time.Duration, error) { +// endpoint/key actually work. Returns the round-trip latency and the model's +// reply text. +func TestConnection(ctx context.Context, c Config) (time.Duration, string, error) { prov, err := c.NewProvider() if err != nil { - return 0, err + return 0, "", err } ctx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() @@ -371,7 +372,7 @@ func TestConnection(ctx context.Context, c Config) (time.Duration, error) { // 就撞到输出上限(finish=length)、被截断,连接测试虽仍算通(err=nil)但显示成 // "已中断/length/resume" 一团糟。给足预算让它把 OK 干净吐完(finish=stop)。 // EscalateMaxTokens 保持 false:不因截断而抬额重试,避免 resume 循环空烧。 - _, err = agentcore.Run(ctx, agentcore.Options{ + reply, err := agentcore.Run(ctx, agentcore.Options{ Provider: prov, SystemPrompt: []string{"你是连接测试。直接输出两个字符 OK 即可，不要思考、不要解释、不要别的。"}, PermissionMode: acperm.ModeBypass, @@ -379,5 +380,16 @@ func TestConnection(ctx context.Context, c Config) (time.Duration, error) { MaxTokens: 8192, NonStreaming: !c.Stream, // 用该 profile 的真实收发模式做连接测试 }, "ping") - return time.Since(start), err + lat := time.Since(start) + if err != nil { + return lat, "", err + } + // err==nil 还不够：请求通了但模型一个字都不吐的情况真实存在(思考把预算烧光、 + // 正文被安全策略吞掉、兼容层把 content 丢了)。这种配置在会话里就是"不回话", + // 测试却报成功——正是本项要消除的落差。没有可见正文一律判失败。 + reply = strings.TrimSpace(reply) + if reply == "" { + return lat, "", fmt.Errorf("模型无回复内容（请求已通，但未返回任何文本）") + } + return lat, reply, nil } diff --git a/server/server.go b/server/server.go index dca9193..f7ab8c7 100644 --- a/server/server.go +++ b/server/server.go @@ -1337,6 +1337,7 @@ func (s *Server) testLLM(w http.ResponseWriter, r *http.Request) { ThinkingType string `json:"thinking_type"` ReasoningEffort string `json:"reasoning_effort"` ProfileID *int64 `json:"profile_id"` // 测已存 profile 时传入：api_key 为空则用它存的 key + Streaming *bool `json:"streaming"` // 省略=流式，与保存 profile 时同一套默认 } if err := json.NewDecoder(r.Body).Decode(&req); err != nil { writeErr(w, 400, err.Error()) @@ -1347,6 +1348,11 @@ func (s *Server) testLLM(w http.ResponseWriter, r *http.Request) { // reasoning_effort/thinking field fails the test too (no false "test ok, run 400"). cfg.ThinkingType = req.ThinkingType cfg.ReasoningEffort = req.ReasoningEffort + // 同理，收发模式也照该 profile 的选择来：只支持其中一种通道的端点必须在这里就 + // 暴露，而不是等会话里才发现"测试通过的配置根本跑不动"。 + if req.Streaming != nil { + cfg.Stream = *req.Streaming + } // API Key 解析优先级：表单输入 > 指定 profile 存的 key > 全局配置的 key。 // 已存 profile 的 key 不回传浏览器，所以测试已存配置时表单为空，需从 DB 取。 if cfg.APIKey == "" && req.ProfileID != nil { @@ -1363,12 +1369,27 @@ func (s *Server) testLLM(w http.ResponseWriter, r *http.Request) { writeJSON(w, 200, map[string]any{"ok": false, "error": "未提供 API Key"}) return } - lat, err := agent.TestConnection(r.Context(), cfg) + lat, reply, err := agent.TestConnection(r.Context(), cfg) if err != nil { writeJSON(w, 200, map[string]any{"ok": false, "error": err.Error()}) return } - writeJSON(w, 200, map[string]any{"ok": true, "latency_ms": lat.Milliseconds(), "model": cfg.Model}) + // 回传模型实际回复，让"测试通过"有据可查：看得见它确实说了话，而不只是 HTTP 200。 + writeJSON(w, 200, map[string]any{ + "ok": true, "latency_ms": lat.Milliseconds(), "model": cfg.Model, "reply": truncateReply(reply), + }) +} + +// truncateReply clips a connection-test reply for display. A model told to answer +// "OK" can still ramble (or think out loud); the UI only needs enough to show it +// really said something. Rune-based so multibyte text never splits mid-character. +func truncateReply(s string) string { + const max = 200 + r := []rune(s) + if len(r) <= max { + return s + } + return string(r[:max]) + "…" } type createTaskReq struct { diff --git a/web/src/app/(main)/system/llm/page.tsx b/web/src/app/(main)/system/llm/page.tsx index 3a3e5af..7065a29 100644 --- a/web/src/app/(main)/system/llm/page.tsx +++ b/web/src/app/(main)/system/llm/page.tsx @@ -369,8 +369,22 @@ function ProfileSheet({ try { // 用配置实际会跑的思考参数来测，这样不支持该字段的模型在这里就失败， // 而不是等到跑任务时才炸。传 profile id：Key 输入框留空时用已存的 Key。 - const r = await api.testLLM(format, model, baseUrl, apiKey, proxy, toStore(thinkingType), toStore(effort), profileId); - if (r.ok) toast.success(`连接成功 · ${r.latency_ms ?? "?"}ms · ${r.model ?? model}`); + const r = await api.testLLM( + format, + model, + baseUrl, + apiKey, + proxy, + toStore(thinkingType), + toStore(effort), + profileId, + streaming, + ); + // 回复内容一并展示：看得见模型确实说了话，才算和会话里跑通是一回事。 + if (r.ok) + toast.success(`连接成功 · ${r.latency_ms ?? "?"}ms · ${r.model ?? model}`, { + description: r.reply ? `回复：${r.reply}` : undefined, + }); else toast.error(`连接失败：${r.error ?? "未知"}`); } catch (e) { toast.error(`测试出错：${(e as Error).message}`); diff --git a/web/src/lib/api.ts b/web/src/lib/api.ts index 6a0232e..03ce937 100644 --- a/web/src/lib/api.ts +++ b/web/src/lib/api.ts @@ -646,8 +646,10 @@ export const api = { thinking_type = "", reasoning_effort = "", profile_id?: number, + streaming = true, // 用该配置真实的收发模式来测，别让"流式能通、非流式不通"漏到会话里 ) => - post<{ ok: boolean; error?: string; latency_ms?: number; model?: string }>("/llm/test", { + // reply = 模型实际回复(已截断);一个字都不回的配置后端直接判失败 + post<{ ok: boolean; error?: string; latency_ms?: number; model?: string; reply?: string }>("/llm/test", { provider, model, base_url, @@ -656,6 +658,7 @@ export const api = { thinking_type, reasoning_effort, profile_id, + streaming, }), llmProfiles: () => get<{ profiles: LLMProfile[] }>("/llm/profiles").then((r) => arr(r.profiles)), saveLLMProfile: (p: { diff --git a/web/src/lib/mock/handler.ts b/web/src/lib/mock/handler.ts index 9e69c14..dd13347 100644 --- a/web/src/lib/mock/handler.ts +++ b/web/src/lib/mock/handler.ts @@ -1155,7 +1155,8 @@ function route(m: string, path: string, seg: string[], q: URLSearchParams, b: Re } if (path === "/llm" && m === "GET") return D.llmConfig; if (path === "/llm" && m === "POST") return { ok: true }; - if (path === "/llm/test") return { ok: true, latency_ms: 128, model: String(b.model ?? "claude-opus-4-8") }; + if (path === "/llm/test") + return { ok: true, latency_ms: 128, model: String(b.model ?? "claude-opus-4-8"), reply: "OK" }; if (path === "/llm/profiles" && m === "GET") return { profiles: D.llmProfiles }; if (path === "/llm/profiles" && m === "POST") return { id: Number(b.id) || 3 }; if (path === "/llm/profiles/active") return { ok: true }; From 10f6a4fbc7bde0edec0934594fc116b270ba54b8 Mon Sep 17 00:00:00 2001 From: Autumn-27 <53844290+Autumn-27@users.noreply.github.com> Date: Thu, 27 Aug 2026 12:15:02 -0400 Subject: [PATCH 13/23] =?UTF-8?q?feat(agent):=20list=5Ffacts=20=E6=94=B9?= =?UTF-8?q?=E4=B8=BA=E5=88=86=E9=A1=B5+=E5=85=B3=E9=94=AE=E8=AF=8D?= =?UTF-8?q?=E8=BF=87=E6=BB=A4=EF=BC=8C=E9=81=BF=E5=85=8D=E4=B8=8A=E4=B8=8B?= =?UTF-8?q?=E6=96=87=E8=86=A8=E8=83=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fact 多时旧的 list_facts 一次返回每任务至多 500 条且 summary 不截断， 一次调用就可能撑爆上下文。改为分页(默认 20，最新在前)+ q 关键词过滤， 返回 {facts, total, has_more, next_before}，summary 超长按 rune 截断。 - db: listByKindPageFiltered/countByKindFiltered(payload->>'summary' ILIKE) + ListByKindPageWithSources 跨本任务/继承任务合并分页 - agent: list_facts 加 limit/before/q 参数，compactFact 截断 summary； worker/planner 提示词同步说明分页语义 - server: 旧库一次性迁移把 list_facts 新 schema 刷进 tools 目录表 (SeedTool 首插入only，否则工具管理页显示「无参数」)；flag bump 到 v7 --- agent/blackboard_inheritance_test.go | 2 +- agent/planner.go | 2 +- agent/tools.go | 57 +++++++++++++++++++++++++--- agent/worker.go | 2 +- db/exploration.go | 34 +++++++++++++++++ db/exploration_sources.go | 47 +++++++++++++++++++++++ server/orchestration.go | 6 ++- 7 files changed, 139 insertions(+), 11 deletions(-) diff --git a/agent/blackboard_inheritance_test.go b/agent/blackboard_inheritance_test.go index fcbcb6f..4611bc1 100644 --- a/agent/blackboard_inheritance_test.go +++ b/agent/blackboard_inheritance_test.go @@ -223,7 +223,7 @@ func TestBlackboardToolsReadDirectSources(t *testing.T) { t.Fatalf("inherited host context missing: %#v", coverage) } - facts := callReadJSON(t, tools.listFacts(), `{}`).([]any) + facts := callReadJSON(t, tools.listFacts(), `{}`).(map[string]any)["facts"].([]any) seenSource, seenGrand := false, false for _, raw := range facts { item := raw.(map[string]any) diff --git a/agent/planner.go b/agent/planner.go index 88f2013..95802f8 100644 --- a/agent/planner.go +++ b/agent/planner.go @@ -248,7 +248,7 @@ const plannerDefaultTmpl = `你是一个授权渗透测试系统的"规划者" 反过来：若确有【未被任何意图覆盖、且不依赖在跑 work】的新方向，或【目标尚未达成且范围内仍有未测面】，就【不要】因为"0 意图常见"而收手——该派就派。别把 0 意图当成偷懒的默认。 决策流程（每次唤醒）： -1. **完整态势已直接附在本提示下方（就是 graph_overview 的返回，无需再调它）**：task（**原始任务标题+目标**，即根节点）、资产计数、goals 及其状态、open/running/recent_done 意图、sites_without_endpoints、findings（**确认漏洞**数）、facts（**探索事实/结论**数，与漏洞是两类）、recent_facts（最近事实的 {id, summary, confidence?}，含"端口关闭/不可注入"等**否定结论**——据此别再为已探明的死路生成意图；但 confidence=inferred 的否定结论只是【推断】、证据弱，别当铁案，若该方向对目标很关键值得派一条复核意图）。**这里的探索节点（goals/意图/facts/findings）都只含本任务的**（绝不会有别的任务的目标）；**资产图则全局共享**（多任务同一份，资产计数是全局在范围内的数据，非本任务独有）。需要更深的细节时才按需调：list_facts 列全部事实、list_findings 列全部漏洞、**node_detail(id)** 取某条的完整证据/详情（列表/recent_facts 只给摘要）。 +1. **完整态势已直接附在本提示下方（就是 graph_overview 的返回，无需再调它）**：task（**原始任务标题+目标**，即根节点）、资产计数、goals 及其状态、open/running/recent_done 意图、sites_without_endpoints、findings（**确认漏洞**数）、facts（**探索事实/结论**数，与漏洞是两类）、recent_facts（最近事实的 {id, summary, confidence?}，含"端口关闭/不可注入"等**否定结论**——据此别再为已探明的死路生成意图；但 confidence=inferred 的否定结论只是【推断】、证据弱，别当铁案，若该方向对目标很关键值得派一条复核意图）。**这里的探索节点（goals/意图/facts/findings）都只含本任务的**（绝不会有别的任务的目标）；**资产图则全局共享**（多任务同一份，资产计数是全局在范围内的数据，非本任务独有）。需要更深的细节时才按需调：list_facts 分页列事实（最新在前，默认 20 条，可传 q 关键词过滤、before 翻页，返回带 total/has_more）、list_findings 列全部漏洞、**node_detail(id)** 取某条的完整证据/详情（列表/recent_facts 只给摘要）。 - open_intents / running_intents / recent_done_intents——"哪些方向已经有意图在覆盖/已尝试"。每个意图还带 **parents（上游：它派生自哪些事实/意图）和 yields（下游：它产生了哪些事实/发现）**——这就是探索图的**血缘关系**，据此理解"哪些事实来自哪个方向、能否综合成新方向"。recent_facts 里每个事实带 **from_intent**（由哪个意图产生）。 - sites_without_endpoints / findings——"哪些方向【可能】需要探索"。 - 只有需要某一片的细节时，才**按需**调 list_assets（pull 模式：可用 q 关键字搜索，可叠加 type/company_id/task_id 过滤，分页 limit/offset；或用 id/ids 直接取）、asset_neighbors、list_findings。资产图全局共享，别默认拉全量。资产中可能包含非本次任务涉及到的资产，所以需要主要出现非本次任务相关的资产时忽略这些资产。 diff --git a/agent/tools.go b/agent/tools.go index f8493ab..ebed6d8 100644 --- a/agent/tools.go +++ b/agent/tools.go @@ -898,19 +898,64 @@ func (t *ToolSet) listFindings() actool.CoreTool { }) } +// factsPageSize is the default page size for list_facts. Facts pile up on long +// tasks; returning all of them at once (the old behaviour) could blow up the +// context, so default to the newest page and let the agent page/filter for more. +const factsPageSize = 20 + func (t *ToolSet) listFacts() actool.CoreTool { - return readTool("list_facts", "列本任务及直接关联任务的【探索事实/结论】(紧凑：id+摘要+状态)。关联任务条目带 source_task_id/inherited=true 且只读。详情用 node_detail(id)，漏洞看 list_findings。", - obj(map[string]any{}), - func(context.Context, json.RawMessage) (actool.Result, error) { - f, _ := t.ts.ListByKindWithSources(db.KindFact, 500) + return readTool("list_facts", "分页列本任务及直接关联任务的【探索事实/结论】，最新在前(紧凑：id+摘要+状态，摘要过长会截断，全文用 node_detail(id))。参数均可选：limit(默认 20，上限 100)、before(游标，传上一页返回的 next_before 取更旧的一页；省略/0=最新一页)、q(按摘要关键词过滤)。返回 {facts, total, has_more, next_before}：total 是过滤后的总数，has_more=true 时用 next_before 继续翻页。关联任务条目带 source_task_id/inherited=true 且只读。漏洞看 list_findings。", + obj(map[string]any{ + "limit": intp("返回条数，默认 20，上限 100"), + "before": intp("分页游标：只返回 id 小于该值的更旧事实；省略或 0 = 最新一页"), + "q": str("按事实摘要关键词过滤（不区分大小写）；省略 = 不过滤"), + }), + func(_ context.Context, in json.RawMessage) (actool.Result, error) { + var a struct { + Limit int `json:"limit"` + Before int64 `json:"before"` + Q string `json:"q"` + } + _ = json.Unmarshal(in, &a) + limit := a.Limit + if limit <= 0 { + limit = factsPageSize + } + if limit > 100 { + limit = 100 + } + f, hasMore, total, err := t.ts.ListByKindPageWithSources(db.KindFact, a.Before, limit, strings.TrimSpace(a.Q)) + if err != nil { + return actool.Result{}, err + } out := make([]map[string]any, 0, len(f)) for _, n := range f { - out = append(out, compactNode(n)) + out = append(out, compactFact(n)) } - return jsonResult(out) + res := map[string]any{"facts": out, "total": total, "has_more": hasMore} + if hasMore && len(f) > 0 { + res["next_before"] = f[len(f)-1].ID // 传回它取下一页(更旧的) + } + return jsonResult(res) }) } +// factSummaryMax caps a fact summary in list_facts output. Facts carry one-line +// conclusions, but nothing enforces brevity; a runaway summary must not bloat a +// whole page. Full text stays available via node_detail(id). +const factSummaryMax = 160 + +// compactFact is compactNode with the summary rune-capped for list_facts, so a +// page of facts stays bounded regardless of how long any single summary grew. +func compactFact(n *db.Node) map[string]any { + m := compactNode(n) + if s, ok := m["summary"].(string); ok && len([]rune(s)) > factSummaryMax { + m["summary"] = string([]rune(s)[:factSummaryMax]) + "…" + m["summary_truncated"] = true + } + return m +} + func (t *ToolSet) nodeDetail() actool.CoreTool { return readTool("node_detail", "按 id 取本任务或直接关联任务的【探索图节点】完整内容。继承节点带 source_task_id/inherited=true 且只读。仅限 list_facts/list_findings/graph_overview 返回的探索节点 id；资产请用 list_assets/asset_neighbors。", obj(map[string]any{"id": idp("探索图节点 id(非资产 id)")}, "id"), diff --git a/agent/worker.go b/agent/worker.go index c75c97b..2c77722 100644 --- a/agent/worker.go +++ b/agent/worker.go @@ -171,7 +171,7 @@ const workerDefaultTmpl = `你是一个授权渗透测试系统的"执行者"(wo - insert_assets：登记新资产（资产图）。**你只传原始信息，key 与父子关联由代码算**：新接口→传完整 url+method（代码自动建 domain→site→endpoint、自动抽 URL 里的 query 参数，body/header 参数放 params）；指纹→type=tech,name=技术名,on_url=站点地址,props填{version,category}。多个资产放 assets 数组一次批量登记。属性写在资产自己的 props 上，探索结论不要写这里。 - record_fact：把探索【事实/结论】写入探索图并连到意图（传 intent_id）。正向/否定结论、观察、判断用它；**一次探索的多个观察汇总成一条事实**（summary 总结一句话 + detail 放细节），不要一个属性一条。真有多条不同结论才用 facts 数组。**只写你真实看到的**：给 evidence（一行关键证据：命令+关键输出，简洁，别粘大段——细节在 detail）、标 confidence（observed 直接看到 / inferred 推断）；**否定结论的证据门槛**（不可注入/端口关闭/无登录入口等【可能让规划者放弃一整条方向】的结论）：下结论前先确认你已【穷尽这条意图内的合理手段】（换编码/换参数/换路径/换方法）；手段没走完、或证据只是"看起来像/大概率如此"，一律标 confidence=inferred。**宁可标 inferred 让规划者复核，也别用一个轻率的 observed 否定把一整条路线焊死**——尤其任务早期，一个错误的 observed 否定会把整个任务带偏、且后续很难自己扳回来。 - report_finding：确认漏洞 → 记录(含 PoC，传 intent_id=你领到的意图id)。**只有你在本次运行里真实触发过该漏洞、拿到了可复现的证据（请求/响应或命令输出）才用它。** 严禁把下列当作已确认漏洞上报：仅凭版本号/指纹匹配到某 CVE、仅凭"参数看起来可注入"、仅凭外部漏洞库/更新日志/代码 diff 推断。**不要用查 CVE 库或"对比补丁版本"替代实际触发。** 触发不了但确有嫌疑，就用 record_fact 记一条 confidence=inferred 的事实（描述嫌疑点+为何未能触发），交给规划者派后续意图，别硬记成 finding。 -- list_assets（查询资产，非探索节点） / asset_neighbors / list_facts(探索事实) / list_findings(漏洞) / node_detail(探索节点 id，非资产 id)：按需查上下文。 +- list_assets（查询资产，非探索节点） / asset_neighbors / list_facts(探索事实，分页最新在前；默认 20 条，可传 q 关键词过滤、before 翻页) / list_findings(漏洞) / node_detail(探索节点 id，非资产 id)：按需查上下文。 只在授权范围内操作；若系统提示顶部附有【操作约束】，那是最高优先级红线——任何命令/探测在执行前先自检是否违反，违反即不做（哪怕它落在你领到的意图里）。完成本意图后用一句话总结你做了什么、写回了哪些事实。务实、克制、聚焦这一条意图。` diff --git a/db/exploration.go b/db/exploration.go index 3a8c563..2ede3b5 100644 --- a/db/exploration.go +++ b/db/exploration.go @@ -672,6 +672,40 @@ ORDER BY id DESC LIMIT $4`, s.expID, kind, before, limit+1) return nodes, hasMore, nil } +// listByKindPageFiltered is ListByKindPage with an optional summary keyword +// filter (payload->>'summary' ILIKE %q%). Fetches one extra row so the caller can +// probe hasMore. Empty q = no filter. Newest-first by id. +func (s *ExplorationStore) listByKindPageFiltered(kind string, before int64, limit int, q string) ([]*Node, error) { + where := `exploration_id=$1 AND kind=$2 AND ($3 <= 0 OR id < $3)` + args := []any{s.expID, kind, before} + if q != "" { + args = append(args, "%"+q+"%") + where += fmt.Sprintf(` AND payload->>'summary' ILIKE $%d`, len(args)) + } + args = append(args, limit+1) + rows, err := s.db.Query(`SELECT `+nodeCols+` FROM exploration_nodes +WHERE `+where+` ORDER BY id DESC LIMIT $`+fmt.Sprintf("%d", len(args)), args...) + if err != nil { + return nil, err + } + defer rows.Close() + return scanNodes(rows) +} + +// countByKindFiltered counts nodes of a kind under the same summary filter, for a +// page's total. Empty q = count all of that kind. +func (s *ExplorationStore) countByKindFiltered(kind, q string) (int, error) { + where := `exploration_id=$1 AND kind=$2` + args := []any{s.expID, kind} + if q != "" { + args = append(args, "%"+q+"%") + where += fmt.Sprintf(` AND payload->>'summary' ILIKE $%d`, len(args)) + } + var n int + err := s.db.QueryRow(`SELECT COUNT(*) FROM exploration_nodes WHERE `+where, args...).Scan(&n) + return n, err +} + // GetNode returns one node of this exploration by id (nil, nil if not found). func (s *ExplorationStore) GetNode(id int64) (*Node, error) { n, err := scanNode(s.db.QueryRow(`SELECT `+nodeCols+` FROM exploration_nodes WHERE id=$1 AND exploration_id=$2`, id, s.expID)) diff --git a/db/exploration_sources.go b/db/exploration_sources.go index 3ccbb96..c12d5ba 100644 --- a/db/exploration_sources.go +++ b/db/exploration_sources.go @@ -108,6 +108,53 @@ func (s *ExplorationStore) ListByKindWithSources(kind string, limit int) ([]*Nod return out, nil } +// ListByKindPageWithSources is the paginated, keyword-filterable sibling of +// ListByKindWithSources: it returns one newest-first page (id < before, before<=0 +// = newest) spanning this exploration and its direct sources, plus hasMore and the +// filtered total across all of them. Node ids are globally unique, so merging each +// store's own page and re-sorting by id DESC yields the true global page; fetching +// limit+1 per store guarantees the merged top-`limit` is complete. +func (s *ExplorationStore) ListByKindPageWithSources(kind string, before int64, limit int, q string) (nodes []*Node, hasMore bool, total int, err error) { + if limit <= 0 { + limit = 20 + } + own, err := s.listByKindPageFiltered(kind, before, limit, q) + if err != nil { + return nil, false, 0, err + } + merged := own + total, err = s.countByKindFiltered(kind, q) + if err != nil { + return nil, false, 0, err + } + + sources, err := s.DirectSourceStores() + if err != nil { + return nil, false, 0, err + } + for _, source := range sources { + page, err := source.Store.listByKindPageFiltered(kind, before, limit, q) + if err != nil { + return nil, false, 0, err + } + for _, n := range page { + merged = append(merged, markInheritedNode(n, source.Task.TaskID)) + } + cnt, err := source.Store.countByKindFiltered(kind, q) + if err != nil { + return nil, false, 0, err + } + total += cnt + } + + sort.Slice(merged, func(i, j int) bool { return merged[i].ID > merged[j].ID }) + hasMore = len(merged) > limit + if hasMore { + merged = merged[:limit] + } + return merged, hasMore, total, nil +} + // GetNodeWithSources reads a node only when it belongs to this exploration or // one of its direct sources. Inherited nodes are tagged so tool callers can keep // them read-only and show their provenance. diff --git a/server/orchestration.go b/server/orchestration.go index 287eaa0..08f0167 100644 --- a/server/orchestration.go +++ b/server/orchestration.go @@ -466,7 +466,7 @@ func (s *Server) seedOrchestrationTools() { // reaches an old DB otherwise. Preserves each tool's agent binding + enabled flag. // Bump the flag whenever these tools' schemas/descriptions change in code. func (s *Server) refreshBuiltinToolSchemas() { - const flag = "tool_schema_refresh_v6_insert_assets_related" + const flag = "tool_schema_refresh_v7_list_facts_paging" if v, _, _ := s.m.pg.GetSetting(flag); v == "true" { return } @@ -482,7 +482,9 @@ func (s *Server) refreshBuiltinToolSchemas() { // “结束空轮”的手段、刚开跑就误判整个任务完成。 // - insert_assets：新增 related 入参(标记资产是否与当前任务相关、决定是否入覆盖度)， // SeedTool 首插入only，旧库已 seed 的 schema 否则收不到这个新参数。 - refreshBuiltin := map[string]bool{"goal_met": true, "insert_assets": true} + // - list_facts：改为分页，新增 limit/before/q 入参；旧库已 seed 的空 schema 否则 + // 在工具管理页显示「无参数」，模型也拿不到这几个参数说明。 + refreshBuiltin := map[string]bool{"goal_met": true, "insert_assets": true, "list_facts": true} for _, sd := range agent.BuiltinToolSeeds() { if !refreshBuiltin[sd.Key] { continue From 051ca84a20bbda634cbc40b32211e0a546852070 Mon Sep 17 00:00:00 2001 From: Autumn-27 <53844290+Autumn-27@users.noreply.github.com> Date: Thu, 27 Aug 2026 12:34:51 -0400 Subject: [PATCH 14/23] =?UTF-8?q?docs(changelog):=20=E5=8F=91=E5=B8=83=200?= =?UTF-8?q?.3.6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f9cb758..b6e5616 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,37 @@ ## [Unreleased] +## [0.3.6] - 2026-08-27 + +### LLM + +#### 修复的问题 + +- 修复「非流式」开关保存后重新打开又变回「流式」的问题（#69）：配置列表接口的 DTO 漏了 `streaming` 字段，响应从不返回它，前端读到 `undefined` 后一律回落成默认流式；实际写入与 DB 存储本无问题，只是读不回来。DTO 补回该字段（不加 `omitempty`，`false` 也必须出现在响应里）。 +- 测试连接改为校验模型确有回复、并按该配置真实的收发模式来测（#65）：此前只判 HTTP 是否成功，请求通但模型零回复（思考烧光预算 / 正文被安全策略吞掉 / 兼容层丢 `content`）也报「连接成功」，与会话里「不回话」的表现割裂；且测试恒走流式，非流式配置测的其实是另一条通道。现在空回复直接判失败、成功时在提示里展示模型回复，并把当前流式开关一并带入测试，让「测试通过」与「会话跑得通」保持一致。 + +### Agent + +#### 修改的功能 + +- `list_facts` 改为分页 + 关键词过滤，避免事实多时单次调用撑爆上下文（#74）：默认返回最新 20 条，支持 `limit`（上限 100）/ `before` 游标 / `q` 摘要关键词，返回 `{facts, total, has_more, next_before}`；单条 summary 过长按字数截断（全文仍走 `node_detail`）。worker / planner 提示词同步为分页语义。旧库通过一次性迁移把新参数 schema 刷进工具目录表（`SeedTool` 首插入only，否则工具管理页显示「无参数」）。 + +### 工具 + +#### 新增的功能 + +- 「工具执行」页新增工具调用次数统计（#72）：工具栏「统计」按钮打开弹窗，按工具展示调用次数、占比与失败次数（次数降序）；沿用列表的任务 / 关键词筛选，统计整个结果集而非当前页，弹窗打开时才拉取。 + +### 依赖 + +#### 修改的功能 + +- 升级 norma v0.3.1 → v0.3.2：修复 OpenAI 系响应里被静默丢弃的 reasoning/refusal（Responses 补 `reasoning_text`）、reasoning 字段三名去重、空响应（零内容块）有界重试、压缩边界切断工具配对导致的网关 400（同时修复已烤进 transcript 的存量孤儿）。 + +### 贡献者 + +- [@Autumn-27](https://github.com/Autumn-27) + ## [0.3.5] - 2026-08-25 ### LLM From 22132a4d21e1e43a654f608ac4160398914a6dae Mon Sep 17 00:00:00 2001 From: pphua Date: Fri, 28 Aug 2026 14:03:07 +0800 Subject: [PATCH 15/23] chore: sync upstream v0.3.6 changes --- CHANGELOG.md | 43 +++++++ agent/blackboard_inheritance_test.go | 2 +- agent/capture_usage_test.go | 11 ++ agent/chat.go | 44 +++++--- agent/goals.go | 7 +- agent/mainagent.go | 32 ++++-- agent/planner.go | 38 ++++--- agent/provider.go | 41 ++++++- agent/tools.go | 57 +++++++++- agent/worker.go | 14 ++- db/commands.go | 69 ++++++++++-- db/config.go | 25 +++-- db/exploration.go | 34 ++++++ db/exploration_sources.go | 47 ++++++++ db/schema.sql | 4 + go.mod | 4 +- go.sum | 4 +- llmpool/pool.go | 38 +++++++ llmpool/pool_test.go | 11 ++ llmrec/llmrec.go | 47 +++++++- llmrec/llmrec_test.go | 36 ++++++ server/commands.go | 40 +++++-- server/dto.go | 4 + server/goals.go | 3 +- server/orchestration.go | 10 +- server/server.go | 50 ++++++++- server/server_mgmt.go | 9 +- server/task_llm.go | 106 +++++++++++++++++- server/task_llm_test.go | 21 ++++ web/src/app/(main)/function/commands/page.tsx | 93 ++++++++++++++- web/src/app/(main)/system/llm/page.tsx | 43 +++++-- web/src/lib/api.ts | 14 ++- web/src/lib/mock/handler.ts | 14 ++- web/src/lib/types.ts | 9 ++ 34 files changed, 897 insertions(+), 127 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a3fbdeb..f38f7b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,49 @@ ## [Unreleased] +## [0.3.6] - 2026-08-27 + +### LLM + +#### 修复的问题 + +- 修复「非流式」开关保存后重新打开又变回「流式」的问题（#69）：配置列表接口的 DTO 漏了 `streaming` 字段，响应从不返回它，前端读到 `undefined` 后一律回落成默认流式；实际写入与 DB 存储本无问题，只是读不回来。DTO 补回该字段（不加 `omitempty`，`false` 也必须出现在响应里）。 +- 测试连接改为校验模型确有回复、并按该配置真实的收发模式来测（#65）：此前只判 HTTP 是否成功，请求通但模型零回复（思考烧光预算 / 正文被安全策略吞掉 / 兼容层丢 `content`）也报「连接成功」，与会话里「不回话」的表现割裂；且测试恒走流式，非流式配置测的其实是另一条通道。现在空回复直接判失败、成功时在提示里展示模型回复，并把当前流式开关一并带入测试，让「测试通过」与「会话跑得通」保持一致。 + +### Agent + +#### 修改的功能 + +- `list_facts` 改为分页 + 关键词过滤，避免事实多时单次调用撑爆上下文（#74）：默认返回最新 20 条，支持 `limit`（上限 100）/ `before` 游标 / `q` 摘要关键词，返回 `{facts, total, has_more, next_before}`；单条 summary 过长按字数截断（全文仍走 `node_detail`）。worker / planner 提示词同步为分页语义。旧库通过一次性迁移把新参数 schema 刷进工具目录表（`SeedTool` 首插入only，否则工具管理页显示「无参数」）。 + +### 工具 + +#### 新增的功能 + +- 「工具执行」页新增工具调用次数统计（#72）：工具栏「统计」按钮打开弹窗，按工具展示调用次数、占比与失败次数（次数降序）；沿用列表的任务 / 关键词筛选，统计整个结果集而非当前页，弹窗打开时才拉取。 + +### 依赖 + +#### 修改的功能 + +- 升级 norma v0.2.9 → v0.3.2：修复 OpenAI 系响应里被静默丢弃的 reasoning/refusal（Responses 补 `reasoning_text`）、reasoning 字段三名去重、空响应（零内容块）有界重试、压缩边界切断工具配对导致的网关 400（同时修复已烤进 transcript 的存量孤儿）。 + +### 贡献者 + +- [@Autumn-27](https://github.com/Autumn-27) + +## [0.3.5] - 2026-08-25 + +### LLM + +#### 新增的功能 + +- 每个 LLM 配置支持流式/非流式切换（默认流式）：开走流式 SSE；关走真·非流式（`stream:false`、一次性返回完整 JSON），可绕开部分网关糟糕的 SSE 实现（空帧、思考字段丢帧），代价是失去运行中的实时进度与实时 Token 计数。worker / planner / mainagent / chat / goals 均按当前激活配置动态取值；`llmpool` / `llmrec` / 任务运行时三层 Provider 包装均兼容非流式；`llm_profiles` 新增 `streaming` 列并 `ALTER` 补旧库（默认 `true`，旧配置无感）。 + +### 贡献者 + +- [@Autumn-27](https://github.com/Autumn-27) + ## [0.3.4] - 2026-08-24 ### UI diff --git a/agent/blackboard_inheritance_test.go b/agent/blackboard_inheritance_test.go index ad4fe4b..4549ff5 100644 --- a/agent/blackboard_inheritance_test.go +++ b/agent/blackboard_inheritance_test.go @@ -326,7 +326,7 @@ func TestBlackboardToolsReadDirectSources(t *testing.T) { t.Fatalf("inherited host context missing: %#v", coverage) } - facts := callReadJSON(t, tools.listFacts(), `{}`).([]any) + facts := callReadJSON(t, tools.listFacts(), `{}`).(map[string]any)["facts"].([]any) seenSource, seenGrand := false, false for _, raw := range facts { item := raw.(map[string]any) diff --git a/agent/capture_usage_test.go b/agent/capture_usage_test.go index c76f79c..5b6389c 100644 --- a/agent/capture_usage_test.go +++ b/agent/capture_usage_test.go @@ -19,6 +19,17 @@ func (p captureUsageProvider) Stream(ctx context.Context, _ llm.CompletionReques return func(yield func(llm.StreamEvent, error) bool) { p.stream(ctx, yield) } } +func (p captureUsageProvider) Complete(ctx context.Context, req llm.CompletionRequest) (llm.Message, string, llm.Usage, error) { + acc := llm.NewAccumulator() + for ev, err := range p.Stream(ctx, req) { + if err != nil { + return llm.Message{}, "", llm.Usage{}, err + } + acc.Add(ev) + } + return acc.Message(), acc.StopReason, acc.Usage, nil +} + func TestCaptureRunPersistsUsageOnProviderFailure(t *testing.T) { wantErr := errors.New("provider failed after reporting usage") provider := captureUsageProvider{stream: func(_ context.Context, yield func(llm.StreamEvent, error) bool) { diff --git a/agent/chat.go b/agent/chat.go index 41be93b..0f38582 100644 --- a/agent/chat.go +++ b/agent/chat.go @@ -6,13 +6,13 @@ import ( "path/filepath" "time" + "github.com/Autumn-27/artex/db" + "github.com/Autumn-27/artex/guard" "github.com/Autumn-27/norma/agentcore" "github.com/Autumn-27/norma/llm" "github.com/Autumn-27/norma/permission" actool "github.com/Autumn-27/norma/tool" "github.com/Autumn-27/norma/transcript" - "github.com/Autumn-27/artex/db" - "github.com/Autumn-27/artex/guard" ) // ChatAgent is the generic, task-independent conversational runner behind the chat @@ -22,21 +22,28 @@ import ( // LS/Glob/Grep) plus whatever skills/MCP the key is made visible; NO pentest // graph/task context is injected (that stays exclusive to MainAgent). type ChatAgent struct { - prov llm.Provider - model string - workDir string - tx *transcript.Store - window int - proxyAddr string - proxyCACert string - webSearch WebSearchOpts - guard *guard.Guard // optional; nil disables intercept hooks for chat + prov llm.Provider + model string + workDir string + tx *transcript.Store + window int + proxyAddr string + proxyCACert string + webSearch WebSearchOpts + guard *guard.Guard // optional; nil disables intercept hooks for chat + nonStreamingFn func() bool // resolver: use non-streaming (Complete) path? (nil = streaming) } func NewChatAgent(prov llm.Provider, model, workDir string, tx *transcript.Store, window int) *ChatAgent { return &ChatAgent{prov: prov, model: model, workDir: workDir, tx: tx, window: window} } +// SetNonStreaming wires a resolver deciding whether chat runs use the +// non-streaming model path (true = non-streaming). nil/unset = streaming. +func (c *ChatAgent) SetNonStreaming(fn func() bool) { c.nonStreamingFn = fn } + +func (c *ChatAgent) nonStreaming() bool { return c.nonStreamingFn != nil && c.nonStreamingFn() } + // SetProxy points the chat agent's WebFetch/Bash at the recording proxy plus the // CA cert it trusts (empty addr = direct). Kept for parity with the other agents. func (c *ChatAgent) SetProxy(addr, caCert string) { c.proxyAddr, c.proxyCACert = addr, caCert } @@ -115,18 +122,19 @@ func (c *ChatAgent) Chat(ctx context.Context, agentKey, sessionID, message strin BraveSearchAPIKey: ws.BraveKey, TavilySearchAPIKey: ws.TavilyKey, WebSearchProxy: ws.Proxy, - BashEnv: proxyEnv(c.proxyAddr, c.proxyCACert), // Bash 子命令默认走代理+信任 CA - WorkingDir: sessionWorkDir, - MaxTurns: maxTurns, - MaxDuration: maxDuration, - Compaction: compactionConfig(c.window), - Todos: actool.NewTodoStore(), + BashEnv: proxyEnv(c.proxyAddr, c.proxyCACert), // Bash 子命令默认走代理+信任 CA + WorkingDir: sessionWorkDir, + MaxTurns: maxTurns, + MaxDuration: maxDuration, + Compaction: compactionConfig(c.window), + Todos: actool.NewTodoStore(), // large tool output spills to cmd-output/ under the session dir. // 截断上限用 SDK 默认(tool.Capture 的 30000 字符)。 ToolOutputDir: filepath.Join(sessionWorkDir, "cmd-output"), // 命中预算(步数)→ SDK 跑收尾:输出一句总结。Prompt 与收尾轮数按本 agent key 后台可编辑 // (自定义 agent 各自一份;留空/0 用通用默认:10 轮)。 - Settlement: wrapupSettlement(agentKey, nil), + Settlement: wrapupSettlement(agentKey, nil), + NonStreaming: c.nonStreaming(), // 该 profile 选非流式时走 Provider.Complete } if c.guard != nil { opts.Hooks = c.guard.Hooks() diff --git a/agent/goals.go b/agent/goals.go index 388515a..b98e9fe 100644 --- a/agent/goals.go +++ b/agent/goals.go @@ -98,13 +98,13 @@ func DecomposeGoals(ctx context.Context, prov llm.Provider, dataDir, goalText, d if prov == nil { return nil } - return DecomposeGoalsWithProvider(ctx, prov, dataDir, goalText, desc, as, ts, taskID, emit) + return DecomposeGoalsWithProvider(ctx, prov, dataDir, goalText, desc, as, ts, taskID, false, emit) } // DecomposeGoalsWithProvider is the task-runtime variant used when a task has an // ordered provider chain. It preserves the same tools and write behavior while // letting the caller own provider selection/failover. -func DecomposeGoalsWithProvider(ctx context.Context, prov llm.Provider, dataDir, goalText, desc string, as *db.AssetStore, ts *db.ExplorationStore, taskID int64, emit func(db.Activity)) []GoalSpec { +func DecomposeGoalsWithProvider(ctx context.Context, prov llm.Provider, dataDir, goalText, desc string, as *db.AssetStore, ts *db.ExplorationStore, taskID int64, nonStreaming bool, emit func(db.Activity)) []GoalSpec { if prov == nil { return nil } @@ -145,7 +145,8 @@ func DecomposeGoalsWithProvider(ctx context.Context, prov llm.Provider, dataDir, PermissionMode: acperm.ModeBypass, DisableBackgroundTasks: true, // 3 步(抽约束 → 登记范围 → 拆目标)各需一次工具调用,给足回合避免收尾前漏调 set_goals。 - MaxTurns: 8, + MaxTurns: 8, + NonStreaming: nonStreaming, // 该 profile 选非流式时走 Provider.Complete }, userMsg, captureEmit) // set_goals persisted the goals directly; read them back so the caller sees what // was written (empty slice ⇒ the LLM produced nothing ⇒ caller falls back). diff --git a/agent/mainagent.go b/agent/mainagent.go index 99531df..9c40eca 100644 --- a/agent/mainagent.go +++ b/agent/mainagent.go @@ -17,19 +17,26 @@ import ( // (→planner) or direct high-priority intents (→frontier). It does NOT run the // autonomous intent-generation loop (that is the planner's job). type MainAgent struct { - prov llm.Provider - model string - tx *transcript.Store // raw LLM conversation persistence (nil = off) - window int // context window in tokens (for compaction) - windowFn func() int // optional dynamic task-chain minimum - maxTurns int // max agent turns per run (0 = unlimited) - proxyAddr string // recording proxy for WebFetch (empty = direct) - proxyCACert string // recording proxy's CA cert path (HTTPS verify) - webSearch WebSearchOpts // web_search tool backend selection (off by default) - workDir string // shared work dir (surfaced in prompt as artifact-output target) - steerWork func(intentID int64, msg string) error // engine callback: steer a running work (nil = off) + prov llm.Provider + model string + tx *transcript.Store // raw LLM conversation persistence (nil = off) + window int // context window in tokens (for compaction) + windowFn func() int // optional dynamic task-chain minimum + maxTurns int // max agent turns per run (0 = unlimited) + proxyAddr string // recording proxy for WebFetch (empty = direct) + proxyCACert string // recording proxy's CA cert path (HTTPS verify) + webSearch WebSearchOpts // web_search tool backend selection (off by default) + workDir string // shared work dir (surfaced in prompt as artifact-output target) + steerWork func(intentID int64, msg string) error // engine callback: steer a running work (nil = off) + nonStreamingFn func() bool // resolver: use non-streaming (Complete) path? (nil = streaming) } +// SetNonStreaming wires a resolver deciding whether runs use the non-streaming +// model path (true = non-streaming). nil/unset = streaming (default). +func (m *MainAgent) SetNonStreaming(fn func() bool) { m.nonStreamingFn = fn } + +func (m *MainAgent) nonStreaming() bool { return m.nonStreamingFn != nil && m.nonStreamingFn() } + func NewMainAgent(prov llm.Provider, model, workDir string, tx *transcript.Store, window, maxTurns int) *MainAgent { return &MainAgent{prov: prov, model: model, workDir: workDir, tx: tx, window: window, maxTurns: maxTurns} } @@ -128,7 +135,8 @@ func (m *MainAgent) Chat(ctx context.Context, taskID int64, as *db.AssetStore, t Compaction: compactionConfig(m.compactionWindow()), // long chats stay within the window Todos: actool.NewTodoStore(), // 会话级临时待办（TodoWrite），纯规划用，退出即丢 // 命中预算(步数)→ SDK 跑收尾:向用户输出一句进展总结。Prompt 与收尾轮数可后台编辑(默认 10 轮)。 - Settlement: wrapupSettlement("mainagent", nil), + Settlement: wrapupSettlement("mainagent", nil), + NonStreaming: m.nonStreaming(), // 该 profile 选非流式时走 Provider.Complete } if m.tx != nil { // persist raw human↔AI conversation; one accumulating file per task opts.Transcript = m.tx diff --git a/agent/planner.go b/agent/planner.go index dfacc91..2647314 100644 --- a/agent/planner.go +++ b/agent/planner.go @@ -20,19 +20,20 @@ import ( // assets, judges whether the task goal is met, and emits 0..N exploration intents // into the frontier. It is the sole intent generator. type Planner struct { - prov llm.Provider - model string - tx *transcript.Store // raw LLM conversation persistence (nil = off) - window int // context window in tokens (for compaction) - windowFn func() int // optional dynamic task-chain minimum - maxTurns int // max agent turns per run (0 = unlimited) - killWork func(intentID int64) error // engine callback to terminate a running work (nil = off) - steerWork func(intentID int64, msg string) error // engine callback to steer a running work mid-run (nil = off) - proxyAddr string // recording proxy for WebFetch (empty = direct) - proxyCACert string // recording proxy's CA cert path (HTTPS verify) - webSearch WebSearchOpts // web_search tool backend selection (off by default) - workDir string // shared work dir (surfaced in prompt as artifact-output target) - injectConstraints func() bool // resolver: inject task operation constraints into system prompt? (nil = yes) + prov llm.Provider + model string + tx *transcript.Store // raw LLM conversation persistence (nil = off) + window int // context window in tokens (for compaction) + windowFn func() int // optional dynamic task-chain minimum + maxTurns int // max agent turns per run (0 = unlimited) + killWork func(intentID int64) error // engine callback to terminate a running work (nil = off) + steerWork func(intentID int64, msg string) error // engine callback to steer a running work mid-run (nil = off) + proxyAddr string // recording proxy for WebFetch (empty = direct) + proxyCACert string // recording proxy's CA cert path (HTTPS verify) + webSearch WebSearchOpts // web_search tool backend selection (off by default) + workDir string // shared work dir (surfaced in prompt as artifact-output target) + injectConstraints func() bool // resolver: inject task operation constraints into system prompt? (nil = yes) + nonStreamingFn func() bool // resolver: use non-streaming (Complete) path? (nil = streaming) // todos keeps ONE plan-scratchpad per task (keyed by exploration id) so the // planner's multi-step plan survives across wake-ups — each Plan() is a fresh @@ -48,6 +49,12 @@ func NewPlanner(prov llm.Provider, model, workDir string, tx *transcript.Store, func (p *Planner) SetCompactionWindowResolver(fn func() int) { p.windowFn = fn } +// SetNonStreaming wires a resolver deciding whether runs use the non-streaming +// model path (true = non-streaming). nil/unset = streaming (default). +func (p *Planner) SetNonStreaming(fn func() bool) { p.nonStreamingFn = fn } + +func (p *Planner) nonStreaming() bool { return p.nonStreamingFn != nil && p.nonStreamingFn() } + func (p *Planner) compactionWindow() int { if p.windowFn != nil { return p.windowFn() @@ -238,7 +245,7 @@ const plannerDefaultTmpl = `你是一个授权渗透测试系统的"规划者" ⚠️ 最重要的原则：**这一轮生成 0 个意图是完全正常、而且是最常见的结果。** 你不是"每次都要产出意图"的机器。绝大多数唤醒，frontier 里已有的意图已经覆盖了所有已知方向，你应当什么都不加、直接结束。重复/换措辞地生成已经存在的意图是严重错误。 决策流程（每次唤醒）： -1. **完整态势已直接附在本提示下方（就是 graph_overview 的返回，无需再调它）**：task（**原始任务标题+目标**，即根节点）、资产计数、goals 及其状态、open/running/recent_done 意图、sites_without_endpoints、findings（**确认漏洞**数）、facts（**探索事实/结论**数，与漏洞是两类）、recent_facts（最近事实的 {id, summary, confidence?}，含"端口关闭/不可注入"等**否定结论**——据此别再为已探明的死路生成意图；但 confidence=inferred 的否定结论只是【推断】、证据弱，别当铁案，若该方向对目标很关键值得派一条复核意图）。**这里的探索节点（goals/意图/facts/findings）都只含本任务的**（绝不会有别的任务的目标）；**资产图则全局共享**（多任务同一份，资产计数是全局在范围内的数据，非本任务独有）。需要更深的细节时才按需调：list_facts 列全部事实、list_findings 列全部漏洞、**node_detail(id)** 取某条的完整证据/详情（列表/recent_facts 只给摘要）。 +1. **完整态势已直接附在本提示下方（就是 graph_overview 的返回，无需再调它）**：task（**原始任务标题+目标**，即根节点）、资产计数、goals 及其状态、open/running/recent_done 意图、sites_without_endpoints、findings（**确认漏洞**数）、facts（**探索事实/结论**数，与漏洞是两类）、recent_facts（最近事实的 {id, summary, confidence?}，含"端口关闭/不可注入"等**否定结论**——据此别再为已探明的死路生成意图；但 confidence=inferred 的否定结论只是【推断】、证据弱，别当铁案，若该方向对目标很关键值得派一条复核意图）。**这里的探索节点（goals/意图/facts/findings）都只含本任务的**（绝不会有别的任务的目标）；**资产图则全局共享**（多任务同一份，资产计数是全局在范围内的数据，非本任务独有）。需要更深的细节时才按需调：list_facts 分页列事实（最新在前，默认 20 条，可传 q 关键词过滤、before 翻页，返回带 total/has_more）、list_findings 列全部漏洞、**node_detail(id)** 取某条的完整证据/详情（列表/recent_facts 只给摘要）。 - open_intents / running_intents / recent_done_intents——"哪些方向已经有意图在覆盖/已尝试"。每个意图还带 **parents（上游：它派生自哪些事实/意图）和 yields（下游：它产生了哪些事实/发现）**——这就是探索图的**血缘关系**，据此理解"哪些事实来自哪个方向、能否综合成新方向"。recent_facts 里每个事实带 **from_intent**（由哪个意图产生）。 - sites_without_endpoints / findings——"哪些方向【可能】需要探索"。 - 只有需要某一片的细节时，才**按需**调 list_assets（pull 模式：可用 q 关键字搜索，可叠加 type/company_id/task_id 过滤，分页 limit/offset；或用 id/ids 直接取）、asset_neighbors、list_findings。资产图全局共享，别默认拉全量。资产中可能包含非本次任务涉及到的资产，所以需要主要出现非本次任务相关的资产时忽略这些资产。 @@ -353,7 +360,8 @@ func (p *Planner) Plan(ctx context.Context, taskID int64, as *db.AssetStore, ts // 命中【本轮】步数预算→ SDK 跑收尾:把本轮已想清楚的结论落地(该派的 add_intent、 // 能证的 prove_goal、串行链记 TodoWrite),而非停止规划——planner 之后仍会被反复唤醒。 // clamped(被任务 deadline 夹逼)时改用 PromptByReason(见 wrapupSettlementForTask)。 - Settlement: settle, + Settlement: settle, + NonStreaming: p.nonStreaming(), // 该 profile 选非流式时走 Provider.Complete } if p.tx != nil { // persist raw LLM conversation; one accumulating file per task's planner opts.Transcript = p.tx diff --git a/agent/provider.go b/agent/provider.go index 6011ada..c66a1b8 100644 --- a/agent/provider.go +++ b/agent/provider.go @@ -51,6 +51,11 @@ type Config struct { // "" = 不发送(默认); "low"/"medium"/"high"/"xhigh"/"max" = 对应强度. // OpenAI 映射为顶层 reasoning_effort;Anthropic 映射为 output_config.effort. ReasoningEffort string + // Stream 控制该 profile 是否使用流式(SSE)接口。true(默认)= 流式;false = 真· + // 非流式(发 stream:false,一次性拿完整 JSON,走 Provider.Complete)。非流式可绕开 + // 某些网关糟糕的 SSE 实现(空帧、思考字段丢帧),代价是失去运行中的实时进度/实时 + // token 计数。映射为 agentcore.Options.NonStreaming = !Stream。 + Stream bool } // compaction window resolution bounds (in K tokens). Below the floor the @@ -116,6 +121,8 @@ func FromEnv() (Config, bool) { BaseURL: os.Getenv("ARTEX_LLM_BASE_URL"), Model: os.Getenv("ARTEX_LLM_MODEL"), Proxy: strings.TrimSpace(os.Getenv("ARTEX_LLM_PROXY")), + // 默认流式;ARTEX_LLM_STREAM=false/0/off 显式关闭走非流式。 + Stream: !isFalsy(os.Getenv("ARTEX_LLM_STREAM")), } switch prov { case "openai": @@ -147,6 +154,7 @@ func ConfigFrom(provider, model, baseURL, apiKey, proxy string) Config { BaseURL: strings.TrimRight(strings.TrimSpace(baseURL), "/"), APIKey: strings.TrimSpace(apiKey), Proxy: strings.TrimSpace(proxy), + Stream: true, // 默认流式;调用方按 profile 覆盖 } switch strings.TrimSpace(provider) { case "openai": @@ -167,6 +175,16 @@ func ConfigFrom(provider, model, baseURL, apiKey, proxy string) Config { return c } +// isFalsy reports whether an env-var string explicitly requests "off". Empty or +// unrecognized → false (so an unset var keeps the streaming default). +func isFalsy(s string) bool { + switch strings.ToLower(strings.TrimSpace(s)) { + case "0", "false", "off", "no": + return true + } + return false +} + // Provider returns the short provider name ("anthropic"/"openai"). func (c Config) Provider() string { if c.Format == llm.FormatOpenAI { @@ -288,11 +306,12 @@ func quotaAwareHTTPClient(proxy string) (*http.Client, error) { } // TestConnection makes a minimal real completion to verify the provider/model/ -// endpoint/key actually work. Returns the round-trip latency. -func TestConnection(ctx context.Context, c Config) (time.Duration, error) { +// endpoint/key actually work. Returns the round-trip latency and the model's +// reply text. +func TestConnection(ctx context.Context, c Config) (time.Duration, string, error) { prov, err := c.NewProvider() if err != nil { - return 0, err + return 0, "", err } ctx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() @@ -302,12 +321,24 @@ func TestConnection(ctx context.Context, c Config) (time.Duration, error) { // 就撞到输出上限(finish=length)、被截断,连接测试虽仍算通(err=nil)但显示成 // "已中断/length/resume" 一团糟。给足预算让它把 OK 干净吐完(finish=stop)。 // EscalateMaxTokens 保持 false:不因截断而抬额重试,避免 resume 循环空烧。 - _, err = agentcore.Run(ctx, agentcore.Options{ + reply, err := agentcore.Run(ctx, agentcore.Options{ Provider: prov, SystemPrompt: []string{"你是连接测试。直接输出两个字符 OK 即可，不要思考、不要解释、不要别的。"}, PermissionMode: acperm.ModeBypass, MaxTurns: 1, MaxTokens: 8192, + NonStreaming: !c.Stream, // 用该 profile 的真实收发模式做连接测试 }, "ping") - return time.Since(start), err + lat := time.Since(start) + if err != nil { + return lat, "", err + } + // err==nil 还不够：请求通了但模型一个字都不吐的情况真实存在(思考把预算烧光、 + // 正文被安全策略吞掉、兼容层把 content 丢了)。这种配置在会话里就是"不回话", + // 测试却报成功——正是本项要消除的落差。没有可见正文一律判失败。 + reply = strings.TrimSpace(reply) + if reply == "" { + return lat, "", fmt.Errorf("模型无回复内容（请求已通，但未返回任何文本）") + } + return lat, reply, nil } diff --git a/agent/tools.go b/agent/tools.go index f8493ab..ebed6d8 100644 --- a/agent/tools.go +++ b/agent/tools.go @@ -898,19 +898,64 @@ func (t *ToolSet) listFindings() actool.CoreTool { }) } +// factsPageSize is the default page size for list_facts. Facts pile up on long +// tasks; returning all of them at once (the old behaviour) could blow up the +// context, so default to the newest page and let the agent page/filter for more. +const factsPageSize = 20 + func (t *ToolSet) listFacts() actool.CoreTool { - return readTool("list_facts", "列本任务及直接关联任务的【探索事实/结论】(紧凑：id+摘要+状态)。关联任务条目带 source_task_id/inherited=true 且只读。详情用 node_detail(id)，漏洞看 list_findings。", - obj(map[string]any{}), - func(context.Context, json.RawMessage) (actool.Result, error) { - f, _ := t.ts.ListByKindWithSources(db.KindFact, 500) + return readTool("list_facts", "分页列本任务及直接关联任务的【探索事实/结论】，最新在前(紧凑：id+摘要+状态，摘要过长会截断，全文用 node_detail(id))。参数均可选：limit(默认 20，上限 100)、before(游标，传上一页返回的 next_before 取更旧的一页；省略/0=最新一页)、q(按摘要关键词过滤)。返回 {facts, total, has_more, next_before}：total 是过滤后的总数，has_more=true 时用 next_before 继续翻页。关联任务条目带 source_task_id/inherited=true 且只读。漏洞看 list_findings。", + obj(map[string]any{ + "limit": intp("返回条数，默认 20，上限 100"), + "before": intp("分页游标：只返回 id 小于该值的更旧事实；省略或 0 = 最新一页"), + "q": str("按事实摘要关键词过滤（不区分大小写）；省略 = 不过滤"), + }), + func(_ context.Context, in json.RawMessage) (actool.Result, error) { + var a struct { + Limit int `json:"limit"` + Before int64 `json:"before"` + Q string `json:"q"` + } + _ = json.Unmarshal(in, &a) + limit := a.Limit + if limit <= 0 { + limit = factsPageSize + } + if limit > 100 { + limit = 100 + } + f, hasMore, total, err := t.ts.ListByKindPageWithSources(db.KindFact, a.Before, limit, strings.TrimSpace(a.Q)) + if err != nil { + return actool.Result{}, err + } out := make([]map[string]any, 0, len(f)) for _, n := range f { - out = append(out, compactNode(n)) + out = append(out, compactFact(n)) } - return jsonResult(out) + res := map[string]any{"facts": out, "total": total, "has_more": hasMore} + if hasMore && len(f) > 0 { + res["next_before"] = f[len(f)-1].ID // 传回它取下一页(更旧的) + } + return jsonResult(res) }) } +// factSummaryMax caps a fact summary in list_facts output. Facts carry one-line +// conclusions, but nothing enforces brevity; a runaway summary must not bloat a +// whole page. Full text stays available via node_detail(id). +const factSummaryMax = 160 + +// compactFact is compactNode with the summary rune-capped for list_facts, so a +// page of facts stays bounded regardless of how long any single summary grew. +func compactFact(n *db.Node) map[string]any { + m := compactNode(n) + if s, ok := m["summary"].(string); ok && len([]rune(s)) > factSummaryMax { + m["summary"] = string([]rune(s)[:factSummaryMax]) + "…" + m["summary_truncated"] = true + } + return m +} + func (t *ToolSet) nodeDetail() actool.CoreTool { return readTool("node_detail", "按 id 取本任务或直接关联任务的【探索图节点】完整内容。继承节点带 source_task_id/inherited=true 且只读。仅限 list_facts/list_findings/graph_overview 返回的探索节点 id；资产请用 list_assets/asset_neighbors。", obj(map[string]any{"id": idp("探索图节点 id(非资产 id)")}, "id"), diff --git a/agent/worker.go b/agent/worker.go index f2e5ce8..b7a9e7a 100644 --- a/agent/worker.go +++ b/agent/worker.go @@ -62,8 +62,19 @@ type Worker struct { // injected into the worker system prompt. Read per run so the settings toggle // takes effect without rebuilding the agent. nil = inject (default). injectConstraints func() bool + // nonStreamingFn resolves whether this run uses the non-streaming (Complete) + // path. Read per run so a profile/task toggle takes effect without rebuilding + // the agent. nil = streaming (default). + nonStreamingFn func() bool } +// SetNonStreaming wires a resolver deciding whether runs use the non-streaming +// model path (true = non-streaming). nil/unset = streaming (default). Read per +// run so a profile or task-chain toggle takes effect without rebuilding. +func (w *Worker) SetNonStreaming(fn func() bool) { w.nonStreamingFn = fn } + +func (w *Worker) nonStreaming() bool { return w.nonStreamingFn != nil && w.nonStreamingFn() } + // SetConstraintInject wires a resolver deciding whether this task's operation // constraints get injected into the worker system prompt. nil = inject (default). func (w *Worker) SetConstraintInject(fn func() bool) { w.injectConstraints = fn } @@ -160,7 +171,7 @@ const workerDefaultTmpl = `你是一个授权渗透测试系统的"执行者"(wo - insert_assets：登记新资产（资产图）。**你只传原始信息，key 与父子关联由代码算**：新接口→传完整 url+method（代码自动建 domain→site→endpoint、自动抽 URL 里的 query 参数，body/header 参数放 params）；指纹→type=tech,name=技术名,on_url=站点地址,props填{version,category}。多个资产放 assets 数组一次批量登记。属性写在资产自己的 props 上，探索结论不要写这里。 - record_fact：把探索【事实/结论】写入探索图并连到意图（传 intent_id）。正向/否定结论、观察、判断用它；**一次探索的多个观察汇总成一条事实**（summary 总结一句话 + detail 放细节），不要一个属性一条。真有多条不同结论才用 facts 数组。**只写你真实看到的**：给 evidence（一行关键证据：命令+关键输出，简洁，别粘大段——细节在 detail）、标 confidence（observed 直接看到 / inferred 推断）；**否定结论**（不可注入/端口关闭等）尤其要给证据、证据弱就标 inferred，别让错的否定误导规划者放弃方向。 - report_finding：确认漏洞 → 记录(含 PoC，传 intent_id=你领到的意图id)。**只有你在本次运行里真实触发过该漏洞、拿到了可复现的证据（请求/响应或命令输出）才用它。** 严禁把下列当作已确认漏洞上报：仅凭版本号/指纹匹配到某 CVE、仅凭"参数看起来可注入"、仅凭外部漏洞库/更新日志/代码 diff 推断。**不要用查 CVE 库或"对比补丁版本"替代实际触发。** 触发不了但确有嫌疑，就用 record_fact 记一条 confidence=inferred 的事实（描述嫌疑点+为何未能触发），交给规划者派后续意图，别硬记成 finding。 -- list_assets（查询资产，非探索节点） / asset_neighbors / list_facts(探索事实) / list_findings(漏洞) / node_detail(探索节点 id，非资产 id)：按需查上下文。 +- list_assets（查询资产，非探索节点） / asset_neighbors / list_facts(探索事实，分页最新在前；默认 20 条，可传 q 关键词过滤、before 翻页) / list_findings(漏洞) / node_detail(探索节点 id，非资产 id)：按需查上下文。 只在授权范围内操作；若系统提示顶部附有【操作约束】，那是最高优先级红线——任何命令/探测在执行前先自检是否违反，违反即不做（哪怕它落在你领到的意图里）。完成本意图后用一句话总结你做了什么、写回了哪些事实。务实、克制、聚焦这一条意图。` @@ -337,6 +348,7 @@ func (w *Worker) Execute(ctx context.Context, name string, taskID int64, as *db. ToolOutputDir: cmdOutDir(runDir), Compaction: compactionConfig(w.compactionWindow()), // long tool-heavy runs stay within the window Todos: actool.NewTodoStore(), // 会话级临时待办（TodoWrite），纯规划用，退出即丢 + NonStreaming: w.nonStreaming(), // 该 profile 选非流式时走 Provider.Complete } if hooks != nil { // typed-nil guard: only set when concrete (avoids harness panic) opts.Hooks = hooks diff --git a/db/commands.go b/db/commands.go index 27172da..22fe14e 100644 --- a/db/commands.go +++ b/db/commands.go @@ -19,18 +19,10 @@ type CommandRecord struct { CreatedAt time.Time `json:"created_at"` } -// ListCommands returns tool executions (tool_use + paired tool_result) across all -// explorations, with optional filtering and pagination. Covers every tool, not -// just Bash; q matches the tool name or its input. -func (d *DB) ListCommands(expID *int64, q string, page, size int) ([]CommandRecord, int, error) { - if size <= 0 { - size = 50 - } - if page < 0 { - page = 0 - } - offset := page * size - +// commandFilter builds the WHERE clause shared by the tool-execution list and +// its per-tool tally, so the summary always describes exactly the rows the table +// pages through. Returns the clause, its args, and the next placeholder index. +func commandFilter(expID *int64, q string) (string, []any, int) { where := `WHERE u.kind = 'tool_use'` args := []any{} argN := 1 @@ -45,6 +37,59 @@ func (d *DB) ListCommands(expID *int64, q string, page, size int) ([]CommandReco args = append(args, "%"+q+"%") argN++ } + return where, args, argN +} + +// ToolStat is one tool's execution tally for the usage summary. +type ToolStat struct { + Tool string `json:"tool"` + Total int `json:"total"` + Errors int `json:"errors"` +} + +// ToolStats counts executions grouped by tool under the same filters +// ListCommands takes. Unpaginated on purpose: the tally describes the whole +// filtered set, not the page currently on screen. +func (d *DB) ToolStats(expID *int64, q string) ([]ToolStat, error) { + where, args, _ := commandFilter(expID, q) + + rows, err := d.Query(` +SELECT COALESCE(NULLIF(u.tool,''),'-') AS tool, COUNT(*) AS total, + COUNT(*) FILTER (WHERE COALESCE(r.is_error,false)) AS errors +FROM activity u +LEFT JOIN activity r ON r.tool_use_id = u.tool_use_id AND r.kind = 'tool_result' +`+where+` +GROUP BY 1 +ORDER BY total DESC, tool ASC`, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + out := []ToolStat{} + for rows.Next() { + var s ToolStat + if err := rows.Scan(&s.Tool, &s.Total, &s.Errors); err != nil { + return nil, err + } + out = append(out, s) + } + return out, rows.Err() +} + +// ListCommands returns tool executions (tool_use + paired tool_result) across all +// explorations, with optional filtering and pagination. Covers every tool, not +// just Bash; q matches the tool name or its input. +func (d *DB) ListCommands(expID *int64, q string, page, size int) ([]CommandRecord, int, error) { + if size <= 0 { + size = 50 + } + if page < 0 { + page = 0 + } + offset := page * size + + where, args, argN := commandFilter(expID, q) // count var total int diff --git a/db/config.go b/db/config.go index 40684d6..410994a 100644 --- a/db/config.go +++ b/db/config.go @@ -39,18 +39,23 @@ type LLMProfile struct { // usable when an agent/task binds it explicitly, it just never gets picked up // as a fallback target. PoolExclude bool `json:"pool_exclude"` + // Streaming selects the wire protocol: true (default) = streaming (SSE); + // false = real non-streaming (stream:false, single JSON response via + // Provider.Complete). Non-streaming sidesteps flaky gateway SSE at the cost of + // live in-run progress. Maps to agent.Config.Stream. + Streaming bool `json:"streaming"` } // profileCols is the read column list (hint variant, no api key) shared by the // list query; profileColsKey is the same with api_key for the single-row loads. -const profileCols = `id,name,format,COALESCE(base_url,''),COALESCE(proxy,''),model,COALESCE(api_key_hint,''),rate_per_second,rate_per_minute,context_window_k,COALESCE(reasoning_effort,''),is_default,priority,pool_exclude,COALESCE(thinking_type,'')` -const profileColsKey = `id,name,format,COALESCE(base_url,''),COALESCE(proxy,''),model,COALESCE(api_key,''),rate_per_second,rate_per_minute,context_window_k,COALESCE(reasoning_effort,''),is_default,priority,pool_exclude,COALESCE(thinking_type,'')` +const profileCols = `id,name,format,COALESCE(base_url,''),COALESCE(proxy,''),model,COALESCE(api_key_hint,''),rate_per_second,rate_per_minute,context_window_k,COALESCE(reasoning_effort,''),is_default,priority,pool_exclude,COALESCE(thinking_type,''),COALESCE(streaming,true)` +const profileColsKey = `id,name,format,COALESCE(base_url,''),COALESCE(proxy,''),model,COALESCE(api_key,''),rate_per_second,rate_per_minute,context_window_k,COALESCE(reasoning_effort,''),is_default,priority,pool_exclude,COALESCE(thinking_type,''),COALESCE(streaming,true)` // scanProfile reads one row in the profileCols / profileColsKey column order. The // 7th column lands in APIKeyHint or APIKey depending on which list the caller used. func scanProfile(sc interface{ Scan(...any) error }, into *string, p *LLMProfile) error { return sc.Scan(&p.ID, &p.Name, &p.Format, &p.BaseURL, &p.Proxy, &p.Model, into, - &p.RatePerSecond, &p.RatePerMinute, &p.ContextWindowK, &p.ReasoningEffort, &p.IsDefault, &p.Priority, &p.PoolExclude, &p.ThinkingType) + &p.RatePerSecond, &p.RatePerMinute, &p.ContextWindowK, &p.ReasoningEffort, &p.IsDefault, &p.Priority, &p.PoolExclude, &p.ThinkingType, &p.Streaming) } func (d *DB) ListProfiles() ([]*LLMProfile, error) { @@ -123,18 +128,18 @@ func (d *DB) SaveProfile(p *LLMProfile) (int64, error) { } if p.ID == 0 { var id int64 - err := d.QueryRow(`INSERT INTO llm_profiles(name,format,base_url,proxy,model,api_key,api_key_hint,rate_per_second,rate_per_minute,context_window_k,reasoning_effort,priority,pool_exclude,thinking_type) -VALUES ($1,$2,NULLIF($3,''),NULLIF($4,''),$5,NULLIF($6,''),NULLIF($7,''),$8,$9,$10,$11,$12,$13,$14) RETURNING id`, - p.Name, p.Format, p.BaseURL, p.Proxy, p.Model, p.APIKey, hint, p.RatePerSecond, p.RatePerMinute, p.ContextWindowK, p.ReasoningEffort, p.Priority, p.PoolExclude, p.ThinkingType).Scan(&id) + err := d.QueryRow(`INSERT INTO llm_profiles(name,format,base_url,proxy,model,api_key,api_key_hint,rate_per_second,rate_per_minute,context_window_k,reasoning_effort,priority,pool_exclude,thinking_type,streaming) +VALUES ($1,$2,NULLIF($3,''),NULLIF($4,''),$5,NULLIF($6,''),NULLIF($7,''),$8,$9,$10,$11,$12,$13,$14,$15) RETURNING id`, + p.Name, p.Format, p.BaseURL, p.Proxy, p.Model, p.APIKey, hint, p.RatePerSecond, p.RatePerMinute, p.ContextWindowK, p.ReasoningEffort, p.Priority, p.PoolExclude, p.ThinkingType, p.Streaming).Scan(&id) return id, err } if p.APIKey == "" { - _, err := d.Exec(`UPDATE llm_profiles SET name=$1,format=$2,base_url=NULLIF($3,''),proxy=NULLIF($4,''),model=$5,rate_per_second=$6,rate_per_minute=$7,context_window_k=$8,reasoning_effort=$9,priority=$10,pool_exclude=$11,thinking_type=$12 WHERE id=$13`, - p.Name, p.Format, p.BaseURL, p.Proxy, p.Model, p.RatePerSecond, p.RatePerMinute, p.ContextWindowK, p.ReasoningEffort, p.Priority, p.PoolExclude, p.ThinkingType, p.ID) + _, err := d.Exec(`UPDATE llm_profiles SET name=$1,format=$2,base_url=NULLIF($3,''),proxy=NULLIF($4,''),model=$5,rate_per_second=$6,rate_per_minute=$7,context_window_k=$8,reasoning_effort=$9,priority=$10,pool_exclude=$11,thinking_type=$12,streaming=$13 WHERE id=$14`, + p.Name, p.Format, p.BaseURL, p.Proxy, p.Model, p.RatePerSecond, p.RatePerMinute, p.ContextWindowK, p.ReasoningEffort, p.Priority, p.PoolExclude, p.ThinkingType, p.Streaming, p.ID) return p.ID, err } - _, err := d.Exec(`UPDATE llm_profiles SET name=$1,format=$2,base_url=NULLIF($3,''),proxy=NULLIF($4,''),model=$5,api_key=$6,api_key_hint=$7,rate_per_second=$8,rate_per_minute=$9,context_window_k=$10,reasoning_effort=$11,priority=$12,pool_exclude=$13,thinking_type=$14 WHERE id=$15`, - p.Name, p.Format, p.BaseURL, p.Proxy, p.Model, p.APIKey, hint, p.RatePerSecond, p.RatePerMinute, p.ContextWindowK, p.ReasoningEffort, p.Priority, p.PoolExclude, p.ThinkingType, p.ID) + _, err := d.Exec(`UPDATE llm_profiles SET name=$1,format=$2,base_url=NULLIF($3,''),proxy=NULLIF($4,''),model=$5,api_key=$6,api_key_hint=$7,rate_per_second=$8,rate_per_minute=$9,context_window_k=$10,reasoning_effort=$11,priority=$12,pool_exclude=$13,thinking_type=$14,streaming=$15 WHERE id=$16`, + p.Name, p.Format, p.BaseURL, p.Proxy, p.Model, p.APIKey, hint, p.RatePerSecond, p.RatePerMinute, p.ContextWindowK, p.ReasoningEffort, p.Priority, p.PoolExclude, p.ThinkingType, p.Streaming, p.ID) return p.ID, err } diff --git a/db/exploration.go b/db/exploration.go index f62e280..eb0e677 100644 --- a/db/exploration.go +++ b/db/exploration.go @@ -672,6 +672,40 @@ ORDER BY id DESC LIMIT $4`, s.expID, kind, before, limit+1) return nodes, hasMore, nil } +// listByKindPageFiltered is ListByKindPage with an optional summary keyword +// filter (payload->>'summary' ILIKE %q%). Fetches one extra row so the caller can +// probe hasMore. Empty q = no filter. Newest-first by id. +func (s *ExplorationStore) listByKindPageFiltered(kind string, before int64, limit int, q string) ([]*Node, error) { + where := `exploration_id=$1 AND kind=$2 AND ($3 <= 0 OR id < $3)` + args := []any{s.expID, kind, before} + if q != "" { + args = append(args, "%"+q+"%") + where += fmt.Sprintf(` AND payload->>'summary' ILIKE $%d`, len(args)) + } + args = append(args, limit+1) + rows, err := s.db.Query(`SELECT `+nodeCols+` FROM exploration_nodes +WHERE `+where+` ORDER BY id DESC LIMIT $`+fmt.Sprintf("%d", len(args)), args...) + if err != nil { + return nil, err + } + defer rows.Close() + return scanNodes(rows) +} + +// countByKindFiltered counts nodes of a kind under the same summary filter, for a +// page's total. Empty q = count all of that kind. +func (s *ExplorationStore) countByKindFiltered(kind, q string) (int, error) { + where := `exploration_id=$1 AND kind=$2` + args := []any{s.expID, kind} + if q != "" { + args = append(args, "%"+q+"%") + where += fmt.Sprintf(` AND payload->>'summary' ILIKE $%d`, len(args)) + } + var n int + err := s.db.QueryRow(`SELECT COUNT(*) FROM exploration_nodes WHERE `+where, args...).Scan(&n) + return n, err +} + // GetNode returns one node of this exploration by id (nil, nil if not found). func (s *ExplorationStore) GetNode(id int64) (*Node, error) { n, err := scanNode(s.db.QueryRow(`SELECT `+nodeCols+` FROM exploration_nodes WHERE id=$1 AND exploration_id=$2`, id, s.expID)) diff --git a/db/exploration_sources.go b/db/exploration_sources.go index 3ccbb96..c12d5ba 100644 --- a/db/exploration_sources.go +++ b/db/exploration_sources.go @@ -108,6 +108,53 @@ func (s *ExplorationStore) ListByKindWithSources(kind string, limit int) ([]*Nod return out, nil } +// ListByKindPageWithSources is the paginated, keyword-filterable sibling of +// ListByKindWithSources: it returns one newest-first page (id < before, before<=0 +// = newest) spanning this exploration and its direct sources, plus hasMore and the +// filtered total across all of them. Node ids are globally unique, so merging each +// store's own page and re-sorting by id DESC yields the true global page; fetching +// limit+1 per store guarantees the merged top-`limit` is complete. +func (s *ExplorationStore) ListByKindPageWithSources(kind string, before int64, limit int, q string) (nodes []*Node, hasMore bool, total int, err error) { + if limit <= 0 { + limit = 20 + } + own, err := s.listByKindPageFiltered(kind, before, limit, q) + if err != nil { + return nil, false, 0, err + } + merged := own + total, err = s.countByKindFiltered(kind, q) + if err != nil { + return nil, false, 0, err + } + + sources, err := s.DirectSourceStores() + if err != nil { + return nil, false, 0, err + } + for _, source := range sources { + page, err := source.Store.listByKindPageFiltered(kind, before, limit, q) + if err != nil { + return nil, false, 0, err + } + for _, n := range page { + merged = append(merged, markInheritedNode(n, source.Task.TaskID)) + } + cnt, err := source.Store.countByKindFiltered(kind, q) + if err != nil { + return nil, false, 0, err + } + total += cnt + } + + sort.Slice(merged, func(i, j int) bool { return merged[i].ID > merged[j].ID }) + hasMore = len(merged) > limit + if hasMore { + merged = merged[:limit] + } + return merged, hasMore, total, nil +} + // GetNodeWithSources reads a node only when it belongs to this exploration or // one of its direct sources. Inherited nodes are tagged so tool callers can keep // them read-only and show their provenance. diff --git a/db/schema.sql b/db/schema.sql index de0b7cd..9be7064 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -310,6 +310,8 @@ CREATE TABLE IF NOT EXISTS llm_profiles ( -- pool_exclude true=不作为故障转移目标(仍可被 agent/任务显式绑定使用)。 priority INTEGER NOT NULL DEFAULT 0, pool_exclude BOOLEAN NOT NULL DEFAULT false, + -- streaming=true(默认)走流式 SSE；false 走真·非流式(stream:false，一次性 JSON)。 + streaming BOOLEAN NOT NULL DEFAULT true, created_at TIMESTAMPTZ NOT NULL DEFAULT now(), updated_at TIMESTAMPTZ NOT NULL DEFAULT now() ); @@ -320,6 +322,8 @@ CREATE TRIGGER trg_llm_upd BEFORE UPDATE ON llm_profiles -- 轮询顺位/排除标记；补旧库。默认 0 / false = 全部配置都参与轮询。 ALTER TABLE llm_profiles ADD COLUMN IF NOT EXISTS priority INTEGER NOT NULL DEFAULT 0; ALTER TABLE llm_profiles ADD COLUMN IF NOT EXISTS pool_exclude BOOLEAN NOT NULL DEFAULT false; +-- 流式开关；补旧库。默认 true = 保持既有的流式行为，旧配置无感升级。 +ALTER TABLE llm_profiles ADD COLUMN IF NOT EXISTS streaming BOOLEAN NOT NULL DEFAULT true; -- 思考开关字段 thinking_type，从旧的单一 reasoning_effort 语义一次性拆分而来。 -- schema.sql 每次启动都执行，故迁移必须只跑一次：仅当该列尚不存在时才回填， diff --git a/go.mod b/go.mod index d56340a..b047c4b 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,10 @@ module github.com/Autumn-27/artex go 1.26.3 require ( - github.com/Autumn-27/norma v0.2.9 + github.com/Autumn-27/norma v0.3.2 github.com/golang-jwt/jwt/v5 v5.3.1 github.com/jackc/pgx/v5 v5.10.0 + github.com/klauspost/compress v1.18.6 github.com/lqqyt2423/go-mitmproxy v1.9.2 github.com/miekg/dns v1.1.72 github.com/projectdiscovery/dnsx v1.2.3 @@ -29,7 +30,6 @@ require ( github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect - github.com/klauspost/compress v1.18.6 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/microcosm-cc/bluemonday v1.0.27 // indirect diff --git a/go.sum b/go.sum index 835017c..845ee26 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -github.com/Autumn-27/norma v0.2.9 h1:VTt8Oe7XoiV85Dc+QOPu+zQ7IykQRm2aDTEbLEzbBe4= -github.com/Autumn-27/norma v0.2.9/go.mod h1:BudWrf7iHl96FgHqFr/HwN3mKYmGjJDt+xL2ZHXfFZw= +github.com/Autumn-27/norma v0.3.2 h1:3gW84SbSrz8luW8kbi2tv3O7CLdRMQvyylCHj1QUdNk= +github.com/Autumn-27/norma v0.3.2/go.mod h1:BudWrf7iHl96FgHqFr/HwN3mKYmGjJDt+xL2ZHXfFZw= github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro= github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= diff --git a/llmpool/pool.go b/llmpool/pool.go index 30b3665..be0ddff 100644 --- a/llmpool/pool.go +++ b/llmpool/pool.go @@ -122,6 +122,44 @@ func (p *Pool) Stream(ctx context.Context, req llm.CompletionRequest) iter.Seq2[ } } +// Complete implements llm.Provider with failover for non-streaming calls. A +// non-streaming request is atomic — it never delivers partial output — so every +// failover-eligible failure lands in the safe window and the next member can be +// tried without risk of duplicated output. Mirrors Stream's health-tripping and +// chain-exhaustion behavior. +func (p *Pool) Complete(ctx context.Context, req llm.CompletionRequest) (llm.Message, string, llm.Usage, error) { + order := p.order(req) + var lastErr error + for i, m := range order { + msg, sr, usage, err := m.Prov.Complete(ctx, req) + if err == nil { + p.health.Pass(m.ID) + return msg, sr, usage, nil + } + if !shouldFailover(ctx, err) { + // Non-failover error (e.g. ctx cancel, deterministic 4xx): surface as-is + // without tripping health, matching Stream's non-failover path. + p.health.Pass(m.ID) + return llm.Message{}, "", llm.Usage{}, err + } + lastErr = err + hard := isHardFailure(err) + if p.health.Trip(m.ID, trimErr(err), hard) { + log.Printf("[llmpool] 配置 %q(%s) 已熔断：%s", m.Name, m.Model, trimErr(err)) + } + if i+1 < len(order) { + n := order[i+1] + log.Printf("[llmpool] LLM 故障转移：%q(%s) → %q(%s)，原因：%s", + m.Name, m.Model, n.Name, n.Model, trimErr(err)) + } + } + if lastErr == nil { + lastErr = ErrExhausted + } + log.Printf("[llmpool] 轮询链已耗尽(%d 个配置全部失败)，最后错误：%s", len(order), trimErr(lastErr)) + return llm.Message{}, "", llm.Usage{}, fmt.Errorf("%w：%v", ErrExhausted, lastErr) +} + // order picks the members to try, in order: skip those in a cooling-off window // and those whose context window can't hold this request, then rotate within each // equal-priority group so same-priority profiles share the load. Never returns an diff --git a/llmpool/pool_test.go b/llmpool/pool_test.go index b1e69c8..21b7568 100644 --- a/llmpool/pool_test.go +++ b/llmpool/pool_test.go @@ -55,6 +55,17 @@ func (f *fakeProv) Stream(ctx context.Context, req llm.CompletionRequest) iter.S } } +func (f *fakeProv) Complete(ctx context.Context, req llm.CompletionRequest) (llm.Message, string, llm.Usage, error) { + acc := llm.NewAccumulator() + for ev, err := range f.Stream(ctx, req) { + if err != nil { + return llm.Message{}, "", llm.Usage{}, err + } + acc.Add(ev) + } + return acc.Message(), acc.StopReason, acc.Usage, nil +} + // ok builds a provider that always succeeds with one text delta. func okProv(name, text string) *fakeProv { return &fakeProv{name: name, events: [][]llm.StreamEvent{{{Type: llm.SETextDelta, Text: text}}}} diff --git a/llmrec/llmrec.go b/llmrec/llmrec.go index 1bbd55c..71faf98 100644 --- a/llmrec/llmrec.go +++ b/llmrec/llmrec.go @@ -39,10 +39,10 @@ func TaskIDFrom(ctx context.Context) string { // Recorder wraps an llm.Provider and records every completion call. type Recorder struct { - inner llm.Provider - pg *db.DB - model string // model name (from config, not in CompletionRequest) - prof string // LLM profile name (from llm_profiles) + inner llm.Provider + pg *db.DB + model string // model name (from config, not in CompletionRequest) + prof string // LLM profile name (from llm_profiles) // thinkingType / reasoningEffort 是配置级思考参数(思考开关 / 思考强度)。它们在 // norma 的 buildBody() 里从 provider 配置注入真正的 HTTP body,不出现在 // CompletionRequest 上,故 Recorder 需在此单独带一份,序列化时写进录制。 @@ -179,6 +179,45 @@ func (r *Recorder) Stream(ctx context.Context, req llm.CompletionRequest) iter.S } } +// Complete implements the atomic completion path while preserving the same +// usage metering and optional body recording semantics as Stream. +func (r *Recorder) Complete(ctx context.Context, req llm.CompletionRequest) (llm.Message, string, llm.Usage, error) { + recordBodies := r.enabled == nil || r.enabled() + start := time.Now() + session := transcript.SessionIDFrom(ctx) + parsedID, worker := parseSession(session) + expID := db.ParseExpID(parsedID) + taskID := TaskIDFrom(ctx) + if taskID == "" { + taskID = parsedID + } + + reqBody := "" + if recordBodies { + reqBody = r.serializeRequest(req) + } + + msg, stopReason, usage, err := r.inner.Complete(ctx, req) + status := "ok" + if err != nil { + status = "error" + } + r.recordUsage(taskID, expID, worker, usage, int(time.Since(start).Milliseconds()), status) + if recordBodies { + var textBuf, thinkingBuf strings.Builder + for _, block := range msg.Content { + switch block.Type { + case llm.BlockText: + textBuf.WriteString(block.Text) + case llm.BlockThinking: + thinkingBuf.WriteString(block.Thinking) + } + } + r.record(req, session, taskID, worker, reqBody, start, textBuf.String(), thinkingBuf.String(), usage, stopReason, err) + } + return msg, stopReason, usage, err +} + // recordUsage appends one lightweight metering row to llm_usage (no bodies). Skips // zero-token calls with no model, which carry nothing worth metering. func (r *Recorder) recordUsage(taskID string, expID int64, worker string, usage llm.Usage, latencyMs int, status string) { diff --git a/llmrec/llmrec_test.go b/llmrec/llmrec_test.go index e998684..19c7d4b 100644 --- a/llmrec/llmrec_test.go +++ b/llmrec/llmrec_test.go @@ -2,9 +2,28 @@ package llmrec import ( "context" + "iter" "testing" + + "github.com/Autumn-27/norma/llm" ) +type completeProvider struct{} + +func (completeProvider) Stream(context.Context, llm.CompletionRequest) iter.Seq2[llm.StreamEvent, error] { + return func(func(llm.StreamEvent, error) bool) {} +} + +func (completeProvider) Complete(context.Context, llm.CompletionRequest) (llm.Message, string, llm.Usage, error) { + return llm.Message{ + Role: llm.RoleAssistant, + Content: []llm.ContentBlock{ + {Type: llm.BlockThinking, Thinking: "reasoning"}, + llm.TextBlock("answer"), + }, + }, "stop", llm.Usage{InputTokens: 7, OutputTokens: 3}, nil +} + func TestTaskIDContextUsesExplicitRegistryID(t *testing.T) { ctx := WithTaskID(context.Background(), " 42 ") if got := TaskIDFrom(ctx); got != "42" { @@ -27,3 +46,20 @@ func TestParseSessionFallbackIsExplorationScoped(t *testing.T) { t.Fatalf("unexpected non-session parse: (%q,%q)", taskID, worker) } } + +func TestCompleteForwardsAtomicResponse(t *testing.T) { + recorder := Wrap(completeProvider{}, nil, "model", "profile", "", "", func() bool { return false }) + msg, stopReason, usage, err := recorder.Complete(context.Background(), llm.CompletionRequest{}) + if err != nil { + t.Fatalf("Complete() error: %v", err) + } + if got := msg.Text(); got != "answer" { + t.Fatalf("Complete() text=%q want answer", got) + } + if stopReason != "stop" { + t.Fatalf("Complete() stop reason=%q want stop", stopReason) + } + if usage.InputTokens != 7 || usage.OutputTokens != 3 { + t.Fatalf("Complete() usage=%+v", usage) + } +} diff --git a/server/commands.go b/server/commands.go index 12904a7..aa05427 100644 --- a/server/commands.go +++ b/server/commands.go @@ -15,14 +15,7 @@ func (s *Server) pgListCommands(w http.ResponseWriter, r *http.Request) { size := atoiDefault(q.Get("size"), 50) keyword := q.Get("q") - var expID *int64 - if tv := q.Get("task"); tv != "" { - if n, err := strconv.ParseInt(tv, 10, 64); err == nil { - expID = &n - } - } - - records, total, err := s.m.PG().ListCommands(expID, keyword, page, size) + records, total, err := s.m.PG().ListCommands(commandTaskFilter(q.Get("task")), keyword, page, size) if err != nil { writeErr(w, 500, err.Error()) return @@ -30,6 +23,37 @@ func (s *Server) pgListCommands(w http.ResponseWriter, r *http.Request) { writeJSON(w, 200, map[string]any{"commands": records, "total": total}) } +// pgToolStats returns per-tool call counts for the tool-execution history, under +// the same task/keyword filters the list takes — the summary describes the whole +// filtered set, not the page on screen. +func (s *Server) pgToolStats(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + pg := s.m.PG() + if pg == nil { + writeJSON(w, 200, map[string]any{"stats": []db.ToolStat{}}) + return + } + stats, err := pg.ToolStats(commandTaskFilter(q.Get("task")), q.Get("q")) + if err != nil { + writeErr(w, 500, err.Error()) + return + } + writeJSON(w, 200, map[string]any{"stats": stats}) +} + +// commandTaskFilter parses the ?task= exploration id. Absent or unparsable → nil +// (no filter), matching the list endpoint's long-standing lenient behaviour. +func commandTaskFilter(v string) *int64 { + if v == "" { + return nil + } + n, err := strconv.ParseInt(v, 10, 64) + if err != nil { + return nil + } + return &n +} + // pgListLLMRecords returns paginated LLM call records. func (s *Server) pgListLLMRecords(w http.ResponseWriter, r *http.Request) { q := r.URL.Query() diff --git a/server/dto.go b/server/dto.go index cf7133d..c681bcf 100644 --- a/server/dto.go +++ b/server/dto.go @@ -580,6 +580,9 @@ type LLMProfileDTO struct { // pool_exclude=true 则不作为故障转移目标，但仍可被 agent/任务显式绑定。 Priority int `json:"priority"` PoolExclude bool `json:"pool_exclude"` + // 收发模式：true=流式(SSE) | false=非流式。没有 omitempty —— false 必须出现在 + // 响应里，否则前端读不到「非流式」，开关会回落成默认的流式。 + Streaming bool `json:"streaming"` } func llmProfileDTO(p *db.LLMProfile) LLMProfileDTO { @@ -599,6 +602,7 @@ func llmProfileDTO(p *db.LLMProfile) LLMProfileDTO { IsDefault: p.IsDefault, Priority: p.Priority, PoolExclude: p.PoolExclude, + Streaming: p.Streaming, } } diff --git a/server/goals.go b/server/goals.go index dbc497d..c8e6d98 100644 --- a/server/goals.go +++ b/server/goals.go @@ -383,7 +383,8 @@ func (s *Server) createGoals(ctx context.Context, t *Task, emit func(db.Activity } taskID, _ := strconv.ParseInt(t.ID, 10, 64) s.engine.BeginLLMCall(t.ID) - decomposed := agent.DecomposeGoalsWithProvider(ctx, s.agentsForTask(t).runtime, s.m.dir, t.Goal, t.Description, as, t.Store, taskID, emit) + goalRuntime := s.agentsForTask(t).runtime + decomposed := agent.DecomposeGoalsWithProvider(ctx, goalRuntime, s.m.dir, t.Goal, t.Description, as, t.Store, taskID, goalRuntime.nonStreaming(), emit) s.engine.EndLLMCall(t.ID) for _, g := range decomposed { if strings.TrimSpace(g.Text) != "" { diff --git a/server/orchestration.go b/server/orchestration.go index 9057ea7..a48d83b 100644 --- a/server/orchestration.go +++ b/server/orchestration.go @@ -450,9 +450,9 @@ func (s *Server) seedOrchestrationTools() { s.seedPlannerDefaultBindings() s.seedAutoReportFindingBinding() s.unbindGoalMetDefault() - s.reseedGoalsPrompt() // goals 提示词加入「抽操作约束」步 → 旧库追加一版新默认(一次性) + s.reseedGoalsPrompt() // goals 提示词加入「抽操作约束」步 → 旧库追加一版新默认(一次性) s.reseedMainAgentPrompt() // mainagent 提示词加入「目标达成后 add_intent 反问是否建目标」(一次性) - s.seedReporterAgent() // 预置「报告撰写」agent + 工具绑定 + finding 触发器(一次性) + s.seedReporterAgent() // 预置「报告撰写」agent + 工具绑定 + finding 触发器(一次性) // 注：pentest 的默认工具绑定无需迁移——BuiltinToolSeeds 在全新初始化时就把 // list_assets/insert_assets/report_finding/list_findings/list_companies 连同 // pentest 一起 seed 好了（项目尚无旧库，不做迁移）。 @@ -464,7 +464,7 @@ func (s *Server) seedOrchestrationTools() { // reaches an old DB otherwise. Preserves each tool's agent binding + enabled flag. // Bump the flag whenever these tools' schemas/descriptions change in code. func (s *Server) refreshBuiltinToolSchemas() { - const flag = "tool_schema_refresh_v6_insert_assets_related" + const flag = "tool_schema_refresh_v7_list_facts_paging" if v, _, _ := s.m.pg.GetSetting(flag); v == "true" { return } @@ -480,7 +480,9 @@ func (s *Server) refreshBuiltinToolSchemas() { // “结束空轮”的手段、刚开跑就误判整个任务完成。 // - insert_assets：新增 related 入参(标记资产是否与当前任务相关、决定是否入覆盖度)， // SeedTool 首插入only，旧库已 seed 的 schema 否则收不到这个新参数。 - refreshBuiltin := map[string]bool{"goal_met": true, "insert_assets": true} + // - list_facts：改为分页，新增 limit/before/q 入参；旧库已 seed 的空 schema 否则 + // 在工具管理页显示「无参数」，模型也拿不到这几个参数说明。 + refreshBuiltin := map[string]bool{"goal_met": true, "insert_assets": true, "list_facts": true} for _, sd := range agent.BuiltinToolSeeds() { if !refreshBuiltin[sd.Key] { continue diff --git a/server/server.go b/server/server.go index 942c8ca..4968091 100644 --- a/server/server.go +++ b/server/server.go @@ -311,6 +311,7 @@ func (s *Server) loadLLMConfig() (agent.Config, bool) { cfg.ContextWindowK = p.ContextWindowK cfg.ThinkingType = p.ThinkingType cfg.ReasoningEffort = p.ReasoningEffort + cfg.Stream = p.Streaming if cfg.APIKey == "" { return cfg, false } @@ -328,14 +329,15 @@ func (s *Server) saveLLMConfig(cfg agent.Config) error { } var id int64 // agent.Config carries no failover fields, so carry the stored ones forward — - // otherwise this legacy endpoint would silently reset the profile's priority - // and pool_exclude to zero values on every save. + // otherwise this legacy endpoint would silently reset the profile's priority, + // pool_exclude, and streaming to zero values on every save. var priority int var poolExclude bool + streaming := true // 旧库/新建默认流式 if profs, _ := s.m.pg.ListProfiles(); profs != nil { for _, p := range profs { if p.Name == "default" { - id, priority, poolExclude = p.ID, p.Priority, p.PoolExclude + id, priority, poolExclude, streaming = p.ID, p.Priority, p.PoolExclude, p.Streaming break } } @@ -344,7 +346,7 @@ func (s *Server) saveLLMConfig(cfg agent.Config) error { ID: id, Name: "default", Format: format, Model: cfg.Model, BaseURL: cfg.BaseURL, Proxy: cfg.Proxy, APIKey: cfg.APIKey, RatePerSecond: cfg.RatePerSecond, RatePerMinute: cfg.RatePerMinute, ContextWindowK: cfg.ContextWindowK, ThinkingType: cfg.ThinkingType, ReasoningEffort: cfg.ReasoningEffort, IsDefault: true, - Priority: priority, PoolExclude: poolExclude, + Priority: priority, PoolExclude: poolExclude, Streaming: streaming, }) if err != nil { return err @@ -394,6 +396,7 @@ func (s *Server) buildPlannerWorker(pinID *int64, gProv llm.Provider, gCfg agent wk.SetMemory(memory.NewStore(filepath.Join(s.m.dir, "memory"))) wk.SetWebSearch(s.webSearchFor("worker")) wk.SetConstraintInject(s.constraintInjectWorker) // 操作约束注入 worker(可配置,默认开;每轮读) + wk.SetNonStreaming(nonStreamingResolver(wCfg)) // 该 profile 选非流式时走 Provider.Complete pProv, pCfg := s.providerForAgent("planner", pinID, gProv, gCfg) pl := agent.NewPlanner(pProv, pCfg.Model, s.m.dir, tx, pCfg.CompactionWindow(), s.agentMaxTurns("planner")) pl.SetKillWork(s.engine.KillWork) // planner kill_work → terminate a running work @@ -401,9 +404,18 @@ func (s *Server) buildPlannerWorker(pinID *int64, gProv llm.Provider, gCfg agent pl.SetProxy(s.m.ProxyAddr(), s.m.ProxyCACert()) // WebFetch through the recording proxy pl.SetWebSearch(s.webSearchFor("planner")) pl.SetConstraintInject(s.constraintInjectPlanner) // 操作约束注入 planner(可配置,默认开;每轮读) + pl.SetNonStreaming(nonStreamingResolver(pCfg)) // 该 profile 选非流式时走 Provider.Complete return pl, wk } +// nonStreamingResolver returns a resolver capturing a profile's streaming choice. +// The agents read it per run; a profile change rebuilds the agents (applyLLM), +// so the captured value is always the one in effect for this build. +func nonStreamingResolver(cfg agent.Config) func() bool { + nonStreaming := !cfg.Stream + return func() bool { return nonStreaming } +} + // applyLLM (re)builds the planner/worker/main-agent from cfg and installs them on // the running engine as the GLOBAL active pair. Safe to call at runtime (UI configures LLM). func (s *Server) applyLLM(cfg agent.Config) error { @@ -447,12 +459,14 @@ func (s *Server) applyLLM(cfg agent.Config) error { s.mainAgent.SetProxy(s.m.ProxyAddr(), s.m.ProxyCACert()) // WebFetch through the recording proxy s.mainAgent.SetWebSearch(s.webSearchFor("mainagent")) s.mainAgent.SetSteerWork(s.engine.SteerWork) // steer_work：人对运行中 work 实时纠偏 + s.mainAgent.SetNonStreaming(nonStreamingResolver(mCfg)) // chat agent serves MANY custom agents by key → it holds the GLOBAL opts // (backend/key) and gates Enabled per-conversation-agent at Chat time. 对话始终用激活配置。 s.chatAgent = agent.NewChatAgent(prov, cfg.Model, s.m.dir, tx, win) // chat page runner s.chatAgent.SetProxy(s.m.ProxyAddr(), s.m.ProxyCACert()) s.chatAgent.SetWebSearch(s.m.WebSearchOpts()) s.chatAgent.SetGuard(s.chatGuard()) + s.chatAgent.SetNonStreaming(nonStreamingResolver(cfg)) s.llmProv = prov s.llmCfg = cfg s.llmOn = true @@ -478,6 +492,7 @@ func (s *Server) loadProfileConfig(id int64) (agent.Config, bool) { cfg.ContextWindowK = p.ContextWindowK cfg.ThinkingType = p.ThinkingType cfg.ReasoningEffort = p.ReasoningEffort + cfg.Stream = p.Streaming if cfg.APIKey == "" { return cfg, false } @@ -629,6 +644,7 @@ func (s *Server) chatAgentForProfile(id int64) *agent.ChatAgent { ca.SetProxy(s.m.ProxyAddr(), s.m.ProxyCACert()) ca.SetWebSearch(s.m.WebSearchOpts()) ca.SetGuard(s.chatGuard()) + ca.SetNonStreaming(nonStreamingResolver(cfg)) s.profMu.Lock() if ex := s.profChatAgents[id]; ex != nil { // lost the race → keep the winner ca = ex @@ -786,6 +802,7 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("GET /api/traffic/exchange", s.getTrafficExchange) mux.HandleFunc("GET /api/traffic/blob", s.getTrafficBlob) mux.HandleFunc("GET /api/commands", s.pgListCommands) + mux.HandleFunc("GET /api/commands/stats", s.pgToolStats) // 按工具聚合调用次数 mux.HandleFunc("GET /api/llm/records", s.pgListLLMRecords) mux.HandleFunc("DELETE /api/llm/records", s.pgDeleteLLMRecords) mux.HandleFunc("GET /api/llm/records/tasks", s.pgLLMTasks) @@ -1335,6 +1352,7 @@ func (s *Server) testLLM(w http.ResponseWriter, r *http.Request) { ThinkingType string `json:"thinking_type"` ReasoningEffort string `json:"reasoning_effort"` ProfileID *int64 `json:"profile_id"` // 测已存 profile 时传入：api_key 为空则用它存的 key + Streaming *bool `json:"streaming"` // 省略=流式，与保存 profile 时同一套默认 } if err := json.NewDecoder(r.Body).Decode(&req); err != nil { writeErr(w, 400, err.Error()) @@ -1345,6 +1363,11 @@ func (s *Server) testLLM(w http.ResponseWriter, r *http.Request) { // reasoning_effort/thinking field fails the test too (no false "test ok, run 400"). cfg.ThinkingType = req.ThinkingType cfg.ReasoningEffort = req.ReasoningEffort + // 同理，收发模式也照该 profile 的选择来：只支持其中一种通道的端点必须在这里就 + // 暴露，而不是等会话里才发现"测试通过的配置根本跑不动"。 + if req.Streaming != nil { + cfg.Stream = *req.Streaming + } // API Key 解析优先级：表单输入 > 指定 profile 存的 key > 全局配置的 key。 // 已存 profile 的 key 不回传浏览器，所以测试已存配置时表单为空，需从 DB 取。 if cfg.APIKey == "" && req.ProfileID != nil { @@ -1361,12 +1384,27 @@ func (s *Server) testLLM(w http.ResponseWriter, r *http.Request) { writeJSON(w, 200, map[string]any{"ok": false, "error": "未提供 API Key"}) return } - lat, err := agent.TestConnection(r.Context(), cfg) + lat, reply, err := agent.TestConnection(r.Context(), cfg) if err != nil { writeJSON(w, 200, map[string]any{"ok": false, "error": err.Error()}) return } - writeJSON(w, 200, map[string]any{"ok": true, "latency_ms": lat.Milliseconds(), "model": cfg.Model}) + // 回传模型实际回复，让"测试通过"有据可查：看得见它确实说了话，而不只是 HTTP 200。 + writeJSON(w, 200, map[string]any{ + "ok": true, "latency_ms": lat.Milliseconds(), "model": cfg.Model, "reply": truncateReply(reply), + }) +} + +// truncateReply clips a connection-test reply for display. A model told to answer +// "OK" can still ramble (or think out loud); the UI only needs enough to show it +// really said something. Rune-based so multibyte text never splits mid-character. +func truncateReply(s string) string { + const max = 200 + r := []rune(s) + if len(r) <= max { + return s + } + return string(r[:max]) + "…" } type createTaskReq struct { diff --git a/server/server_mgmt.go b/server/server_mgmt.go index 3b2fad7..02baba2 100644 --- a/server/server_mgmt.go +++ b/server/server_mgmt.go @@ -1736,10 +1736,14 @@ func (s *Server) pgSaveProfile(w http.ResponseWriter, r *http.Request) { return } // APIKey has json:"-" on db.LLMProfile so it is never leaked to the UI on - // read; accept it here via a sibling field for create/update. + // read; accept it here via a sibling field for create/update. Streaming is a + // *bool shadowing the embedded json:"streaming": an absent field must default + // to streaming (true), which a plain bool zero value (false = non-streaming) + // would get wrong — legacy/partial clients that never send it stay streaming. var body struct { db.LLMProfile - APIKey string `json:"api_key"` + APIKey string `json:"api_key"` + Streaming *bool `json:"streaming"` } if err := decode(r, &body); err != nil { writeErr(w, 400, err.Error()) @@ -1747,6 +1751,7 @@ func (s *Server) pgSaveProfile(w http.ResponseWriter, r *http.Request) { } p := body.LLMProfile p.APIKey = body.APIKey + p.Streaming = body.Streaming == nil || *body.Streaming id, err := pg.SaveProfile(&p) if err != nil { writeErr(w, 500, err.Error()) diff --git a/server/task_llm.go b/server/task_llm.go index 01a94b7..5a22600 100644 --- a/server/task_llm.go +++ b/server/task_llm.go @@ -139,6 +139,36 @@ func (r *taskLLMRuntime) current() (*Task, int64, int64, llm.Provider, error) { return t, 0, pt.LLMChainRevision, prov, nil } +// nonStreaming reports whether the task's currently-active LLM source is set to +// non-streaming. It mirrors current()'s source precedence (agent binding → +// active chain profile → global) but reads only the streaming flag. Read-only +// and best-effort: any resolution error falls back to streaming (false), the +// safe default. If failover switches to a profile with a different streaming +// setting, the change takes effect on the next agent run (a fresh Session is +// built per run in captureRun). +func (r *taskLLMRuntime) nonStreaming() bool { + taskNum, err := parseTaskID(r.taskID) + if err != nil { + return false + } + pt, err := r.s.m.pg.GetTask(taskNum) + if err != nil || pt == nil { + return false + } + if _, cfg, ok := r.s.agentBindingProvider(r.agentKey); ok { + return !cfg.Stream + } + if len(pt.LLMProfileIDs) > 0 && pt.ActiveLLMProfileID != nil { + if _, cfg, ok := r.s.providerForProfile(*pt.ActiveLLMProfileID); ok { + return !cfg.Stream + } + } + if _, cfg, ok := r.s.globalProvider(); ok { + return !cfg.Stream + } + return false +} + func parseTaskID(id string) (int64, error) { n, err := strconv.ParseInt(id, 10, 64) if err != nil || n <= 0 { @@ -147,9 +177,11 @@ func parseTaskID(id string) (int64, error) { return n, nil } -func (r *taskLLMRuntime) Stream(ctx context.Context, req llm.CompletionRequest) iter.Seq2[llm.StreamEvent, error] { - ctx = llmrec.WithTaskID(ctx, r.taskID) - hooks := taskLLMStreamHooks{ +// streamHooks builds the failover/exhaustion callbacks shared by Stream and +// Complete: how to read the current profile selection, how to mark it quota +// exhausted, and how to emit a failover transition. +func (r *taskLLMRuntime) streamHooks() taskLLMStreamHooks { + return taskLLMStreamHooks{ current: func() (taskLLMSelection, error) { task, profileID, revision, provider, err := r.current() return taskLLMSelection{task: task, profileID: profileID, revision: revision, provider: provider}, err @@ -169,7 +201,70 @@ func (r *taskLLMRuntime) Stream(ctx context.Context, req llm.CompletionRequest) r.s.emitTaskLLMTransition(selection.task, transition, cause) }, } - return streamTaskLLM(ctx, r.taskID, req, hooks) +} + +func (r *taskLLMRuntime) Stream(ctx context.Context, req llm.CompletionRequest) iter.Seq2[llm.StreamEvent, error] { + ctx = llmrec.WithTaskID(ctx, r.taskID) + return streamTaskLLM(ctx, r.taskID, req, r.streamHooks()) +} + +// Complete is the non-streaming counterpart of Stream. A non-streaming call is +// atomic — it never delivers partial output — so every failure is safe to retry +// on the same provider or fail over to the next profile without risking +// duplicated model output or tool execution (the "committed" bookkeeping the +// streaming path needs is unnecessary here). +func (r *taskLLMRuntime) Complete(ctx context.Context, req llm.CompletionRequest) (llm.Message, string, llm.Usage, error) { + ctx = llmrec.WithTaskID(ctx, r.taskID) + return completeTaskLLM(ctx, r.taskID, req, r.streamHooks()) +} + +func completeTaskLLM(ctx context.Context, taskID string, req llm.CompletionRequest, hooks taskLLMStreamHooks) (llm.Message, string, llm.Usage, error) { + for { + selection, err := hooks.current() + if err != nil { + return llm.Message{}, "", llm.Usage{}, err + } + var ( + msg llm.Message + sr string + usage llm.Usage + callErr error + ) + // 同 provider 安全窗口重试:非流式调用要么整体成功、要么整体失败,没有 + // 中途已交付输出的问题,所以任何瞬时失败都可原样重试。 + for attempt := 0; ; attempt++ { + msg, sr, usage, callErr = selection.provider.Complete(ctx, req) + if callErr != nil && ctx.Err() == nil && + attempt < sameProviderStreamRetries && isRetryableStreamError(callErr) { + backoff := sameProviderRetryBackoff(attempt) + log.Printf("[task-llm] task %s 非流式调用失败,%v 后同 provider 重试 (%d/%d): %v", + taskID, backoff, attempt+1, sameProviderStreamRetries, callErr) + if sleepCtx(ctx, backoff) { + break // 退避期间 ctx 取消 → 停止重试 + } + continue + } + break + } + if callErr == nil { + return msg, sr, usage, nil + } + // profileID=0: 显式链已被清空;非额度错误:透传。二者都不改任务 failover 状态。 + if selection.profileID == 0 || !isQuotaExhaustedError(callErr) { + return llm.Message{}, "", llm.Usage{}, callErr + } + transition, markErr := hooks.exhaust(selection, callErr) + if markErr != nil { + return llm.Message{}, "", llm.Usage{}, fmt.Errorf("mark profile quota exhausted after %v: %w", callErr, markErr) + } + if transition.Advanced && !transition.Stale && hooks.transition != nil { + hooks.transition(selection, transition, callErr) + } + if !transition.Stale && transition.NextProfileID == nil { + return llm.Message{}, "", llm.Usage{}, &taskLLMError{taskID: taskID, chainExhausted: transition.ChainExhausted, cause: callErr} + } + // 没有任何输出交付给调用方,换下一个 profile 重放同一逻辑请求是安全的。 + } } func streamTaskLLM(ctx context.Context, taskID string, req llm.CompletionRequest, hooks taskLLMStreamHooks) iter.Seq2[llm.StreamEvent, error] { @@ -435,6 +530,7 @@ func (s *Server) agentsForTask(t *Task) *taskAgentBundle { window := workerRuntime.CompactionWindow() wk := agent.NewWorker(workerRuntime, "task-router", s.m.dir, tx, window, s.agentMaxTurns("worker")) wk.SetCompactionWindowResolver(workerRuntime.CompactionWindow) + wk.SetNonStreaming(workerRuntime.nonStreaming) // 按任务当前激活 profile 的流式开关(每轮读) wk.SetRunTimeout(time.Duration(s.agentRunSeconds("worker")) * time.Second) wk.SetProxy(s.m.ProxyAddr(), s.m.ProxyCACert()) wk.SetMemory(memory.NewStore(filepath.Join(s.m.dir, "memory"))) @@ -442,6 +538,7 @@ func (s *Server) agentsForTask(t *Task) *taskAgentBundle { wk.SetConstraintInject(s.constraintInjectWorker) // 操作约束注入 worker(可配置,默认开) pl := agent.NewPlanner(plannerRuntime, "task-router", s.m.dir, tx, plannerRuntime.CompactionWindow(), s.agentMaxTurns("planner")) pl.SetCompactionWindowResolver(plannerRuntime.CompactionWindow) + pl.SetNonStreaming(plannerRuntime.nonStreaming) pl.SetKillWork(s.engine.KillWork) pl.SetSteerWork(s.engine.SteerWork) pl.SetProxy(s.m.ProxyAddr(), s.m.ProxyCACert()) @@ -449,6 +546,7 @@ func (s *Server) agentsForTask(t *Task) *taskAgentBundle { pl.SetConstraintInject(s.constraintInjectPlanner) // 操作约束注入 planner(可配置,默认开) main := agent.NewMainAgent(mainRuntime, "task-router", s.m.dir, tx, mainRuntime.CompactionWindow(), s.agentMaxTurns("mainagent")) main.SetCompactionWindowResolver(mainRuntime.CompactionWindow) + main.SetNonStreaming(mainRuntime.nonStreaming) main.SetProxy(s.m.ProxyAddr(), s.m.ProxyCACert()) main.SetWebSearch(s.webSearchFor("mainagent")) main.SetSteerWork(s.engine.SteerWork) // steer_work：人对运行中 work 实时纠偏 diff --git a/server/task_llm_test.go b/server/task_llm_test.go index 48da7d3..5316b9b 100644 --- a/server/task_llm_test.go +++ b/server/task_llm_test.go @@ -37,6 +37,23 @@ func (p *scriptedLLMProvider) Stream(context.Context, llm.CompletionRequest) ite } } +func (p *scriptedLLMProvider) Complete(ctx context.Context, req llm.CompletionRequest) (llm.Message, string, llm.Usage, error) { + return accumulateStreamForTest(ctx, p.Stream, req) +} + +// accumulateStreamForTest drains a mock's Stream to satisfy the non-streaming +// Complete method. +func accumulateStreamForTest(ctx context.Context, stream func(context.Context, llm.CompletionRequest) iter.Seq2[llm.StreamEvent, error], req llm.CompletionRequest) (llm.Message, string, llm.Usage, error) { + acc := llm.NewAccumulator() + for ev, err := range stream(ctx, req) { + if err != nil { + return llm.Message{}, "", llm.Usage{}, err + } + acc.Add(ev) + } + return acc.Message(), acc.StopReason, acc.Usage, nil +} + // withZeroRetryBackoff sets the same-provider retry backoff to zero for the // duration of a (serial) test and returns a restore func for defer. func withZeroRetryBackoff() func() { @@ -73,6 +90,10 @@ func (p *flakyThenOKProvider) Stream(context.Context, llm.CompletionRequest) ite } } +func (p *flakyThenOKProvider) Complete(ctx context.Context, req llm.CompletionRequest) (llm.Message, string, llm.Usage, error) { + return accumulateStreamForTest(ctx, p.Stream, req) +} + func collectTaskLLMStream(seq iter.Seq2[llm.StreamEvent, error]) ([]llm.StreamEvent, error) { var events []llm.StreamEvent var streamErr error diff --git a/web/src/app/(main)/function/commands/page.tsx b/web/src/app/(main)/function/commands/page.tsx index 7d58b48..6a5954e 100644 --- a/web/src/app/(main)/function/commands/page.tsx +++ b/web/src/app/(main)/function/commands/page.tsx @@ -2,17 +2,18 @@ import * as React from "react"; -import { ChevronLeftIcon, ChevronRightIcon, Loader2Icon, SearchIcon, TerminalIcon } from "lucide-react"; +import { BarChart3Icon, ChevronLeftIcon, ChevronRightIcon, Loader2Icon, SearchIcon, TerminalIcon } from "lucide-react"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { Card } from "@/components/ui/card"; +import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle } from "@/components/ui/dialog"; import { Input } from "@/components/ui/input"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { Sheet, SheetContent, SheetDescription, SheetHeader, SheetTitle } from "@/components/ui/sheet"; import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; import { api } from "@/lib/api"; -import type { CommandRecord } from "@/lib/types"; +import type { CommandRecord, ToolStat } from "@/lib/types"; import { cn } from "@/lib/utils"; function fmtTime(ts: string) { @@ -59,6 +60,11 @@ export default function CommandsPage() { const [total, setTotal] = React.useState(0); const [loading, setLoading] = React.useState(false); + // 各工具调用次数。弹窗打开时才拉取（多一次聚合查询，不必每次翻页都付）。 + const [statsOpen, setStatsOpen] = React.useState(false); + const [stats, setStats] = React.useState([]); + const [statsLoading, setStatsLoading] = React.useState(false); + // Selected execution is rendered in a right-side detail sheet. const [selected, setSelected] = React.useState(null); @@ -96,6 +102,24 @@ export default function CommandsPage() { }; }, [page, size, queryQ, taskFilter]); + // 统计跟随筛选条件走，和表格描述的是同一批记录（但不分页）。 + React.useEffect(() => { + if (!statsOpen) return; + let alive = true; + setStatsLoading(true); + api + .commandStats({ task: taskFilter || undefined, q: queryQ || undefined }) + .then((r) => alive && setStats(r.stats ?? [])) + .catch(() => alive && setStats([])) + .finally(() => alive && setStatsLoading(false)); + return () => { + alive = false; + }; + }, [statsOpen, queryQ, taskFilter]); + + const statsTotal = stats.reduce((n, s) => n + s.total, 0); + const statsErrors = stats.reduce((n, s) => n + s.errors, 0); + const statsMax = stats.reduce((n, s) => Math.max(n, s.total), 0); const totalPages = Math.max(1, Math.ceil(total / size)); const rangeStart = total === 0 ? 0 : page * size + 1; const rangeEnd = page * size + commands.length; @@ -141,6 +165,11 @@ export default function CommandsPage() { + +
{rangeStart}–{rangeEnd} / {total} @@ -243,6 +272,66 @@ export default function CommandsPage() {
+ {/* 工具调用统计：与表格同一批记录（同筛选、不分页） */} + + + + 工具调用统计 + + {taskFilter || queryQ ? "当前筛选条件下的全部记录" : "全部工具执行记录"} + {stats.length > 0 && ( + <> + {" · "} + {stats.length} 个工具 ·{" "} + {statsTotal} 次调用 + {statsErrors > 0 && ( + <> + {" · 失败 "} + {statsErrors} + + )} + + )} + + + + {statsLoading && stats.length === 0 ? ( +
+ +
+ ) : stats.length === 0 ? ( +
暂无统计数据
+ ) : ( +
+ {stats.map((s) => ( +
+
+
+ {s.tool} + {s.errors > 0 && ( + 失败 {s.errors} + )} +
+
+
0 ? (s.total / statsMax) * 100 : 0}%` }} + /> +
+
+
+
{s.total}
+
+ {statsTotal > 0 ? ((s.total / statsTotal) * 100).toFixed(1) : "0.0"}% +
+
+
+ ))} +
+ )} + +
+ !open && setSelected(null)}> {selected && ( diff --git a/web/src/app/(main)/system/llm/page.tsx b/web/src/app/(main)/system/llm/page.tsx index b08c6ef..35c7806 100644 --- a/web/src/app/(main)/system/llm/page.tsx +++ b/web/src/app/(main)/system/llm/page.tsx @@ -309,6 +309,7 @@ function ProfileSheet({ const [effort, setEffort] = React.useState(NONE); const [priority, setPriority] = React.useState("0"); // 轮询顺位;越大越先 const [poolExclude, setPoolExclude] = React.useState(false); + const [streaming, setStreaming] = React.useState(true); // true=流式(默认);false=非流式 const [testing, setTesting] = React.useState(false); const [saving, setSaving] = React.useState(false); const [models, setModels] = React.useState([]); @@ -331,6 +332,7 @@ function ProfileSheet({ setEffort(fromStore(profile?.reasoning_effort)); setPriority(String(profile?.priority ?? 0)); setPoolExclude(profile?.pool_exclude ?? false); + setStreaming(profile?.streaming ?? true); setApiKey(""); setKeyHint(profile?.api_key_hint ?? ""); setModels([]); @@ -365,8 +367,22 @@ function ProfileSheet({ try { // 用配置实际会跑的思考参数来测，这样不支持该字段的模型在这里就失败， // 而不是等到跑任务时才炸。传 profile id：Key 输入框留空时用已存的 Key。 - const r = await api.testLLM(format, model, baseUrl, apiKey, proxy, toStore(thinkingType), toStore(effort), profileId); - if (r.ok) toast.success(`连接成功 · ${r.latency_ms ?? "?"}ms · ${r.model ?? model}`); + const r = await api.testLLM( + format, + model, + baseUrl, + apiKey, + proxy, + toStore(thinkingType), + toStore(effort), + profileId, + streaming, + ); + // 回复内容一并展示：看得见模型确实说了话，才算和会话里跑通是一回事。 + if (r.ok) + toast.success(`连接成功 · ${r.latency_ms ?? "?"}ms · ${r.model ?? model}`, { + description: r.reply ? `回复：${r.reply}` : undefined, + }); else toast.error(`连接失败：${r.error ?? "未知"}`); } catch (e) { toast.error(`测试出错：${(e as Error).message}`); @@ -398,6 +414,7 @@ function ProfileSheet({ reasoning_effort: toStore(effort), priority: Number(priority) || 0, pool_exclude: poolExclude, + streaming, }); if (isNew) toast.success(`已新建：${name.trim()}（在卡片上「设为激活」以启用）`); else toast.success(profile?.is_default ? "已保存，激活配置即时生效，无需重启" : "已保存"); @@ -525,7 +542,8 @@ function ProfileSheet({ onChange={(e) => setProxy(e.target.value)} />

- 仅 LLM 出站请求走此代理，支持 http/https/socks5，可带账号密码（如 socks5://user:pass@host:port，密码含特殊字符需 URL 编码）；留空表示不使用代理（直连）。 + 仅 LLM 出站请求走此代理，支持 http/https/socks5，可带账号密码（如 + socks5://user:pass@host:port，密码含特殊字符需 URL 编码）；留空表示不使用代理（直连）。

@@ -595,6 +613,17 @@ function ProfileSheet({ +
+
+ +

+ 开启（默认）走流式 SSE，有运行中实时进度与实时 token 计数。 关闭则走真·非流式（stream:false， + 一次性返回完整响应）——可绕开部分网关糟糕的 SSE 实现（空帧 / 思考字段丢帧）， + 代价是失去运行中的实时进度。 +

+
+ +
@@ -602,8 +631,8 @@ function ProfileSheet({

- 控制是否发送 thinking 字段。不发送=不带该字段（兼容 MiniMax 等不支持 - 的模型）；关闭=发 disabled；开启=发 enabled。与下面的强度互相独立。 + 控制是否发送 thinking 字段。不发送=不带该字段（兼容 MiniMax 等不支持 的模型）；关闭=发 + disabled；开启=发 enabled。与下面的强度互相独立。

diff --git a/web/src/lib/api.ts b/web/src/lib/api.ts index 9a9f3b7..ba197c9 100644 --- a/web/src/lib/api.ts +++ b/web/src/lib/api.ts @@ -74,6 +74,7 @@ import type { TokenTotal, TokenUsage, Tool, + ToolStat, TrafficDetail, TrafficHost, TrafficResp, @@ -677,8 +678,10 @@ export const api = { thinking_type = "", reasoning_effort = "", profile_id?: number, + streaming = true, // 用该配置真实的收发模式来测，别让"流式能通、非流式不通"漏到会话里 ) => - post<{ ok: boolean; error?: string; latency_ms?: number; model?: string }>("/llm/test", { + // reply = 模型实际回复(已截断);一个字都不回的配置后端直接判失败 + post<{ ok: boolean; error?: string; latency_ms?: number; model?: string; reply?: string }>("/llm/test", { provider, model, base_url, @@ -687,6 +690,7 @@ export const api = { thinking_type, reasoning_effort, profile_id, + streaming, }), llmProfiles: () => get<{ profiles: LLMProfile[] }>("/llm/profiles").then((r) => arr(r.profiles)), saveLLMProfile: (p: { @@ -704,6 +708,7 @@ export const api = { reasoning_effort?: string; // ""(不发送)|"low"|"medium"|"high"|"xhigh"|"max" priority?: number; // 轮询顺位，越大越先用 pool_exclude?: boolean; // true=不作为故障转移目标 + streaming?: boolean; // true(默认)=流式 | false=非流式 }) => post<{ id: number }>("/llm/profiles", p), deleteLLMProfile: (id: string) => del<{ deleted: number }>(`/llm/profiles/${id}`), activateLLMProfile: (id: string) => post<{ ok: boolean }>("/llm/profiles/active", { id: Number(id) }), @@ -972,6 +977,13 @@ export const api = { sp.set("size", String(params?.size ?? 50)); return get<{ commands: CommandRecord[]; total: number }>(`/commands?${sp}`); }, + // 各工具调用次数；沿用列表的 task/q 筛选，统计的是整个结果集而非当前页。 + commandStats: (params?: { task?: string; q?: string }) => { + const sp = new URLSearchParams(); + if (params?.task) sp.set("task", params.task); + if (params?.q) sp.set("q", params.q); + return get<{ stats: ToolStat[] }>(`/commands/stats?${sp}`); + }, // ---- LLM records ---- llmRecords: (params?: { model?: string; session?: string; task?: string; page?: number; size?: number }) => { diff --git a/web/src/lib/mock/handler.ts b/web/src/lib/mock/handler.ts index 6002dae..407fe4e 100644 --- a/web/src/lib/mock/handler.ts +++ b/web/src/lib/mock/handler.ts @@ -1493,6 +1493,17 @@ function route(m: string, path: string, seg: string[], q: URLSearchParams, b: Re // ── 工具执行历史 ── if (path === "/commands" && m === "GET") return { commands: D.commandRecords, total: D.commandRecords.length }; + if (path === "/commands/stats" && m === "GET") { + const tally = new Map(); + for (const c of D.commandRecords) { + const tool = c.tool || "-"; + const s = tally.get(tool) ?? { tool, total: 0, errors: 0 }; + s.total++; + if (c.is_error) s.errors++; + tally.set(tool, s); + } + return { stats: [...tally.values()].sort((a, b) => b.total - a.total || a.tool.localeCompare(b.tool)) }; + } // ── LLM ── if (path === "/llm/records" && m === "GET") return { records: mockLLMRecords, total: mockLLMRecords.length }; @@ -1509,7 +1520,8 @@ function route(m: string, path: string, seg: string[], q: URLSearchParams, b: Re } if (path === "/llm" && m === "GET") return D.llmConfig; if (path === "/llm" && m === "POST") return { ok: true }; - if (path === "/llm/test") return { ok: true, latency_ms: 128, model: String(b.model ?? "claude-opus-4-8") }; + if (path === "/llm/test") + return { ok: true, latency_ms: 128, model: String(b.model ?? "claude-opus-4-8"), reply: "OK" }; if (path === "/llm/profiles" && m === "GET") return { profiles: D.llmProfiles }; if (path === "/llm/profiles" && m === "POST") return { id: Number(b.id) || 3 }; if (path === "/llm/profiles/active") return { ok: true }; diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts index 633688e..5c672d1 100644 --- a/web/src/lib/types.ts +++ b/web/src/lib/types.ts @@ -869,6 +869,8 @@ export interface LLMProfile { priority?: number; // true = 不作为故障转移目标（仍可被 agent/任务显式绑定使用）。 pool_exclude?: boolean; + // true（默认）= 流式(SSE) | false = 真·非流式(stream:false，一次性返回)。 + streaming?: boolean; } // ---- LLM 轮询（故障转移）---- @@ -1133,6 +1135,13 @@ export interface CommandRecord { created_at: string; } +// 单个工具的调用统计（/commands/stats）；errors 为其中失败的次数。 +export interface ToolStat { + tool: string; + total: number; + errors: number; +} + // ---- LLM recording ---- export interface LLMRecordItem { id: number; From 1fcc4b0588b6b056efdb4734ca03673234e8afd8 Mon Sep 17 00:00:00 2001 From: pphua Date: Fri, 28 Aug 2026 14:16:00 +0800 Subject: [PATCH 16/23] fix: stream large task archive snapshots --- db/task_archives.go | 41 +++++++++++++++++++++++++++++++++------- db/task_archives_test.go | 32 +++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 7 deletions(-) diff --git a/db/task_archives.go b/db/task_archives.go index bef9f3a..c21c89d 100644 --- a/db/task_archives.go +++ b/db/task_archives.go @@ -1,6 +1,7 @@ package db import ( + "bytes" "context" "database/sql" "encoding/json" @@ -423,11 +424,37 @@ func (d *DB) FailTaskArchiveJob(id int64, activeState string, cause error) error } func queryArchiveRows(q interface { - QueryRow(query string, args ...any) *sql.Row -}, inner string, args ...any) (json.RawMessage, error) { - var raw []byte - err := q.QueryRow(`SELECT COALESCE(jsonb_agg(to_jsonb(row_data)), '[]'::jsonb) FROM (`+inner+`) row_data`, args...).Scan(&raw) - return json.RawMessage(raw), err + Query(query string, args ...any) (*sql.Rows, error) +}, inner string, args ...any) (json.RawMessage, int64, error) { + // Do not aggregate the result in PostgreSQL. A jsonb array has a hard limit + // of 256 MiB for its elements, which large LLM request/response histories can + // exceed even though every individual record is valid. Reading row JSON in + // order also avoids building a second copy of the full table in PostgreSQL. + rows, err := q.Query(`SELECT row_to_json(row_data)::text FROM (`+inner+`) row_data`, args...) + if err != nil { + return nil, 0, err + } + defer rows.Close() + + var output bytes.Buffer + output.WriteByte('[') + var count int64 + for rows.Next() { + var raw []byte + if err := rows.Scan(&raw); err != nil { + return nil, 0, err + } + if count > 0 { + output.WriteByte(',') + } + output.Write(raw) + count++ + } + if err := rows.Err(); err != nil { + return nil, 0, err + } + output.WriteByte(']') + return json.RawMessage(output.Bytes()), count, nil } func rawRowCount(raw json.RawMessage) int64 { @@ -492,12 +519,12 @@ func (d *DB) SnapshotTaskArchive(taskID int64) (*TaskArchiveSnapshot, error) { } counts := make(map[string]int64, len(queries)) for _, query := range queries { - raw, err := queryArchiveRows(tx, query.query, query.args...) + raw, count, err := queryArchiveRows(tx, query.query, query.args...) if err != nil { return nil, fmt.Errorf("snapshot %s: %w", query.name, err) } tables[query.name] = raw - counts[query.name] = rawRowCount(raw) + counts[query.name] = count } assetIDs, exclusiveAssetIDs, hosts, exclusiveHosts, err := archiveAssetMetadata(tx, taskID, expID, tables["assets"]) diff --git a/db/task_archives_test.go b/db/task_archives_test.go index dbe282f..c9a6d18 100644 --- a/db/task_archives_test.go +++ b/db/task_archives_test.go @@ -8,6 +8,38 @@ import ( "time" ) +func TestQueryArchiveRowsStreamsJSON(t *testing.T) { + d, err := Open(testDSN(t)) + if err != nil { + t.Skipf("postgres unavailable (%v) — skipping", err) + } + defer d.Close() + + payload := strings.Repeat("large request/response payload ", 64*1024) + raw, count, err := queryArchiveRows(d, `SELECT * FROM (VALUES +($1::bigint,$2::text),($3::bigint,$4::text)) archived_row(id,body) ORDER BY id`, + int64(1), payload, int64(2), "quoted: \"value\"\nline") + if err != nil { + t.Fatal(err) + } + if count != 2 { + t.Fatalf("row count=%d, want 2", count) + } + var rows []struct { + ID int64 `json:"id"` + Body string `json:"body"` + } + if err := json.Unmarshal(raw, &rows); err != nil { + t.Fatal(err) + } + if len(rows) != 2 { + t.Fatalf("decoded row count=%d, want 2", len(rows)) + } + if rows[0].ID != 1 || rows[0].Body != payload || rows[1].ID != 2 || rows[1].Body != "quoted: \"value\"\nline" { + t.Fatal("streamed rows were reordered or truncated") + } +} + func TestTaskArchiveDatabaseRoundTrip(t *testing.T) { d, err := Open(testDSN(t)) if err != nil { From 0ec45bb45d5351a8d61127cd012f979d073252ca Mon Sep 17 00:00:00 2001 From: pphua Date: Fri, 28 Aug 2026 15:25:20 +0800 Subject: [PATCH 17/23] fix: prevent OOM when archiving large tasks --- db/commands.go | 16 ++- db/db.go | 135 ++++++++++++++++------- db/db_test.go | 47 ++++++++ db/llm_usage.go | 14 ++- db/schema.sql | 3 +- db/task_archives.go | 123 ++++++++++++++++++--- db/task_archives_restore.go | 101 +++++++++++++++-- db/task_archives_test.go | 161 ++++++++++++++++++++++++++-- server/task_archive_package.go | 13 ++- server/task_archive_package_test.go | 16 ++- server/task_archives.go | 63 +++++++++-- 11 files changed, 605 insertions(+), 87 deletions(-) diff --git a/db/commands.go b/db/commands.go index d95f0b8..3edefa6 100644 --- a/db/commands.go +++ b/db/commands.go @@ -189,11 +189,21 @@ ALTER TABLE llm_records ADD COLUMN IF NOT EXISTS raw_response TEXT; // EnsureLLMRecordsTable creates the llm_records table if it does not exist. func (d *DB) EnsureLLMRecordsTable() error { - if _, err := d.Exec(llmRecordsSchema); err != nil { + tx, err := d.Begin() + if err != nil { return err } - _, err := d.Exec(llmRecordsMigrate) - return err + defer tx.Rollback() //nolint:errcheck + if err := coordinateWithSchemaMigration(tx); err != nil { + return err + } + if _, err := tx.Exec(llmRecordsSchema); err != nil { + return err + } + if _, err := tx.Exec(llmRecordsMigrate); err != nil { + return err + } + return tx.Commit() } // InsertLLMRecord stores one LLM call record. diff --git a/db/db.go b/db/db.go index de0a738..20ec514 100644 --- a/db/db.go +++ b/db/db.go @@ -3,19 +3,84 @@ package db import ( + "context" "database/sql" _ "embed" + "errors" "fmt" "net/url" "strings" + "time" "github.com/Autumn-27/artex/config" + "github.com/jackc/pgx/v5/pgconn" _ "github.com/jackc/pgx/v5/stdlib" // pgx database/sql driver ("pgx") ) //go:embed schema.sql var schemaSQL string +const schemaMigrationLockKey int64 = 7337741001 + +var schemaDeadlockRetryDelays = [...]time.Duration{ + 100 * time.Millisecond, + 250 * time.Millisecond, + 500 * time.Millisecond, + time.Second, +} + +type schemaExecer interface { + ExecContext(context.Context, string, ...any) (sql.Result, error) +} + +func isPostgresDeadlock(err error) bool { + var pgErr *pgconn.PgError + return errors.As(err, &pgErr) && pgErr.Code == "40P01" +} + +func applySchemaWithRetry(ctx context.Context, execer schemaExecer, sleep func(time.Duration)) error { + for attempt := 0; ; attempt++ { + if _, err := execer.ExecContext(ctx, schemaSQL); err != nil { + if !isPostgresDeadlock(err) || attempt >= len(schemaDeadlockRetryDelays) { + return err + } + sleep(schemaDeadlockRetryDelays[attempt]) + continue + } + return nil + } +} + +// withSchemaMigrationLock pins the session-level lock to one checked-out +// connection. Running pg_advisory_lock through *sql.DB is incorrect because a +// later schema or unlock call may use a different pooled PostgreSQL session. +func withSchemaMigrationLock(ctx context.Context, sqlDB *sql.DB, action func(*sql.Conn) error) (err error) { + conn, err := sqlDB.Conn(ctx) + if err != nil { + return err + } + defer conn.Close() + if _, err := conn.ExecContext(ctx, `SELECT pg_advisory_lock($1)`, schemaMigrationLockKey); err != nil { + return fmt.Errorf("advisory lock: %w", err) + } + defer func() { + if _, unlockErr := conn.ExecContext(context.Background(), `SELECT pg_advisory_unlock($1)`, schemaMigrationLockKey); unlockErr != nil && err == nil { + err = fmt.Errorf("advisory unlock: %w", unlockErr) + } + }() + return action(conn) +} + +// coordinateWithSchemaMigration makes long, multi-table archive transactions +// mutually exclusive with startup DDL while allowing ordinary runtime queries +// to continue normally. +func coordinateWithSchemaMigration(tx *sql.Tx) error { + if _, err := tx.Exec(`SELECT pg_advisory_xact_lock($1)`, schemaMigrationLockKey); err != nil { + return fmt.Errorf("coordinate with schema migration: %w", err) + } + return nil +} + // DSN resolves the PostgreSQL connection string and reports where it came from. // Precedence: env ARTEX_PG_DSN > config file (config.json). There is no // built-in default — it errors if neither source is configured. @@ -72,28 +137,22 @@ func Open(dsn string) (*DB, error) { sqlDB.Close() return nil, fmt.Errorf("ping postgres (%s): %w", config.Redact(dsn), err) } - // Serialize schema migration + seeding across concurrent openers (e.g. parallel - // test packages) using a session-level advisory lock. The lock is released - // automatically when the connection is returned to the pool or closed. - if _, err := sqlDB.Exec(`SELECT pg_advisory_lock(7337741001)`); err != nil { - sqlDB.Close() - return nil, fmt.Errorf("advisory lock: %w", err) - } - // pgx runs multi-statement Exec via the simple protocol when there are no args. - if _, err := sqlDB.Exec(schemaSQL); err != nil { - sqlDB.Exec(`SELECT pg_advisory_unlock(7337741001)`) //nolint:errcheck - sqlDB.Close() - return nil, fmt.Errorf("apply schema: %w", err) - } d := &DB{sqlDB} - if err := d.seedBuiltins(); err != nil { - sqlDB.Exec(`SELECT pg_advisory_unlock(7337741001)`) //nolint:errcheck + // pgx runs multi-statement Exec via the simple protocol when there are no args. + // Keep the dedicated lock connection checked out until both DDL and seeding + // finish so concurrent application instances cannot initialize out of order. + err = withSchemaMigrationLock(context.Background(), sqlDB, func(conn *sql.Conn) error { + if err := applySchemaWithRetry(context.Background(), conn, time.Sleep); err != nil { + return fmt.Errorf("apply schema: %w", err) + } + if err := d.seedBuiltins(); err != nil { + return fmt.Errorf("seed builtins: %w", err) + } + return nil + }) + if err != nil { sqlDB.Close() - return nil, fmt.Errorf("seed builtins: %w", err) - } - if _, err := sqlDB.Exec(`SELECT pg_advisory_unlock(7337741001)`); err != nil { - sqlDB.Close() - return nil, fmt.Errorf("advisory unlock: %w", err) + return nil, err } return d, nil } @@ -375,30 +434,30 @@ func (d *DB) seedDefaultInterceptRules() error { // 2. Python HTTP 客户端 .delete() 方法 // 3. JS/通用脚本里的 method: 'DELETE' / method="DELETE" { - name: "[内置] curl / wget 发送 DELETE 请求", - target: "tool_input", - typ: "regex", - pattern: `(?i)\bcurl\b[^|\n&;"]{0,300}(?:-X\s*DELETE|--request\s+DELETE|-XDELETE)|\bwget\b[^|\n&;"]{0,300}--method[=\s]+DELETE`, - action: "deny", - message: "禁止通过 curl/wget 发送 HTTP DELETE 请求，可能删除目标系统数据", + name: "[内置] curl / wget 发送 DELETE 请求", + target: "tool_input", + typ: "regex", + pattern: `(?i)\bcurl\b[^|\n&;"]{0,300}(?:-X\s*DELETE|--request\s+DELETE|-XDELETE)|\bwget\b[^|\n&;"]{0,300}--method[=\s]+DELETE`, + action: "deny", + message: "禁止通过 curl/wget 发送 HTTP DELETE 请求，可能删除目标系统数据", priority: 80, }, { - name: "[内置] Python HTTP 客户端 DELETE（requests/httpx/aiohttp）", - target: "tool_input", - typ: "regex", - pattern: `(?i)\b(?:requests|httpx|aiohttp|urllib\.request)\.delete\s*\(|session\.delete\s*\(|client\.delete\s*\(`, - action: "deny", - message: "禁止使用 Python HTTP 客户端发送 DELETE 请求", + name: "[内置] Python HTTP 客户端 DELETE（requests/httpx/aiohttp）", + target: "tool_input", + typ: "regex", + pattern: `(?i)\b(?:requests|httpx|aiohttp|urllib\.request)\.delete\s*\(|session\.delete\s*\(|client\.delete\s*\(`, + action: "deny", + message: "禁止使用 Python HTTP 客户端发送 DELETE 请求", priority: 80, }, { - name: "[内置] 脚本中声明 HTTP DELETE 方法（JS/通用）", - target: "tool_input", - typ: "regex", - pattern: `(?i)axios\.delete\s*\(|method\s*[:=]\s*['"]DELETE['"]`, - action: "deny", - message: "禁止在脚本中声明并发送 HTTP DELETE 请求", + name: "[内置] 脚本中声明 HTTP DELETE 方法（JS/通用）", + target: "tool_input", + typ: "regex", + pattern: `(?i)axios\.delete\s*\(|method\s*[:=]\s*['"]DELETE['"]`, + action: "deny", + message: "禁止在脚本中声明并发送 HTTP DELETE 请求", priority: 80, }, { diff --git a/db/db_test.go b/db/db_test.go index d549cdd..41c5c05 100644 --- a/db/db_test.go +++ b/db/db_test.go @@ -1,9 +1,56 @@ package db import ( + "context" + "database/sql" + "errors" "testing" + "time" + + "github.com/jackc/pgx/v5/pgconn" ) +type schemaExecFunc func(context.Context, string, ...any) (sql.Result, error) + +func (f schemaExecFunc) ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) { + return f(ctx, query, args...) +} + +func TestApplySchemaRetriesOnlyDeadlocks(t *testing.T) { + deadlockAttempts := 0 + delays := []time.Duration{} + err := applySchemaWithRetry(t.Context(), schemaExecFunc(func(context.Context, string, ...any) (sql.Result, error) { + deadlockAttempts++ + if deadlockAttempts < 3 { + return nil, &pgconn.PgError{Code: "40P01", Message: "deadlock detected"} + } + return nil, nil + }), func(delay time.Duration) { + delays = append(delays, delay) + }) + if err != nil { + t.Fatal(err) + } + if deadlockAttempts != 3 { + t.Fatalf("schema attempts=%d, want 3", deadlockAttempts) + } + if len(delays) != 2 || delays[0] != schemaDeadlockRetryDelays[0] || delays[1] != schemaDeadlockRetryDelays[1] { + t.Fatalf("schema retry delays=%v", delays) + } + + nonRetryable := errors.New("permission denied") + nonRetryableAttempts := 0 + err = applySchemaWithRetry(t.Context(), schemaExecFunc(func(context.Context, string, ...any) (sql.Result, error) { + nonRetryableAttempts++ + return nil, nonRetryable + }), func(time.Duration) { + t.Fatal("non-deadlock error must not be retried") + }) + if !errors.Is(err, nonRetryable) || nonRetryableAttempts != 1 { + t.Fatalf("non-retryable result: attempts=%d err=%v", nonRetryableAttempts, err) + } +} + // testDSN returns the configured DSN, skipping the test when neither the env var // nor a config file supplies one (DSN no longer has a built-in default). func testDSN(t *testing.T) string { diff --git a/db/llm_usage.go b/db/llm_usage.go index 21b3b8c..25d3abe 100644 --- a/db/llm_usage.go +++ b/db/llm_usage.go @@ -49,8 +49,18 @@ CREATE INDEX IF NOT EXISTS idx_llm_usage_exp ON llm_usage(exploration_id); // EnsureLLMUsageTable creates the llm_usage metering table if it does not exist. func (d *DB) EnsureLLMUsageTable() error { - _, err := d.Exec(llmUsageSchema) - return err + tx, err := d.Begin() + if err != nil { + return err + } + defer tx.Rollback() //nolint:errcheck + if err := coordinateWithSchemaMigration(tx); err != nil { + return err + } + if _, err := tx.Exec(llmUsageSchema); err != nil { + return err + } + return tx.Commit() } // InsertLLMUsage appends one metering row. Best-effort: callers log and continue on diff --git a/db/schema.sql b/db/schema.sql index 2f7c22f..88deae6 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -452,7 +452,7 @@ CREATE TABLE IF NOT EXISTS task_archives ( progress INTEGER NOT NULL DEFAULT 0 CHECK (progress BETWEEN 0 AND 100), error TEXT NOT NULL DEFAULT '', warnings JSONB NOT NULL DEFAULT '[]', - format_version INTEGER NOT NULL DEFAULT 1, + format_version INTEGER NOT NULL DEFAULT 2, archive_path TEXT NOT NULL DEFAULT '', sha256 TEXT NOT NULL DEFAULT '', original_size BIGINT NOT NULL DEFAULT 0, @@ -472,6 +472,7 @@ CREATE TABLE IF NOT EXISTS task_archives ( created_at TIMESTAMPTZ NOT NULL DEFAULT now(), updated_at TIMESTAMPTZ NOT NULL DEFAULT now() ); +ALTER TABLE task_archives ALTER COLUMN format_version SET DEFAULT 2; CREATE INDEX IF NOT EXISTS idx_task_archives_state ON task_archives(state, requested_at, id); CREATE INDEX IF NOT EXISTS idx_task_archives_archived ON task_archives(archived_at DESC, id DESC); CREATE INDEX IF NOT EXISTS idx_task_archives_sources ON task_archives USING GIN(source_task_ids); diff --git a/db/task_archives.go b/db/task_archives.go index c21c89d..e38a4cd 100644 --- a/db/task_archives.go +++ b/db/task_archives.go @@ -7,6 +7,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "net/url" "sort" "strconv" @@ -14,7 +15,15 @@ import ( "time" ) -const TaskArchiveFormatVersion = 1 +const ( + TaskArchiveFormatVersion = 2 + TaskArchiveLegacyFormatVersion = 1 + TaskArchiveLLMRecordsPath = "database/llm_records.ndjson" +) + +func IsTaskArchiveFormatSupported(version int) bool { + return version >= TaskArchiveLegacyFormatVersion && version <= TaskArchiveFormatVersion +} const ( ArchiveQueued = "archive_queued" @@ -103,8 +112,8 @@ type TaskArchivePage struct { } // TaskArchiveSnapshot is serialized into manifest.json inside the cold package. -// Each value in Tables is a JSON array generated by PostgreSQL row_to_json, which -// preserves exact IDs, timestamps, arrays and JSON payloads for restoration. +// Small tables remain JSON arrays in Tables. Large v2 tables are streamed to +// package files listed in StreamedTables so their size is not bounded by memory. type TaskArchiveSnapshot struct { FormatVersion int `json:"format_version"` CreatedAt time.Time `json:"created_at"` @@ -115,6 +124,7 @@ type TaskArchiveSnapshot struct { ExclusiveHosts []string `json:"exclusive_hosts"` ExclusiveAssetIDs []int64 `json:"exclusive_asset_ids"` Tables map[string]json.RawMessage `json:"tables"` + StreamedTables map[string]string `json:"streamed_tables,omitempty"` DataCounts map[string]int64 `json:"data_counts"` AggregateStats map[string]any `json:"aggregate_stats"` } @@ -279,12 +289,13 @@ LIMIT 1`, taskID).Scan(&dependent) task_description,task_goal,original_status,category_id_snapshot, category_name_snapshot,source_task_ids,remaining_timeout_seconds,requested_at) VALUES ($1,$2,'queued',0,'','[]',$3,$4,$5,$6,$7,$8,$9,$10,$11,now()) -ON CONFLICT (task_id) DO UPDATE SET - state=CASE WHEN task_archives.state IN ('archive_failed') THEN EXCLUDED.state ELSE task_archives.state END, - phase=CASE WHEN task_archives.state IN ('archive_failed') THEN 'queued' ELSE task_archives.phase END, - progress=CASE WHEN task_archives.state IN ('archive_failed') THEN 0 ELSE task_archives.progress END, - error=CASE WHEN task_archives.state IN ('archive_failed') THEN '' ELSE task_archives.error END, - requested_at=CASE WHEN task_archives.state IN ('archive_failed') THEN now() ELSE task_archives.requested_at END`, + ON CONFLICT (task_id) DO UPDATE SET + state=CASE WHEN task_archives.state IN ('archive_failed') THEN EXCLUDED.state ELSE task_archives.state END, + phase=CASE WHEN task_archives.state IN ('archive_failed') THEN 'queued' ELSE task_archives.phase END, + progress=CASE WHEN task_archives.state IN ('archive_failed') THEN 0 ELSE task_archives.progress END, + error=CASE WHEN task_archives.state IN ('archive_failed') THEN '' ELSE task_archives.error END, + format_version=CASE WHEN task_archives.state IN ('archive_failed') THEN EXCLUDED.format_version ELSE task_archives.format_version END, + requested_at=CASE WHEN task_archives.state IN ('archive_failed') THEN now() ELSE task_archives.requested_at END`, taskID, ArchiveQueued, TaskArchiveFormatVersion, name, description, goal, status, categoryID, categoryName, sources, remaining) if err != nil { @@ -341,15 +352,17 @@ WHERE id=$1 RETURNING `+taskArchiveCols, id, DeleteQueued)) return item, tx.Commit() } -// RecoverTaskArchiveJobs makes active operations retryable after an unclean -// shutdown. Idempotent restore/import semantics keep already-completed pieces safe. +// RecoverTaskArchiveJobs keeps restore/delete resumable after an unclean shutdown. +// An interrupted archive requires an explicit retry: automatic startup retries can +// otherwise form a crash loop when the prior process was killed by resource limits. func (d *DB) RecoverTaskArchiveJobs() error { _, err := d.Exec(`UPDATE task_archives SET - state=CASE state WHEN 'archiving' THEN 'archive_queued' - WHEN 'restoring' THEN 'restore_queued' - WHEN 'deleting' THEN 'delete_queued' ELSE state END, - phase='recovered', error='' -WHERE state IN ('archiving','restoring','deleting')`) + state=CASE state WHEN 'archiving' THEN 'archive_failed' + WHEN 'restoring' THEN 'restore_queued' + WHEN 'deleting' THEN 'delete_queued' ELSE state END, + phase='interrupted', + error=CASE WHEN state='archiving' THEN '上次归档进程异常退出，请手动重试' ELSE '' END + WHERE state IN ('archiving','restoring','deleting')`) return err } @@ -435,7 +448,14 @@ func queryArchiveRows(q interface { return nil, 0, err } defer rows.Close() + return encodeArchiveRows(rows) +} +func encodeArchiveRows(rows interface { + Next() bool + Scan(dest ...any) error + Err() error +}) (json.RawMessage, int64, error) { var output bytes.Buffer output.WriteByte('[') var count int64 @@ -457,6 +477,46 @@ func queryArchiveRows(q interface { return json.RawMessage(output.Bytes()), count, nil } +func writeArchiveRows(rows interface { + Next() bool + Scan(dest ...any) error + Err() error +}, writer io.Writer) (int64, error) { + var count int64 + for rows.Next() { + var raw []byte + if err := rows.Scan(&raw); err != nil { + return 0, err + } + if written, err := writer.Write(raw); err != nil { + return 0, err + } else if written != len(raw) { + return 0, io.ErrShortWrite + } + if written, err := io.WriteString(writer, "\n"); err != nil { + return 0, err + } else if written != 1 { + return 0, io.ErrShortWrite + } + count++ + } + if err := rows.Err(); err != nil { + return 0, err + } + return count, nil +} + +func streamArchiveRows(q interface { + Query(query string, args ...any) (*sql.Rows, error) +}, writer io.Writer, inner string, args ...any) (int64, error) { + rows, err := q.Query(`SELECT row_to_json(row_data)::text FROM (`+inner+`) row_data`, args...) + if err != nil { + return 0, err + } + defer rows.Close() + return writeArchiveRows(rows, writer) +} + func rawRowCount(raw json.RawMessage) int64 { var rows []json.RawMessage if json.Unmarshal(raw, &rows) != nil { @@ -480,11 +540,28 @@ UNION SELECT value::bigint FROM findings finding // writes are already quiescent at the server barrier; repeatable-read also keeps // the asset and accounting views mutually consistent during serialization. func (d *DB) SnapshotTaskArchive(taskID int64) (*TaskArchiveSnapshot, error) { + return d.snapshotTaskArchive(taskID, nil) +} + +// SnapshotTaskArchiveWithLLMRecords streams the heavyweight record history to +// llmRecords while all other task-owned data is read from the same repeatable +// PostgreSQL snapshot. +func (d *DB) SnapshotTaskArchiveWithLLMRecords(taskID int64, llmRecords io.Writer) (*TaskArchiveSnapshot, error) { + if llmRecords == nil { + return nil, errors.New("nil LLM record archive writer") + } + return d.snapshotTaskArchive(taskID, llmRecords) +} + +func (d *DB) snapshotTaskArchive(taskID int64, llmRecords io.Writer) (*TaskArchiveSnapshot, error) { tx, err := d.BeginTx(context.Background(), &sql.TxOptions{Isolation: sql.LevelRepeatableRead, ReadOnly: true}) if err != nil { return nil, err } defer tx.Rollback() //nolint:errcheck + if err := coordinateWithSchemaMigration(tx); err != nil { + return nil, err + } var expID int64 if err := tx.QueryRow(`SELECT exploration_id FROM tasks WHERE id=$1 AND deleted_at IS NULL`, taskID).Scan(&expID); err != nil { if errors.Is(err, sql.ErrNoRows) { @@ -518,7 +595,18 @@ func (d *DB) SnapshotTaskArchive(taskID int64) (*TaskArchiveSnapshot, error) { {"assets", `SELECT asset.* FROM assets asset WHERE asset.id IN (` + archiveAssetIDsQuery() + `) ORDER BY asset.id`, []any{taskID, expID}}, } counts := make(map[string]int64, len(queries)) + streamedTables := map[string]string{} for _, query := range queries { + if query.name == "llm_records" && llmRecords != nil { + count, err := streamArchiveRows(tx, llmRecords, query.query, query.args...) + if err != nil { + return nil, fmt.Errorf("snapshot %s: %w", query.name, err) + } + tables[query.name] = json.RawMessage("[]") + counts[query.name] = count + streamedTables[query.name] = TaskArchiveLLMRecordsPath + continue + } raw, count, err := queryArchiveRows(tx, query.query, query.args...) if err != nil { return nil, fmt.Errorf("snapshot %s: %w", query.name, err) @@ -555,7 +643,8 @@ func (d *DB) SnapshotTaskArchive(taskID int64) (*TaskArchiveSnapshot, error) { snapshot := &TaskArchiveSnapshot{ FormatVersion: TaskArchiveFormatVersion, CreatedAt: time.Now().UTC(), TaskID: taskID, ExplorationID: expID, SourceTaskIDs: sources, Hosts: hosts, ExclusiveHosts: exclusiveHosts, - ExclusiveAssetIDs: exclusiveAssetIDs, Tables: tables, DataCounts: counts, AggregateStats: stats, + ExclusiveAssetIDs: exclusiveAssetIDs, Tables: tables, StreamedTables: streamedTables, + DataCounts: counts, AggregateStats: stats, } return snapshot, tx.Commit() } diff --git a/db/task_archives_restore.go b/db/task_archives_restore.go index d3f15b0..71e916a 100644 --- a/db/task_archives_restore.go +++ b/db/task_archives_restore.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "strconv" "strings" "time" @@ -36,6 +37,9 @@ func (d *DB) CompleteTaskArchive( return err } defer tx.Rollback() //nolint:errcheck + if err := coordinateWithSchemaMigration(tx); err != nil { + return err + } var taskID, expID int64 var state string if err := tx.QueryRow(`SELECT archive.task_id,task.exploration_id,archive.state @@ -120,10 +124,11 @@ WHERE id=$1`, taskID); err != nil { return err } if _, err := tx.Exec(`UPDATE task_archives SET - state=$2,phase='ready',progress=100,error='',archive_path=$3,sha256=$4, - original_size=$5,compressed_size=$6,data_counts=$7,aggregate_stats=$8, - archived_at=now(),warnings='[]' -WHERE id=$1`, archiveID, ArchiveReady, archivePath, sha256, originalSize, compressedSize, string(countsRaw), string(statsRaw)); err != nil { + state=$2,phase='ready',progress=100,error='',archive_path=$3,sha256=$4, + original_size=$5,compressed_size=$6,data_counts=$7,aggregate_stats=$8, + format_version=$9,archived_at=now(),warnings='[]' +WHERE id=$1`, archiveID, ArchiveReady, archivePath, sha256, originalSize, compressedSize, + string(countsRaw), string(statsRaw), snapshot.FormatVersion); err != nil { return err } return tx.Commit() @@ -133,14 +138,49 @@ WHERE id=$1`, archiveID, ArchiveReady, archivePath, sha256, originalSize, compre // idempotent for accounting/traffic retry scenarios and returns non-fatal // warnings for global objects that intentionally are not recreated. func (d *DB) RestoreTaskArchive(archiveID int64, snapshot *TaskArchiveSnapshot, remainingTimeoutSeconds int64) ([]string, error) { - if snapshot == nil || snapshot.FormatVersion != TaskArchiveFormatVersion { + return d.restoreTaskArchive(archiveID, snapshot, remainingTimeoutSeconds, nil) +} + +// RestoreTaskArchiveWithLLMRecords restores a v2 package whose heavyweight LLM +// record history is stored as a sequence of JSON objects outside manifest.json. +func (d *DB) RestoreTaskArchiveWithLLMRecords( + archiveID int64, + snapshot *TaskArchiveSnapshot, + remainingTimeoutSeconds int64, + llmRecords io.Reader, +) ([]string, error) { + if llmRecords == nil { + return nil, errors.New("nil streamed LLM record reader") + } + return d.restoreTaskArchive(archiveID, snapshot, remainingTimeoutSeconds, llmRecords) +} + +func (d *DB) restoreTaskArchive( + archiveID int64, + snapshot *TaskArchiveSnapshot, + remainingTimeoutSeconds int64, + llmRecords io.Reader, +) ([]string, error) { + if snapshot == nil || !IsTaskArchiveFormatSupported(snapshot.FormatVersion) { return nil, ErrTaskArchiveFormatMismatch } + streamedLLMRecords := snapshot.StreamedTables["llm_records"] + if (len(snapshot.StreamedTables) > 0 && snapshot.FormatVersion < 2) || + len(snapshot.StreamedTables) > 1 || + (len(snapshot.StreamedTables) == 1 && streamedLLMRecords != TaskArchiveLLMRecordsPath) { + return nil, fmt.Errorf("%w: unsupported streamed table metadata", ErrTaskArchiveFormatMismatch) + } + if streamedLLMRecords != "" && llmRecords == nil { + return nil, fmt.Errorf("%w: streamed LLM records are missing", ErrTaskArchiveFormatMismatch) + } tx, err := d.Begin() if err != nil { return nil, err } defer tx.Rollback() //nolint:errcheck + if err := coordinateWithSchemaMigration(tx); err != nil { + return nil, err + } var taskID, expID int64 var state string if err := tx.QueryRow(`SELECT archive.task_id,task.exploration_id,archive.state @@ -215,7 +255,23 @@ WHERE archive.id=$1 FOR UPDATE OF archive,task`, archiveID).Scan(&taskID, &expID } else { warnings = append(warnings, warning...) } - for _, table := range []string{"task_asset_links", "findings", "llm_records", "llm_usage", "skill_usage", "tool_usage"} { + for _, table := range []string{"task_asset_links", "findings"} { + if err := insertArchiveRows(tx, table, remappedTables[table]); err != nil { + return nil, fmt.Errorf("restore %s: %w", table, err) + } + } + if streamedLLMRecords != "" { + count, err := insertArchiveJSONSequenceRows(tx, "llm_records", llmRecords) + if err != nil { + return nil, fmt.Errorf("restore llm_records: %w", err) + } + if expected := snapshot.DataCounts["llm_records"]; count != expected { + return nil, fmt.Errorf("restore llm_records: row count %d does not match manifest %d", count, expected) + } + } else if err := insertArchiveRows(tx, "llm_records", remappedTables["llm_records"]); err != nil { + return nil, fmt.Errorf("restore llm_records: %w", err) + } + for _, table := range []string{"llm_usage", "skill_usage", "tool_usage"} { if err := insertArchiveRows(tx, table, remappedTables[table]); err != nil { return nil, fmt.Errorf("restore %s: %w", table, err) } @@ -306,6 +362,36 @@ func insertArchiveRows(tx *sql.Tx, table string, raw json.RawMessage) error { return err } +func insertArchiveJSONSequenceRows(tx *sql.Tx, table string, reader io.Reader) (int64, error) { + if table != "llm_records" { + return 0, fmt.Errorf("archive streamed restore table %q is not allowed", table) + } + statement, err := tx.Prepare(`INSERT INTO llm_records SELECT * FROM json_populate_record(NULL::llm_records,$1::json)`) + if err != nil { + return 0, err + } + defer statement.Close() + decoder := json.NewDecoder(reader) + var count int64 + for { + var raw json.RawMessage + if err := decoder.Decode(&raw); errors.Is(err, io.EOF) { + break + } else if err != nil { + return 0, err + } + trimmed := bytes.TrimSpace(raw) + if len(trimmed) == 0 || trimmed[0] != '{' { + return 0, errors.New("streamed archive row must be a JSON object") + } + if _, err := statement.Exec(string(trimmed)); err != nil { + return 0, err + } + count++ + } + return count, nil +} + func restoreArchiveAssets(tx *sql.Tx, taskID int64, raw json.RawMessage) (map[int64]int64, []string, error) { rows, err := decodeArchiveRows(raw) if err != nil { @@ -598,6 +684,9 @@ func (d *DB) DeleteTaskArchiveStub(archiveID int64) error { return err } defer tx.Rollback() //nolint:errcheck + if err := coordinateWithSchemaMigration(tx); err != nil { + return err + } var taskID, expID int64 var state string if err := tx.QueryRow(`SELECT archive.task_id,task.exploration_id,archive.state diff --git a/db/task_archives_test.go b/db/task_archives_test.go index c9a6d18..498fa97 100644 --- a/db/task_archives_test.go +++ b/db/task_archives_test.go @@ -1,24 +1,55 @@ package db import ( + "bytes" "encoding/json" + "errors" "fmt" "strings" "testing" "time" ) -func TestQueryArchiveRowsStreamsJSON(t *testing.T) { - d, err := Open(testDSN(t)) - if err != nil { - t.Skipf("postgres unavailable (%v) — skipping", err) - } - defer d.Close() +type archiveJSONTestRows struct { + values [][]byte + next int +} +func (r *archiveJSONTestRows) Next() bool { + if r.next >= len(r.values) { + return false + } + r.next++ + return true +} + +func (r *archiveJSONTestRows) Scan(dest ...any) error { + if len(dest) != 1 || r.next == 0 || r.next > len(r.values) { + return fmt.Errorf("invalid archive test row scan") + } + target, ok := dest[0].(*[]byte) + if !ok { + return fmt.Errorf("archive test row destination is %T", dest[0]) + } + *target = append((*target)[:0], r.values[r.next-1]...) + return nil +} + +func (r *archiveJSONTestRows) Err() error { return nil } + +func TestQueryArchiveRowsStreamsJSON(t *testing.T) { payload := strings.Repeat("large request/response payload ", 64*1024) - raw, count, err := queryArchiveRows(d, `SELECT * FROM (VALUES -($1::bigint,$2::text),($3::bigint,$4::text)) archived_row(id,body) ORDER BY id`, - int64(1), payload, int64(2), "quoted: \"value\"\nline") + values := make([][]byte, 2) + var err error + values[0], err = json.Marshal(map[string]any{"id": int64(1), "body": payload}) + if err != nil { + t.Fatal(err) + } + values[1], err = json.Marshal(map[string]any{"id": int64(2), "body": "quoted: \"value\"\nline"}) + if err != nil { + t.Fatal(err) + } + raw, count, err := encodeArchiveRows(&archiveJSONTestRows{values: values}) if err != nil { t.Fatal(err) } @@ -40,6 +71,55 @@ func TestQueryArchiveRowsStreamsJSON(t *testing.T) { } } +func TestWriteArchiveRowsProducesJSONSequence(t *testing.T) { + values := [][]byte{ + json.RawMessage(`{"id":1,"body":"first"}`), + json.RawMessage(`{"id":2,"body":"second\\nline"}`), + } + var output bytes.Buffer + count, err := writeArchiveRows(&archiveJSONTestRows{values: values}, &output) + if err != nil { + t.Fatal(err) + } + if count != 2 { + t.Fatalf("streamed row count=%d, want 2", count) + } + decoder := json.NewDecoder(&output) + for wantID := int64(1); wantID <= 2; wantID++ { + var row struct { + ID int64 `json:"id"` + } + if err := decoder.Decode(&row); err != nil { + t.Fatal(err) + } + if row.ID != wantID { + t.Fatalf("streamed row id=%d, want %d", row.ID, wantID) + } + } +} + +func TestTaskArchiveFormatCompatibility(t *testing.T) { + for _, version := range []int{TaskArchiveLegacyFormatVersion, TaskArchiveFormatVersion} { + if !IsTaskArchiveFormatSupported(version) { + t.Fatalf("archive format %d should be supported", version) + } + } + for _, version := range []int{0, TaskArchiveFormatVersion + 1} { + if IsTaskArchiveFormatSupported(version) { + t.Fatalf("archive format %d should be rejected", version) + } + } + invalidSnapshots := []*TaskArchiveSnapshot{ + {FormatVersion: TaskArchiveLegacyFormatVersion, StreamedTables: map[string]string{"llm_records": TaskArchiveLLMRecordsPath}}, + {FormatVersion: TaskArchiveFormatVersion, StreamedTables: map[string]string{"unknown": "database/unknown.ndjson"}}, + } + for _, snapshot := range invalidSnapshots { + if _, err := (&DB{}).RestoreTaskArchive(1, snapshot, 0); !errors.Is(err, ErrTaskArchiveFormatMismatch) { + t.Fatalf("invalid streamed table metadata returned %v", err) + } + } +} + func TestTaskArchiveDatabaseRoundTrip(t *testing.T) { d, err := Open(testDSN(t)) if err != nil { @@ -111,6 +191,12 @@ VALUES($1,$2,0,'quota_exhausted','balance exhausted',$3,$4,$3)`, task.ID, llmPro if err := d.InsertToolUsage(&ToolUsage{ToolKey: toolName, AgentKey: "worker", TaskID: task.ID, ExplorationID: task.ExplorationID}); err != nil { t.Fatal(err) } + if err := d.InsertLLMRecord(&LLMRecord{ + TaskID: fmt.Sprint(task.ID), Model: "archive-model", SessionID: "archive-session", + Status: "ok", RawRequest: strings.Repeat("request", 1024), RawResponse: strings.Repeat("response", 1024), + }); err != nil { + t.Fatal(err) + } if _, err := d.AddFinding(task.ID, nodeID, vulnclass, "archive finding", SeverityCritical, "summary", "evidence", "worker", []int64{assetID}); err != nil { t.Fatal(err) } @@ -122,10 +208,17 @@ VALUES($1,$2,0,'quota_exhausted','balance exhausted',$3,$4,$3)`, task.ID, llmPro if err != nil || claimed == nil || claimed.ID != archive.ID || claimed.State != Archiving { t.Fatalf("claim = %+v, %v", claimed, err) } - snapshot, err := d.SnapshotTaskArchive(task.ID) + var llmRecords bytes.Buffer + snapshot, err := d.SnapshotTaskArchiveWithLLMRecords(task.ID, &llmRecords) if err != nil { t.Fatal(err) } + if snapshot.StreamedTables["llm_records"] != TaskArchiveLLMRecordsPath || snapshot.DataCounts["llm_records"] != 1 { + t.Fatalf("unexpected streamed LLM metadata: paths=%v counts=%v", snapshot.StreamedTables, snapshot.DataCounts) + } + if rawRowCount(snapshot.Tables["llm_records"]) != 0 { + t.Fatal("streamed LLM records were also retained in manifest memory") + } if snapshot.DataCounts["assets"] != 1 || snapshot.DataCounts["exploration_nodes"] < 2 { t.Fatalf("unexpected snapshot counts: %#v", snapshot.DataCounts) } @@ -170,7 +263,9 @@ VALUES($1,$2,0,'quota_exhausted','balance exhausted',$3,$4,$3)`, task.ID, llmPro if err != nil || claimed == nil || claimed.State != Restoring { t.Fatalf("restore claim = %+v, %v", claimed, err) } - warnings, err := d.RestoreTaskArchive(archive.ID, snapshot, int64((10 * time.Minute).Seconds())) + warnings, err := d.RestoreTaskArchiveWithLLMRecords( + archive.ID, snapshot, int64((10 * time.Minute).Seconds()), bytes.NewReader(llmRecords.Bytes()), + ) if err != nil { t.Fatal(err) } @@ -202,6 +297,14 @@ VALUES($1,$2,0,'quota_exhausted','balance exhausted',$3,$4,$3)`, task.ID, llmPro if nodes < 2 || assets != 1 || usage != 1 { t.Fatalf("restore incomplete nodes=%d assets=%d usage=%d", nodes, assets, usage) } + var restoredRawRequest, restoredRawResponse string + if err := d.QueryRow(`SELECT COALESCE(raw_request,''),COALESCE(raw_response,'') FROM llm_records WHERE task_id=$1`, fmt.Sprint(task.ID)). + Scan(&restoredRawRequest, &restoredRawResponse); err != nil { + t.Fatal(err) + } + if restoredRawRequest != strings.Repeat("request", 1024) || restoredRawResponse != strings.Repeat("response", 1024) { + t.Fatal("restored streamed LLM record body was truncated") + } var restoredCompanyID *int64 var restoredStatus, restoredError string var restoredExhaustedAt, restoredCreatedAt time.Time @@ -274,6 +377,42 @@ func TestTaskArchiveBlockersIgnoreQueuedDependents(t *testing.T) { } } +func TestRecoverInterruptedArchiveRequiresManualRetry(t *testing.T) { + d, err := Open(testDSN(t)) + if err != nil { + t.Skipf("postgres unavailable (%v) — skipping", err) + } + defer d.Close() + task, err := d.CreateTaskWithOptions("interrupted archive", "must not restart automatically", TaskCreateOptions{}) + if err != nil { + t.Fatal(err) + } + defer func() { + _, _ = d.Exec(`DELETE FROM task_archives WHERE task_id=$1`, task.ID) + _ = d.DeleteTask(task.ID) + }() + if err := d.SetPaused(task.ID, true); err != nil { + t.Fatal(err) + } + queued, err := d.QueueTaskArchive(task.ID) + if err != nil { + t.Fatal(err) + } + if _, err := d.Exec(`UPDATE task_archives SET state=$2,phase='snapshot_database',progress=10 WHERE id=$1`, queued.ID, Archiving); err != nil { + t.Fatal(err) + } + if err := d.RecoverTaskArchiveJobs(); err != nil { + t.Fatal(err) + } + recovered, err := d.GetTaskArchive(queued.ID) + if err != nil { + t.Fatal(err) + } + if recovered == nil || recovered.State != ArchiveFailed || recovered.Phase != "interrupted" || recovered.Error == "" { + t.Fatalf("recovered archive=%+v, want explicit-retry failure", recovered) + } +} + func assertArchiveGlobalStats(t *testing.T, d *DB, profileName, skillName, toolName, vulnclass string) { t.Helper() profiles, err := d.UsageByProfile() diff --git a/server/task_archive_package.go b/server/task_archive_package.go index ce4494d..bc55617 100644 --- a/server/task_archive_package.go +++ b/server/task_archive_package.go @@ -480,11 +480,20 @@ func writeTaskArchivePackage(path, payloadDir string, snapshot *pgdb.TaskArchive if err = os.MkdirAll(filepath.Dir(path), archiveDirMode); err != nil { return 0, 0, "", err } - manifest, err := json.Marshal(snapshot) + manifestFile, err := os.OpenFile( + filepath.Join(payloadDir, "manifest.json"), + os.O_CREATE|os.O_TRUNC|os.O_WRONLY, + archiveFileMode, + ) if err != nil { return 0, 0, "", err } - if err = os.WriteFile(filepath.Join(payloadDir, "manifest.json"), manifest, archiveFileMode); err != nil { + encodeErr := json.NewEncoder(manifestFile).Encode(snapshot) + var syncErr error + if encodeErr == nil { + syncErr = manifestFile.Sync() + } + if err := errors.Join(encodeErr, syncErr, manifestFile.Close()); err != nil { return 0, 0, "", err } temporary := path + ".partial" diff --git a/server/task_archive_package_test.go b/server/task_archive_package_test.go index 74e86ad..9846673 100644 --- a/server/task_archive_package_test.go +++ b/server/task_archive_package_test.go @@ -47,7 +47,18 @@ func TestTaskArchivePackageFilesRoundTrip(t *testing.T) { } archivePath := taskArchivePath(dataDir, 1, taskID) - snapshot := &pgdb.TaskArchiveSnapshot{FormatVersion: pgdb.TaskArchiveFormatVersion, TaskID: 42, ExplorationID: explorationID} + streamPath := filepath.Join(stage.payload, filepath.FromSlash(pgdb.TaskArchiveLLMRecordsPath)) + if err := os.MkdirAll(filepath.Dir(streamPath), archiveDirMode); err != nil { + t.Fatal(err) + } + streamBody := []byte("{\"id\":1,\"raw_request\":\"large\"}\n") + if err := os.WriteFile(streamPath, streamBody, archiveFileMode); err != nil { + t.Fatal(err) + } + snapshot := &pgdb.TaskArchiveSnapshot{ + FormatVersion: pgdb.TaskArchiveFormatVersion, TaskID: 42, ExplorationID: explorationID, + StreamedTables: map[string]string{"llm_records": pgdb.TaskArchiveLLMRecordsPath}, + } original, compressed, checksum, err := writeTaskArchivePackage(archivePath, stage.payload, snapshot) if err != nil { t.Fatal(err) @@ -62,6 +73,9 @@ func TestTaskArchivePackageFilesRoundTrip(t *testing.T) { if err := extractTaskArchivePackage(archivePath, checksum, extracted); err != nil { t.Fatal(err) } + if got, err := os.ReadFile(filepath.Join(extracted, filepath.FromSlash(pgdb.TaskArchiveLLMRecordsPath))); err != nil || !bytes.Equal(got, streamBody) { + t.Fatalf("streamed LLM archive payload=%q err=%v", got, err) + } installed, err := installTaskArchiveFiles(dataDir, extracted, taskID, 1) if err != nil { t.Fatal(err) diff --git a/server/task_archives.go b/server/task_archives.go index c953a1a..30cea69 100644 --- a/server/task_archives.go +++ b/server/task_archives.go @@ -131,13 +131,15 @@ func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) { if err := s.waitTaskQuiescent(drainCtx, taskID); err != nil { return errors.New("任务仍有运行中的 Agent，请先暂停后重试归档") } - _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "snapshot_database", 10) - snapshot, err := s.m.pg.SnapshotTaskArchive(job.TaskID) + task, err := s.m.pg.GetTask(job.TaskID) if err != nil { return err } - _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "stage_files", 25) - fileStage, err := stageTaskArchiveFiles(s.m.dir, job.ID, taskID, snapshot.ExplorationID) + if task == nil { + return pgdb.ErrTaskArchiveNotFound + } + _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "stage_files", 10) + fileStage, err := stageTaskArchiveFiles(s.m.dir, job.ID, taskID, task.ExplorationID) if err != nil { return err } @@ -146,6 +148,27 @@ func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) { runErr = errors.Join(runErr, fileStage.rollback()) } }() + streamPath := filepath.Join(fileStage.payload, filepath.FromSlash(pgdb.TaskArchiveLLMRecordsPath)) + if err := os.MkdirAll(filepath.Dir(streamPath), archiveDirMode); err != nil { + return err + } + streamFile, err := os.OpenFile(streamPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, archiveFileMode) + if err != nil { + return err + } + _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "snapshot_database", 20) + snapshot, snapshotErr := s.m.pg.SnapshotTaskArchiveWithLLMRecords(job.TaskID, streamFile) + var syncErr error + if snapshotErr == nil { + syncErr = streamFile.Sync() + } + closeErr := streamFile.Close() + if err := errors.Join(snapshotErr, syncErr, closeErr); err != nil { + return err + } + if snapshot.ExplorationID != task.ExplorationID { + return errors.New("任务探索记录在归档快照期间发生变化") + } if s.m.traffic != nil && len(snapshot.Hosts) > 0 { _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "snapshot_traffic", 38) trafficCount, err := s.m.traffic.ExportHosts(snapshot.Hosts, filepath.Join(fileStage.payload, "traffic")) @@ -244,9 +267,27 @@ func (s *Server) restoreTaskArchive(job *pgdb.TaskArchive) (runErr error) { if err := json.Unmarshal(manifestRaw, &snapshot); err != nil { return err } - if snapshot.TaskID != job.TaskID || snapshot.FormatVersion != pgdb.TaskArchiveFormatVersion { + if snapshot.TaskID != job.TaskID || !pgdb.IsTaskArchiveFormatSupported(snapshot.FormatVersion) { return pgdb.ErrTaskArchiveFormatMismatch } + var llmRecordsFile *os.File + relative, hasStreamedLLMRecords := snapshot.StreamedTables["llm_records"] + if len(snapshot.StreamedTables) > 0 && + (snapshot.FormatVersion < 2 || !hasStreamedLLMRecords || len(snapshot.StreamedTables) != 1 || + relative != pgdb.TaskArchiveLLMRecordsPath) { + return pgdb.ErrTaskArchiveFormatMismatch + } + if hasStreamedLLMRecords { + llmRecordsFile, err = os.Open(filepath.Join(extracted, filepath.FromSlash(relative))) + if err != nil { + return err + } + defer func() { + if llmRecordsFile != nil { + _ = llmRecordsFile.Close() + } + }() + } if s.m.traffic != nil { _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "restore_traffic", 35) if _, err := s.m.traffic.ImportArchive(filepath.Join(extracted, "traffic")); err != nil { @@ -265,7 +306,17 @@ func (s *Server) restoreTaskArchive(job *pgdb.TaskArchive) (runErr error) { } }() _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "restore_database", 70) - warnings, err := s.m.pg.RestoreTaskArchive(job.ID, &snapshot, job.RemainingTimeoutSeconds) + var warnings []string + if llmRecordsFile != nil { + warnings, err = s.m.pg.RestoreTaskArchiveWithLLMRecords( + job.ID, &snapshot, job.RemainingTimeoutSeconds, llmRecordsFile, + ) + closeErr := llmRecordsFile.Close() + llmRecordsFile = nil + err = errors.Join(err, closeErr) + } else { + warnings, err = s.m.pg.RestoreTaskArchive(job.ID, &snapshot, job.RemainingTimeoutSeconds) + } if err != nil { return err } From 21656cc2e31000803782bd532836f7b463bd304c Mon Sep 17 00:00:00 2001 From: Autumn-27 <53844290+Autumn-27@users.noreply.github.com> Date: Sat, 29 Aug 2026 01:37:46 -0400 Subject: [PATCH 18/23] fix: restore changes dropped by the upstream sync merge The manual v0.3.6 sync (22132a4) was incomplete, and the follow-up merge (d803ae2) resolved conflicts in favour of that copy, silently reverting three shipped changes. main is already an ancestor of this branch, so re-merging cannot bring them back; restore them explicitly. - schema.sql: re-add the idempotent llm_profiles_format_check migration. CREATE TABLE IF NOT EXISTS leaves an existing table alone, so databases created before openai-responses kept the old two-value CHECK and rejected every openai-responses profile. - orchestration.go: re-add the reseedPlannerPrompt/reseedWorkerPrompt calls, dropped by a comment-alignment hunk while the functions stayed behind as dead code. Without them existing installs never receive the quantified acceptance check or the negative-conclusion evidence threshold. - CHANGELOG.md: restore the 0.3.6 entries, the @neouks credit and the correct norma version range. Also close three recovery gaps in the new cold-archive path: - traffic: write the staging journal even when no legacy host directory exists. Modern installs keep traffic in SQLite only, so a crash between the PostgreSQL commit and the SQLite commit left the archived task's exclusive traffic in the hot store with nothing for recovery to act on. - recoverTaskArchiveStages: treat a missing journal.json as a discardable directory instead of an error, which used to set recoveryFailed and stop the archive worker from starting for the rest of the process lifetime. - stageTaskArchiveFiles: replay the journal's renames when resuming an existing staging root. The journal is written before the first rename, so its presence did not guarantee a complete payload, and a resumed round could package an archive missing transcripts or workspace files. --- CHANGELOG.md | 39 +++++++++++++++++++- db/schema.sql | 5 +++ server/orchestration.go | 2 ++ server/task_archive_package.go | 65 ++++++++++++++++++++++++++++------ traffic/traffic.go | 9 +++-- 5 files changed, 107 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d96ad57..b6e5616 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,7 +29,7 @@ #### 修改的功能 -- 升级 norma v0.2.9 → v0.3.2：修复 OpenAI 系响应里被静默丢弃的 reasoning/refusal（Responses 补 `reasoning_text`）、reasoning 字段三名去重、空响应（零内容块）有界重试、压缩边界切断工具配对导致的网关 400（同时修复已烤进 transcript 的存量孤儿）。 +- 升级 norma v0.3.1 → v0.3.2：修复 OpenAI 系响应里被静默丢弃的 reasoning/refusal（Responses 补 `reasoning_text`）、reasoning 字段三名去重、空响应（零内容块）有界重试、压缩边界切断工具配对导致的网关 400（同时修复已烤进 transcript 的存量孤儿）。 ### 贡献者 @@ -42,10 +42,47 @@ #### 新增的功能 - 每个 LLM 配置支持流式/非流式切换（默认流式）：开走流式 SSE；关走真·非流式（`stream:false`、一次性返回完整 JSON），可绕开部分网关糟糕的 SSE 实现（空帧、思考字段丢帧），代价是失去运行中的实时进度与实时 Token 计数。worker / planner / mainagent / chat / goals 均按当前激活配置动态取值；`llmpool` / `llmrec` / 任务运行时三层 Provider 包装均兼容非流式；`llm_profiles` 新增 `streaming` 列并 `ALTER` 补旧库（默认 `true`，旧配置无感）。 +- 支持 OpenAI Responses API 格式的 LLM 配置：每个配置新增第三种格式 `openai-responses`（打 `POST /v1/responses`），与 Chat Completions / Anthropic 并列；`BaseURL` 归一化、默认模型 `gpt-5`；`llm_profiles` 的 `format` 约束加入 `openai-responses` 并幂等迁移补旧库；前端格式下拉新增「OpenAI (Responses API)」。依赖升级 norma v0.3.1（含 `reasoning_content` 回传修复）。 +- 录制 LLM 请求/响应 HTTP 原文：在 HTTP transport 层捕获真实 wire body，保留归一化视图看不到的工具 schema、`tool_use` 块与原始 SSE 帧；norma 内部重试的多次尝试逐次保留；`llm_records` 新增 `raw_request` / `raw_response` 两列并 `ALTER` 补旧库；录制页详情面板增加「原文」视图切换与请求/响应复制按钮（兼容非安全上下文的 `execCommand` 回退）。 + +### 对话 + +#### 修改的功能 + +- 会话列表多选改为「多选」模式开关：默认列表不再常驻每行勾选框（观感更干净），表头改为「共 N 个 + 多选」按钮；点「多选」进入选择模式（勾选框、全选、批量删除），「完成」退出并清空选择，批量删除全部成功后自动退出（有失败则留在选择模式便于重试）；单条重命名 / 置顶 / 删除仍走每行 ⋯ 菜单。 + +### UI + +#### 修改的功能 + +- 任务管理、会话操作与流量查看体验增强（#57）：任务列表列排序偏好可持久化记忆、行内重命名改用图标直接触发（不再经菜单）、抽屉（sheet）交互与流量查看细节优化。 + +#### 修复的问题 + +- Worker 资产标签只显示域名与 IP，并正确处理空标签的场景。 + +### Agent + +#### 新增的功能 + +- planner 判目标时新增「量化验收核对」：目标含可量化验收条件（资产测试覆盖度达到 X%、拿到 N 个 flag、获得某权限）时，`prove_goal` 前必须核对 `graph_overview` 的实测值（`coverage.pct`、findings 计数等），实测未达标则禁止 `prove_goal`、改为派意图补足差距，不得以「主要部分已完成/大体达成」为由提前标 met。修复了目标要求覆盖度 100%、实测仅 40% 却被判完成的问题。 + +#### 修改的功能 + +- 重写 planner 提示词中「本轮 0 意图」的判定依据：原文将 0 意图描述为「最常见、最重要的原则」，会让规划者在目标未达成、范围内仍有未测面时过早收手。现改为只在两种具体情况下才应产出 0 意图——① 想到的方向都已被 open/running/recent_done 意图覆盖；② 下一步依赖当前正在运行的 work 的产出、而产出尚未出现（应等其跑完、图更新后的下一次唤醒再规划）。并补充反向约束：确有未被覆盖且不依赖在跑 work 的新方向、或目标未达成且仍有未测面时，不要因「0 意图常见」而收手。 +- 强化 worker 提示词中「否定结论的证据门槛」：对「不可注入/端口关闭/无登录入口」等可能让规划者放弃一整条方向的否定结论，要求下结论前先穷尽该意图内的合理手段（换编码/参数/路径/方法）；手段未走完或证据偏弱一律标 `confidence=inferred`，避免用轻率的 `observed` 否定把整条路线焊死（任务早期的错误否定尤其会把方向带偏且难以自愈）。 +- 上述 planner / worker 默认提示词变更通过一次性迁移（`reseedPlannerPrompt` / `reseedWorkerPrompt`，各带 settings flag 守卫）追加为新版本并切为当前版本；旧版本保留在版本历史中，自定义过提示词的用户可在版本记录里恢复。 + +### 运维 + +#### 新增的功能 + +- 新增 `reset-password.sh` 重置管理员（用户名固定 `ARTEX`）密码：支持 local / docker 两种部署，连接信息可显式指定或自动从 `--dsn`/`$ARTEX_PG_DSN`/`config.json` 读取；在库内用 `pgcrypto` 生成与后端登录兼容的 bcrypt 哈希并写回 `settings.auth.password_hash`，重置后无需重启服务。密码经环境变量传入、不进入进程 argv，并做转义防注入。 ### 贡献者 - [@Autumn-27](https://github.com/Autumn-27) +- [@neouks](https://github.com/neouks) ## [0.3.4] - 2026-08-24 diff --git a/db/schema.sql b/db/schema.sql index 88deae6..fe6924e 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -324,6 +324,11 @@ ALTER TABLE llm_profiles ADD COLUMN IF NOT EXISTS priority INTEGER NOT NULL ALTER TABLE llm_profiles ADD COLUMN IF NOT EXISTS pool_exclude BOOLEAN NOT NULL DEFAULT false; -- 流式开关；补旧库。默认 true = 保持既有的流式行为，旧配置无感升级。 ALTER TABLE llm_profiles ADD COLUMN IF NOT EXISTS streaming BOOLEAN NOT NULL DEFAULT true; +-- 放开 format 约束以容纳 openai-responses(OpenAI Responses API)；补旧库。 +-- 每次启动执行,幂等:先删旧 CHECK 再建含三值的新 CHECK。 +ALTER TABLE llm_profiles DROP CONSTRAINT IF EXISTS llm_profiles_format_check; +ALTER TABLE llm_profiles ADD CONSTRAINT llm_profiles_format_check + CHECK (format IN ('openai','anthropic','openai-responses')); -- 思考开关字段 thinking_type，从旧的单一 reasoning_effort 语义一次性拆分而来。 -- schema.sql 每次启动都执行，故迁移必须只跑一次：仅当该列尚不存在时才回填， diff --git a/server/orchestration.go b/server/orchestration.go index 4288d08..191bde2 100644 --- a/server/orchestration.go +++ b/server/orchestration.go @@ -452,6 +452,8 @@ func (s *Server) seedOrchestrationTools() { s.unbindGoalMetDefault() s.reseedGoalsPrompt() // goals 提示词加入「抽操作约束」步 → 旧库追加一版新默认(一次性) s.reseedMainAgentPrompt() // mainagent 提示词加入「目标达成后 add_intent 反问是否建目标」(一次性) + s.reseedPlannerPrompt() // planner 提示词:重写「0 意图」正当理由 + 加量化验收核对(一次性) + s.reseedWorkerPrompt() // worker 提示词:加否定结论证据门槛(一次性) s.seedReporterAgent() // 预置「报告撰写」agent + 工具绑定 + finding 触发器(一次性) // 注：pentest 的默认工具绑定无需迁移——BuiltinToolSeeds 在全新初始化时就把 // list_assets/insert_assets/report_finding/list_findings/list_companies 连同 diff --git a/server/task_archive_package.go b/server/task_archive_package.go index bc55617..a07631f 100644 --- a/server/task_archive_package.go +++ b/server/task_archive_package.go @@ -96,6 +96,17 @@ func stageTaskArchiveFiles(dataDir string, archiveID int64, taskID string, explo if err := json.Unmarshal(raw, &stage.journal); err != nil { return nil, fmt.Errorf("read task archive staging journal: %w", err) } + // The journal is written before the first rename, so its presence does not mean + // the payload is complete: a previous round may have failed mid-loop with a + // rollback that itself errored, which leaves the root in place. Replay the moves + // rather than packaging a payload that is missing transcripts or workspace files. + if err := os.MkdirAll(filepath.Join(stage.payload, "files"), archiveDirMode); err != nil { + return nil, err + } + if err := stage.applyMoves(); err != nil { + _ = stage.rollback() + return nil, err + } return stage, nil } else if !os.IsNotExist(err) { return nil, err @@ -133,20 +144,42 @@ func stageTaskArchiveFiles(dataDir string, archiveID int64, taskID string, explo _ = stage.rollback() return nil, err } - for _, move := range targets { - destination := filepath.Join(stage.payload, move.Relative) - if err := os.MkdirAll(filepath.Dir(destination), archiveDirMode); err != nil { - _ = stage.rollback() - return nil, err - } - if err := os.Rename(move.Source, destination); err != nil { - _ = stage.rollback() - return nil, fmt.Errorf("stage task archive path %s: %w", move.Source, err) - } + if err := stage.applyMoves(); err != nil { + _ = stage.rollback() + return nil, err } return stage, nil } +// applyMoves performs the renames recorded in the journal. Entries already sitting +// in the payload are skipped, so an interrupted staging round can be resumed in +// place without moving anything twice. +func (s *taskArchiveFileStage) applyMoves() error { + for _, move := range s.journal.Moves { + destination := filepath.Join(s.payload, move.Relative) + if _, err := os.Lstat(destination); err == nil { + continue + } else if !os.IsNotExist(err) { + return err + } + if _, err := os.Lstat(move.Source); err != nil { + if os.IsNotExist(err) { + // Neither side exists: the path was removed outside the archive flow + // after the journal was written. Nothing can be staged for it. + continue + } + return err + } + if err := os.MkdirAll(filepath.Dir(destination), archiveDirMode); err != nil { + return err + } + if err := os.Rename(move.Source, destination); err != nil { + return fmt.Errorf("stage task archive path %s: %w", move.Source, err) + } + } + return nil +} + func (s *taskArchiveFileStage) rollback() error { if s == nil || s.done { return nil @@ -368,6 +401,18 @@ func recoverTaskArchiveStages(dataDir string, pg *pgdb.DB) error { } root := filepath.Join(parent, entry.Name()) raw, err := os.ReadFile(filepath.Join(root, "journal.json")) + if os.IsNotExist(err) { + // Either staging died between creating the directory tree and writing the + // journal — no rename had run, so payload holds nothing — or commit/rollback + // failed part-way through removing the root, in which case payload only holds + // copies already inside the package. Without a journal there is nothing to + // roll back, and reporting an error here keeps the archive worker from ever + // starting, so discard the directory instead. + if err := os.RemoveAll(root); err != nil { + errs = append(errs, err) + } + continue + } if err != nil { errs = append(errs, err) continue diff --git a/traffic/traffic.go b/traffic/traffic.go index 6dfbce3..d7a2b7f 100644 --- a/traffic/traffic.go +++ b/traffic/traffic.go @@ -911,7 +911,12 @@ func (t *Traffic) stageTreesForArchive(dirs, hosts []string, archiveID, taskID i } planned = append(planned, stagedTrafficPath{source: source}) } - if len(planned) == 0 { + // 归档路径即使一个历史 host 目录都没有(新装机的流量只落在 SQLite + _blobs) + // 也必须留下 journal：崩溃点若落在 PostgreSQL 提交与 SQLite 提交之间，重启后 + // SQLite 事务被回滚，只有这份 journal 能让恢复流程补做 host 行的删除。少了它， + // 已转冷任务的独占流量会永久留在热库里。 + uniqueHosts := uniqueArchiveHosts(hosts) + if len(planned) == 0 && len(uniqueHosts) == 0 { return "", nil, nil } parent := filepath.Join(t.dir, "_delete_staging") @@ -921,7 +926,7 @@ func (t *Traffic) stageTreesForArchive(dirs, hosts []string, archiveID, taskID i if stageDir, err = os.MkdirTemp(parent, "hosts-"); err != nil { return "", nil, fmt.Errorf("创建流量暂存目录: %w", err) } - journal := hostDeleteStageJournal{Version: 1, ArchiveID: archiveID, TaskID: taskID, Hosts: uniqueArchiveHosts(hosts)} + journal := hostDeleteStageJournal{Version: 1, ArchiveID: archiveID, TaskID: taskID, Hosts: uniqueHosts} for i := range planned { planned[i].staged = filepath.Join(stageDir, fmt.Sprintf("%d-%s", i, filepath.Base(planned[i].source))) journal.Moves = append(journal.Moves, hostDeleteStageMove{Source: planned[i].source, Staged: planned[i].staged}) From 9534e5e8a80a8e5a15c8edb8f48d605bdc5c370d Mon Sep 17 00:00:00 2001 From: Autumn-27 <53844290+Autumn-27@users.noreply.github.com> Date: Sat, 29 Aug 2026 02:19:48 -0400 Subject: [PATCH 19/23] feat(demo): enrich planner rounds in the Vercel mock dataset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The demo's Plan session showed one line per round with no visible reasoning, no cross-reference to worker output, and no intent-generation tool calls, so it looked hollow and unrealistic. Rebuild all five planner rounds as the full decision chain a real planner emits: read the situation, pull the relevant worker facts for detail (node_detail / list_findings), judge goals with prove_goal (including the quantified-acceptance check that refuses to mark g2 met on a foothold alone), and derive new directions through add_intent with asset_ids / parent_ids / priority whose results reference the intent ids already in the graph. Reasoning now cites the worker facts (fa1–fa10) by id, so the plan is tied to what the workers actually found. Also renumber the planner activity seq into a single monotonic range so the rounds render in chronological order; previously round 4 (seq 4–5) sorted ahead of rounds 1–3 (seq 300+). Only the mock dataset changes; this affects the NEXT_PUBLIC_MOCK demo build only, no backend behaviour. --- web/src/lib/mock/data.ts | 321 +++++++++++++++++++++++++++++++++++---- 1 file changed, 294 insertions(+), 27 deletions(-) diff --git a/web/src/lib/mock/data.ts b/web/src/lib/mock/data.ts index 192d92d..b3f64c1 100644 --- a/web/src/lib/mock/data.ts +++ b/web/src/lib/mock/data.ts @@ -1263,63 +1263,330 @@ export const frontier: TaskNode[] = explorationGraph.nodes.filter( // intent_id 的活动流。planner=worker "planner"、主 agent=worker "mainagent"。 // 每个 worker 会话含完整工具执行链（tool_use → tool_result → …→ result）。 export const activity: Activity[] = [ - // planner - { seq: 1, worker: "planner", ts: T("2026-07-24T09:12:10Z"), kind: "round", summary: "第 0 轮 · 目标拆解" }, + // planner —— 每轮完整决策链：读态势 → 拉 worker 产出对照 → 判目标 → add_intent 派方向 + // ── 第 0 轮 · 目标拆解与首批意图（2026-07-24 09:12）── + { seq: 1000, worker: "planner", ts: T("2026-07-24T09:12:10Z"), kind: "round", summary: "第 0 轮 · 目标拆解" }, { - seq: 2, + seq: 1001, worker: "planner", - ts: T("2026-07-24T09:12:30Z"), + ts: T("2026-07-24T09:12:25Z"), kind: "thinking", - summary: "acme.com 是企业站，攻击面：官网(www)、后台(admin)、API(api)、商城(shop)。先枚举资产，再挑高价值目标。", + summary: + "graph_overview：goals 已由目标层拆好——g1「获取 acme.com 后台管理权限」、g2「读取用户敏感数据」，均 open；facts=0、recent_facts 为空（图基本空白，还没有任何 worker 产出）。范围内资产只有根域 acme.com + 一个 C 段。攻击面初判：www / admin / api / shop / vpn 五个入口。图里没有 fact，我只能做【极轻量只读探测】把初始意图说具体，绝不在 plan 里把活干了。", }, { - seq: 3, + seq: 1002, + worker: "planner", + ts: T("2026-07-24T09:12:40Z"), + kind: "tool_use", + tool: "bash", + tool_use_id: "p0-probe", + summary: "curl -sI https://acme.com; curl -sI https://admin.acme.com", + detail: "开局态势几乎空白，做 ≤3 次只读探测校准初始意图方向（不做枚举/验证，那是 worker 的活）。", + }, + { + seq: 1003, + worker: "planner", + ts: T("2026-07-24T09:12:52Z"), + kind: "tool_result", + tool: "bash", + tool_use_id: "p0-probe", + summary: "acme.com → 302 → www；Server: nginx/1.24.0\nadmin.acme.com → 200，X-Powered-By 缺省，body 命中 element-ui/vue 特征", + detail: "admin 是独立后台登录页；确认五入口有效，可把初始意图描述写精准。", + }, + { + seq: 1004, worker: "planner", ts: T("2026-07-24T09:13:00Z"), - kind: "text", - summary: - "拆出两个目标：① 获取后台管理权限 ② 读取用户敏感数据。派生首批意图：子域枚举、后台口令测试、注入探测、API 越权。", + kind: "tool_use", + tool: "add_intent", + tool_use_id: "p0-ai", + summary: "add_intent × 4（批量派首批探索方向）", + detail: + "intents=[\n { summary: 'acme.com 全量子域枚举 + 存活主机端口/服务识别，摸清攻击面', asset_ids: [acme.com], priority: 6 },\n { summary: 'admin.acme.com 后台默认口令/弱口令测试（Element-UI 登录，无验证码）', asset_ids: [admin], priority: 9 },\n { summary: 'www.acme.com/search q 参数 SQL 注入探测', asset_ids: [www], priority: 8 },\n { summary: 'api.acme.com 订单接口越权(IDOR)测试', asset_ids: [api], priority: 8 },\n]\nparent_ids 均空（顶层全新方向，兜底连 origin fact）。", }, - { seq: 300, worker: "planner", ts: T("2026-07-25T14:12:00Z"), kind: "round", summary: "第 1 轮 · 攻击面加码" }, { - seq: 301, + seq: 1005, worker: "planner", - ts: T("2026-07-25T14:12:20Z"), - kind: "text", - summary: - ".git 泄露出源码+硬编码 DB 口令，shop 指纹疑似 Fastjson。新增目标③「突破 DMZ 建立内网立足点」，派生指纹关联、反序列化 RCE 两条意图。", + ts: T("2026-07-24T09:13:02Z"), + kind: "tool_result", + tool: "add_intent", + tool_use_id: "p0-ai", + summary: "4 intents created → ids=[i1, i2, i4, i5]，已连入 frontier", + detail: "刻意保持路线多样：i1 侦察 + i2（g1 认证路线）+ i4/i5（g2 数据路线），2 条独立机理并行，不过早收敛。", }, { - seq: 302, + seq: 1006, + worker: "planner", + ts: T("2026-07-24T09:13:05Z"), + kind: "text", + summary: + "本轮派生首批 4 条意图：i1 资产侦察、i2 后台口令、i4 注入、i5 API 越权。g1 走 i2、g2 走 i4/i5，双目标各有独立入口在推进。等 worker 产出事实后再据血缘加码。", + }, + + // ── 第 1 轮 · 攻击面加码（2026-07-25 14:12）── + { seq: 1100, worker: "planner", ts: T("2026-07-25T14:12:00Z"), kind: "round", summary: "第 1 轮 · 攻击面加码" }, + { + seq: 1101, + worker: "planner", + ts: T("2026-07-25T14:12:15Z"), + kind: "thinking", + summary: + "对照 recent_facts：workG 交回 faGit「.git 可下载，git-dumper 还原出后端源码」与 faCreds「config.php 硬编码 sa/Acme@2021（疑似内网通用）」；work#6 指纹交回 fa3「shop 命中 Fastjson 1.2.24（存在已知反序列化 RCE）」。两条【材料性新机理】：① 硬编码 DB 口令可能横跨内网复用；② Fastjson 有成熟 RCE 打法。这打开一条与现有认证/注入路线【本质不同】的入口——从 shop 直接 RCE 突破进 DMZ。先拉 faGit 详情核实产物。", + }, + { + seq: 1102, + worker: "planner", + ts: T("2026-07-25T14:12:28Z"), + kind: "tool_use", + tool: "node_detail", + tool_use_id: "p1-nd", + summary: "node_detail(faGit) —— 核对 workG 的源码还原产物", + detail: "add_intent 前先看 worker 到底还原出什么，避免凭指纹臆测方向。", + }, + { + seq: 1103, + worker: "planner", + ts: T("2026-07-25T14:12:35Z"), + kind: "tool_result", + tool: "node_detail", + tool_use_id: "p1-nd", + summary: + "faGit 证据：git-dumper 还原出 config.php / db.php / order_api.php；config.php 内 $db=['host'=>'10.10.10.5','user'=>'sa','pass'=>'Acme@2021']", + detail: "硬编码指向内网 10.10.10.5(FS01)，佐证 faCreds 的「内网通用」推断——RCE 落地后这组口令值得复用。", + }, + { + seq: 1104, + worker: "planner", + ts: T("2026-07-25T14:12:50Z"), + kind: "tool_use", + tool: "add_intent", + tool_use_id: "p1-ai", + summary: "add_intent × 2（Fastjson 突破链，串行分步先派可执行一步）", + detail: + "intents=[\n { summary: 'shop.acme.com 组件指纹深挖 + Fastjson 1.2.24 CVE 精确关联(确认 gadget 链)', asset_ids: [shop], parent_ids: [fa3], priority: 7 },\n { summary: '构造 Fastjson 反序列化 payload 触发 shop RCE，获取反弹 shell', asset_ids: [shop], parent_ids: [fa3], priority: 9 },\n]\n注：这是强依赖串行链，已 TodoWrite 记『指纹→RCE→立足点提权』三步，本轮只派前两步（提权依赖 shell 产出，下轮再派）。", + }, + { + seq: 1105, + worker: "planner", + ts: T("2026-07-25T14:12:52Z"), + kind: "tool_result", + tool: "add_intent", + tool_use_id: "p1-ai", + summary: "2 intents created → ids=[i6, i7]，已由 fa3 连入血缘", + }, + { + seq: 1106, + worker: "planner", + ts: T("2026-07-25T14:13:00Z"), + kind: "text", + summary: + "态势升级：源码 + 硬编码内网口令 + Fastjson RCE 构成一条全新突破路线。已请目标层登记 g3「从外网突破 DMZ，建立内网立足点」，派 i6（指纹/CVE 关联）、i7（触发 RCE）。DMZ 提权一步依赖 shell，记入待办下轮派。", + }, + + // ── 第 2 轮 · 立足点已建，转内网（2026-07-25 20:25）── + { + seq: 1200, worker: "planner", ts: T("2026-07-25T20:25:00Z"), kind: "round", summary: "第 2 轮 · 立足点已建，转内网", }, { - seq: 303, + seq: 1201, worker: "planner", - ts: T("2026-07-25T20:26:00Z"), + ts: T("2026-07-25T20:25:18Z"), + kind: "thinking", + summary: + "对照血缘：i7 交回 fa4「反弹 shell www-data@dmz-web01(10.0.20.15)，位于 DMZ 段」；提权链交回 fa5「sudo -l 命中 NOPASSWD python3 → 可提权 root」、fa6「立足点第二网卡直连内网 10.10.10.0/24」。g3 的验收条件（突破 DMZ + 建立足点）已被 fa4+fa6 直接满足。先核实 fa6 的内网可达性再 prove_goal。", + }, + { + seq: 1202, + worker: "planner", + ts: T("2026-07-25T20:25:30Z"), + kind: "tool_use", + tool: "node_detail", + tool_use_id: "p2-nd", + summary: "node_detail(fa6) —— 确认双网卡与内网可达", + }, + { + seq: 1203, + worker: "planner", + ts: T("2026-07-25T20:25:38Z"), + kind: "tool_result", + tool: "node_detail", + tool_use_id: "p2-nd", + summary: "fa6 证据：ip a → eth0 10.0.20.15(DMZ) / eth1 10.10.10.15(内网)；ping 10.10.10.10 通，跨越 DMZ 边界成立", + }, + { + seq: 1204, + worker: "planner", + ts: T("2026-07-25T20:25:50Z"), + kind: "tool_use", + tool: "prove_goal", + tool_use_id: "p2-pg", + summary: "prove_goal(g3, evidence=fa6)", + detail: "reason：i7 RCE 拿到 DMZ 立足点(fa4)、fa5 提权 root、fa6 证明第二网卡直连内网并 ping 通 DC01——『突破 DMZ 建立内网立足点』验收达成。", + }, + { + seq: 1205, + worker: "planner", + ts: T("2026-07-25T20:25:52Z"), + kind: "tool_result", + tool: "prove_goal", + tool_use_id: "p2-pg", + summary: "goal g3 → met（证据 fa6 已连入）；仍有 g1/g2/g4 未达成，任务继续", + }, + { + seq: 1206, + worker: "planner", + ts: T("2026-07-25T20:26:10Z"), + kind: "tool_use", + tool: "add_intent", + tool_use_id: "p2-ai", + summary: "add_intent × 4（以立足点为 pivot 展开内网横向）", + detail: + "intents=[\n { summary: '以 dmz-web01 为 pivot 做内网 10.10.10.0/24 主机发现与端口识别', asset_ids: [dmz-web01], parent_ids: [fa6], priority: 8 },\n { summary: '内网横向：复用硬编码口令 sa/Acme@2021 做凭据复用与密码喷洒', parent_ids: [faCreds], priority: 8 },\n { summary: '内部 Jenkins(10.10.10.20) 未授权 /script Groovy 执行(RCE)', parent_ids: [fa6], priority: 9 },\n { summary: '对域服务账号做 Kerberoasting 并离线破解', parent_ids: [fa6], priority: 8 },\n]\n主机发现是前置(先派)，其余三条是并行的横向探索维度。", + }, + { + seq: 1207, + worker: "planner", + ts: T("2026-07-25T20:26:12Z"), + kind: "tool_result", + tool: "add_intent", + tool_use_id: "p2-ai", + summary: "4 intents created → ids=[i9, i10, i11, i12]", + }, + { + seq: 1208, + worker: "planner", + ts: T("2026-07-25T20:26:20Z"), kind: "text", summary: - "RCE 拿到 www-data 并提权 root，立足点双网卡直连 10.10.10.0/24（g3 达成）。新增目标④「拿下内部靶标 DC01」，派生内网主机发现、凭据复用、Jenkins 利用、Kerberoasting 四条意图。", + "g3 达成，攻击面从外网转入内网。已请登记 g4「内网横向，拿下内部靶标域控 DC01」，派 i9（主机发现）+ i10/i11/i12 三条并行横向。凭据复用直接锚在 faCreds 上——外网源码泄露的口令喂给内网横向。", }, - { seq: 304, worker: "planner", ts: T("2026-07-26T00:25:00Z"), kind: "round", summary: "第 3 轮 · 靶标达成复盘" }, + + // ── 第 3 轮 · 靶标达成复盘（2026-07-26 00:25）── + { seq: 1300, worker: "planner", ts: T("2026-07-26T00:25:00Z"), kind: "round", summary: "第 3 轮 · 靶标达成复盘" }, { - seq: 305, + seq: 1301, worker: "planner", - ts: T("2026-07-26T00:26:00Z"), + ts: T("2026-07-26T00:25:16Z"), + kind: "thinking", + summary: + "对照本轮 worker 产出：i9→fa7「内网存活 DC01/FS01/JENKINS」；i11→fa8「Jenkins /script 无鉴权 Groovy」+ fa9「从 Jenkins 凭据库导出 acme\\svc_deploy 明文口令」；i12→fa10「Kerberoast svc_sql TGS，hashcat 破出 Sql@2020」。svc_deploy 属 Domain Admins，i13 已用其 psexec 拿下 DC01 并 secretsdump。g4 验收（拿下 DC01）应已达成，先核对 finding 证据链再盖章。", + }, + { + seq: 1302, + worker: "planner", + ts: T("2026-07-26T00:25:30Z"), + kind: "tool_use", + tool: "list_findings", + tool_use_id: "p3-lf", + summary: "list_findings —— 核对 DC01 沦陷的确认证据", + }, + { + seq: 1303, + worker: "planner", + ts: T("2026-07-26T00:25:40Z"), + kind: "tool_result", + tool: "list_findings", + tool_use_id: "p3-lf", + summary: + "F-09「域控 DC01 完全控制 / secretsdump 导出全域哈希」critical（证据：psexec svc_deploy@10.10.10.10 + NTDS.dit dump）；上游链 fa9→fa10→i13 完整", + }, + { + seq: 1304, + worker: "planner", + ts: T("2026-07-26T00:25:52Z"), + kind: "tool_use", + tool: "prove_goal", + tool_use_id: "p3-pg", + summary: "prove_goal(g4, evidence=F-09)", + detail: "reason：Jenkins 泄露 svc_deploy(Domain Admin) → psexec 控制 DC01 → secretsdump 全域哈希，F-09 已确认，『拿下内部靶标域控 DC01』达成。", + }, + { + seq: 1305, + worker: "planner", + ts: T("2026-07-26T00:25:54Z"), + kind: "tool_result", + tool: "prove_goal", + tool_use_id: "p3-pg", + summary: "goal g4 → met；系统提示：g2 仍 open，非最后一个目标，任务未整体收官", + }, + { + seq: 1306, + worker: "planner", + ts: T("2026-07-26T00:26:10Z"), kind: "text", summary: - "svc_deploy 属于 Domain Admins，已 psexec 拿下 DC01 并 secretsdump 全域哈希（g4 达成）。内外网链路打通，收敛剩余 g2（后台批量用户数据）。", + "g4 收官，内外网链路完全打通（外网 RCE → DMZ → 内网 → 域控）。逐一比对未覆盖方向：g2「读取用户敏感数据」仍缺临门一脚，但现有 i3（后台用户接口）+ i5（API IDOR）已覆盖该方向、且都在 running。本轮无【本质不同的未覆盖新方向】——按克制原则 0 新意图，等 i3/i5 产出。", }, - { seq: 4, worker: "planner", ts: T("2026-07-26T03:55:00Z"), kind: "round", summary: "第 4 轮 · 复盘" }, + + // ── 第 4 轮 · 后台达成，收敛 g2（2026-07-26 03:55）── + { seq: 1400, worker: "planner", ts: T("2026-07-26T03:55:00Z"), kind: "round", summary: "第 4 轮 · 后台达成，收敛 g2" }, { - seq: 5, + seq: 1401, worker: "planner", - ts: T("2026-07-26T03:55:10Z"), + ts: T("2026-07-26T03:55:14Z"), + kind: "thinking", + summary: + "对照 recent_facts：work#2 用 faCreds 里的 Acme@2021 命中 admin 后台（i2 done），后台管理权限到手——g1 验收达成，先 prove_goal 盖章。随后做【量化验收核对】：g2「读取用户敏感数据」当前只有 i3/i5 摸到接口面、尚未真正取到批量数据，实测未达标，【禁止】prove_goal，改为提优先级补足取数意图。", + }, + { + seq: 1402, + worker: "planner", + ts: T("2026-07-26T03:55:26Z"), + kind: "tool_use", + tool: "prove_goal", + tool_use_id: "p4-pg", + summary: "prove_goal(g1, evidence=fa1)", + detail: "reason：i2 用外网泄露的硬编码口令 Acme@2021 成功登入 admin 后台（Element-UI），获取后台管理权限，g1 达成。", + }, + { + seq: 1403, + worker: "planner", + ts: T("2026-07-26T03:55:28Z"), + kind: "tool_result", + tool: "prove_goal", + tool_use_id: "p4-pg", + summary: "goal g1 → met；剩余未达成：g2", + }, + { + seq: 1404, + worker: "planner", + ts: T("2026-07-26T03:55:40Z"), + kind: "thinking", + summary: + "g2 量化核对：目标要求『读取用户敏感数据』=实际拿到批量用户记录。graph_overview 里 findings 尚无『成功导出用户数据』条目，i3/i5 均 running 未回取数结果——实测未达标。严禁以『后台已进、大体达成』提前标 met，继续派意图补足差距。", + }, + { + seq: 1405, + worker: "planner", + ts: T("2026-07-26T03:55:52Z"), + kind: "tool_use", + tool: "add_intent", + tool_use_id: "p4-ai", + summary: "add_intent × 2（收敛 g2 取数，提高优先级）", + detail: + "intents=[\n { summary: '已登入 admin 后台，枚举用户管理/导出接口并批量拉取用户敏感字段', asset_ids: [admin], parent_ids: [fa1], priority: 9 },\n { summary: 'api.acme.com 订单 IDOR 遍历 order_id 批量拉取用户订单/PII', asset_ids: [api], parent_ids: [fa2], priority: 9 },\n]\n注：i3/i5 已在 running，本轮以更精准的取数描述 + 提优先级(→9)加码，不重复派同义意图。", + }, + { + seq: 1406, + worker: "planner", + ts: T("2026-07-26T03:55:54Z"), + kind: "tool_result", + tool: "add_intent", + tool_use_id: "p4-ai", + summary: "2 intents updated → ids=[i3, i5]（优先级升至 9，描述细化为『批量取数』）", + }, + { + seq: 1407, + worker: "planner", + ts: T("2026-07-26T03:55:58Z"), kind: "text", - summary: "后台已拿下(g1 达成)。继续 g2：派生「枚举后台用户接口」「API IDOR 测试」两条意图并提高优先级。", + summary: + "g1 达成（4/4 里已 met 3 个：g1/g3/g4）。g2 严守量化验收——只进后台不算，必须拿到批量用户数据才盖章；已把 i3/i5 提到 priority 9 并细化为取数意图，等 worker 回传即可收官整个任务。", }, // work#1 · i1 子域枚举与端口扫描（done） From 5f0d91bbb32737bc125eb001b47b6216ad7feb30 Mon Sep 17 00:00:00 2001 From: Autumn-27 <53844290+Autumn-27@users.noreply.github.com> Date: Sat, 29 Aug 2026 09:08:10 -0400 Subject: [PATCH 20/23] =?UTF-8?q?fix(web):=20=E6=89=8B=E6=9C=BA=E7=AB=AF?= =?UTF-8?q?=E4=BC=9A=E8=AF=9D=E8=AF=A6=E6=83=85=E6=8A=98=E5=8F=A0=E4=BC=9A?= =?UTF-8?q?=E8=AF=9D=E5=88=97=E8=A1=A8=EF=BC=8C=E9=81=BF=E5=85=8D=E4=BC=9A?= =?UTF-8?q?=E8=AF=9D=E8=AE=B0=E5=BD=95=E8=A2=AB=E6=8C=A4=E5=8E=8B=E7=9C=8B?= =?UTF-8?q?=E4=B8=8D=E5=85=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 小屏下会话列表固定占 10~15rem，会话记录只剩标题和输入框。改为 ). const [store, setStore] = React.useState({}); // Worker sessions derived from exploration intents (paged past the old 300 cap). @@ -1130,6 +1135,13 @@ export function SessionsTab({ taskId }: { taskId: string }) { const isSystem = active.role === "system"; const activeKey = keyForSession(active); const activeState = store[activeKey]; + // 折叠态（手机端）标题栏要替代整张列表：显示当前会话名 + 其它会话的未读合计， + // 否则收起后既不知道自己在看哪个会话，也看不到别处有新消息。 + const activeDisplayTitle = (active.role === "worker" ? sessionMeta.get(active.id)?.title : "") || active.title; + const hiddenUnread = React.useMemo( + () => Object.entries(store).reduce((sum, [key, s]) => (key === activeKey ? sum : sum + s.unread), 0), + [store, activeKey], + ); // Keep the SSE dispatcher's notion of the active session current, and lazily load // + clear unread whenever the active session changes. @@ -1350,12 +1362,44 @@ export function SessionsTab({ taskId }: { taskId: string }) { return ( -
+ {/* 高度预留：页面头部（标题行 + 目标 + Tabs ≈ 7.5rem）+ 内容内边距。手机端 p-4、 + 桌面端 lg:p-6，且桌面还要留出滚动余量，所以两档分别预留 10rem / 13rem —— 手机端 + 沿用 13rem 会白白吃掉 3rem 的记录高度。 */} +
{/* Left: session list */} -
+
-
-
会话列表
+
+ {!MOCK && ( )}
- + div]:block!", + !listOpen && "max-lg:hidden", + )} + >
{(["mainagent", "planner", "system", "worker"] as const).map((role) => { const items = grouped[role]; @@ -1421,7 +1472,10 @@ export function SessionsTab({ taskId }: { taskId: string }) { displayTitle={meta?.title ?? s.title} hasPending={hasPendingForSession(s)} unread={store[keyForSession(s)]?.unread} - onClick={() => setActiveId(s.id)} + onClick={() => { + setActiveId(s.id); + setListOpen(false); // 手机端选完即收起，把高度还给会话记录 + }} controlling={controllingIntent === s.intent_id} deleted={meta?.deleted} onPause={() => void controlWorker(s, "pause")} @@ -1459,7 +1513,7 @@ export function SessionsTab({ taskId }: { taskId: string }) { {/* Right: transcript */}
-
+
{(() => { const isWorker = active.role === "worker"; const meta = isWorker ? sessionMeta.get(active.id) : undefined; @@ -1518,8 +1572,24 @@ export function SessionsTab({ taskId }: { taskId: string }) {
{tokenTotal.any && ( + {/* 手机端用短标签（入/缓/出）：长标签会把这一行撑成两行，进一步压缩记录区。 */} - + + + + 输入 {activeTokens.input_tokens.toLocaleString()} · 输出{" "} From cb706d3c8c88bf144a597830a93d395a2fc73152 Mon Sep 17 00:00:00 2001 From: pphua Date: Sat, 29 Aug 2026 21:35:15 +0800 Subject: [PATCH 21/23] =?UTF-8?q?=E6=96=B0=E5=A2=9Eworker=E7=9B=B4?= =?UTF-8?q?=E6=8E=A5=E5=AF=B9=E8=AF=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- agent/cancelcause.go | 2 + agent/planner.go | 20 +- agent/tools.go | 40 ++ agent/worker.go | 56 +- agent/worker_intervention_test.go | 28 + db/exploration.go | 473 ++++++++++++++++- db/intent_intervention_lifecycle_test.go | 204 +++++++ server/engine.go | 501 +++++++++++++++++- server/engine_cancelcause_test.go | 1 + server/goals.go | 7 + server/intent_intervention.go | 125 +++++ server/intent_intervention_protocol_test.go | 378 +++++++++++++ server/manager.go | 39 +- server/server.go | 9 + .../tasks/detail/_tabs/sessions-tab.tsx | 173 ++++-- web/src/lib/api.ts | 8 + web/src/lib/mock/handler.ts | 96 +++- 17 files changed, 2082 insertions(+), 78 deletions(-) create mode 100644 agent/worker_intervention_test.go create mode 100644 db/intent_intervention_lifecycle_test.go create mode 100644 server/intent_intervention.go create mode 100644 server/intent_intervention_protocol_test.go diff --git a/agent/cancelcause.go b/agent/cancelcause.go index f4705bc..e61cfaf 100644 --- a/agent/cancelcause.go +++ b/agent/cancelcause.go @@ -45,6 +45,8 @@ var ( "规划者调用 kill_work 主动终止了这条意图，通常表示方向跑偏或已无继续价值；意图会标记为 stopped，不会自动重新领取") AbortWorkPausedByUser = cause("work_paused_by_user", "用户暂停了这条 Worker 意图", "用户暂停了正在运行的 Worker。本次调用被取消，意图转为 paused；已经登记的意图、事实、漏洞和活动记录全部保留，恢复后从头重新执行") + AbortWorkIntervenedByUser = cause("work_intervened_by_user", "用户暂停 Worker 并发起了新对话", + "用户向正在运行的 Worker 发送了新的对话意图。本次调用被立即取消并短暂进入 paused；新消息作为同一会话的下一条用户消息，随后自动恢复并优先重新领取") AbortWorkCancelledByUser = cause("work_cancelled_by_user", "用户取消了这条 Worker 意图", "用户取消了正在运行的 Worker。本次调用被取消；Worker 退出写入区后，服务端会事务性删除该意图及其直接产生的事实、漏洞和执行记录") AbortWorkFinished = cause("work_finished", "Worker 已正常结束并释放 context", diff --git a/agent/planner.go b/agent/planner.go index 95802f8..3f63166 100644 --- a/agent/planner.go +++ b/agent/planner.go @@ -123,6 +123,7 @@ func renderPlannerTodos(items []actool.Todo) string { // // "done" — a worker finished intent IntentID (its output conclusion is fetched). // "finding" — a worker reported a finding on intent IntentID (Detail = 摘要). +// "worker_message" — the human changed one Worker's direction (Detail = 新方向). // "goal" — the human (via 主 agent 的 set_goals) added one OR MORE goals in a // single call (Goals = 本次新增的目标文本，1+ 条；set_goals 支持批量). // "goal_deleted" — the human deleted a goal from 总览的目标管理 (Detail = 被删目标文本). @@ -130,14 +131,17 @@ func renderPlannerTodos(items []actool.Todo) string { // "cancelled" — the human deleted intent IntentID (Detail = 删除原因). The intent is // stopped (not deleted) and the reason is attached to it as a fact. type TriggerEvent struct { - Kind string - IntentID int64 - Detail string - Goals []string // Kind=="goal" 专用：本次 set_goals 新增的目标文本（1 条或多条） - OldGoal string // Kind=="goal_edited" 专用：修改前的目标文本 - NewGoal string // Kind=="goal_edited" 专用：修改后的目标文本 + Kind string + ActivityID int64 // persistent activity id; worker_message notifications use it for de-duplication + IntentID int64 + Detail string + Goals []string // Kind=="goal" 专用：本次 set_goals 新增的目标文本（1 条或多条） + OldGoal string // Kind=="goal_edited" 专用：修改前的目标文本 + NewGoal string // Kind=="goal_edited" 专用：修改后的目标文本 } +const TriggerWorkerMessage = db.PlannerEventWorkerMessage + // renderTriggers spells out the change(s) that fired this round: for a finished // worker — which intent + its output conclusion; for a finding — which intent + // what was found. Empty for time/heartbeat wakes. Reads the store (best-effort; @@ -162,6 +166,9 @@ func renderTriggers(ts *db.ExplorationStore, evs []TriggerEvent) string { b.WriteString(fmt.Sprintf("\n- 人修改了目标，由「%s」变为「%s」—— 请据新目标调整探索方向（原方向若已不适用请停派）。", ev.OldGoal, ev.NewGoal)) case "finding": b.WriteString(fmt.Sprintf("\n- 意图 #%d（%s）的 worker 报告了一个 finding：%s", ev.IntentID, intentSummary(ts, ev.IntentID), ev.Detail)) + case TriggerWorkerMessage: + b.WriteString(fmt.Sprintf("\n- 用户向意图 #%d（%s）的 Worker 下发了新的人工方向：%s —— 该方向已优先交给同一 Worker；后续规划必须以它替代或补充原意图描述，避免生成冲突、回退或重复方向。", + ev.IntentID, intentSummary(ts, ev.IntentID), truncOutput(ev.Detail, 2000))) case "cancelled": b.WriteString(fmt.Sprintf("\n- 意图 #%d 由用户删除，意图内容是：%s、删除原因是：%s。该意图已停止（不再执行），其原因已作为事实挂在该意图上；请据此重新规划。", ev.IntentID, intentSummary(ts, ev.IntentID), ev.Detail)) default: // "done" @@ -250,6 +257,7 @@ const plannerDefaultTmpl = `你是一个授权渗透测试系统的"规划者" 决策流程（每次唤醒）： 1. **完整态势已直接附在本提示下方（就是 graph_overview 的返回，无需再调它）**：task（**原始任务标题+目标**，即根节点）、资产计数、goals 及其状态、open/running/recent_done 意图、sites_without_endpoints、findings（**确认漏洞**数）、facts（**探索事实/结论**数，与漏洞是两类）、recent_facts（最近事实的 {id, summary, confidence?}，含"端口关闭/不可注入"等**否定结论**——据此别再为已探明的死路生成意图；但 confidence=inferred 的否定结论只是【推断】、证据弱，别当铁案，若该方向对目标很关键值得派一条复核意图）。**这里的探索节点（goals/意图/facts/findings）都只含本任务的**（绝不会有别的任务的目标）；**资产图则全局共享**（多任务同一份，资产计数是全局在范围内的数据，非本任务独有）。需要更深的细节时才按需调：list_facts 分页列事实（最新在前，默认 20 条，可传 q 关键词过滤、before 翻页，返回带 total/has_more）、list_findings 列全部漏洞、**node_detail(id)** 取某条的完整证据/详情（列表/recent_facts 只给摘要）。 - open_intents / running_intents / recent_done_intents——"哪些方向已经有意图在覆盖/已尝试"。每个意图还带 **parents（上游：它派生自哪些事实/意图）和 yields（下游：它产生了哪些事实/发现）**——这就是探索图的**血缘关系**，据此理解"哪些事实来自哪个方向、能否综合成新方向"。recent_facts 里每个事实带 **from_intent**（由哪个意图产生）。 + - worker_directions——用户在 Worker 对话里对每个意图最近一次下发的人工方向（持久化、重启后仍存在）。它比该意图最初的 summary 更新；判断在跑方向、补意图和使用 steer/kill 时必须以它为准，禁止重新派发与人工方向冲突的工作。 - sites_without_endpoints / findings——"哪些方向【可能】需要探索"。 - 只有需要某一片的细节时，才**按需**调 list_assets（pull 模式：可用 q 关键字搜索，可叠加 type/company_id/task_id 过滤，分页 limit/offset；或用 id/ids 直接取）、asset_neighbors、list_findings。资产图全局共享，别默认拉全量。资产中可能包含非本次任务涉及到的资产，所以需要主要出现非本次任务相关的资产时忽略这些资产。 2. 判目标（核心职责）：graph_overview 的 goals 字段已含目标与状态；对已被某发现/事实证明的未达成目标，调 prove_goal(goal_id,evidence_id,reason) 标记 met。**当你用 prove_goal 标记的这一个恰好是最后一个未完成目标时，系统会自动判定整个任务完成**——收官完全由逐个 prove_goal 驱动，你无需、也没有别的“一键完成”手段。 diff --git a/agent/tools.go b/agent/tools.go index ebed6d8..3a34ca1 100644 --- a/agent/tools.go +++ b/agent/tools.go @@ -48,6 +48,38 @@ func compactIntents(ns []*db.Node, parentsOf, yieldsOf map[int64][]int64) []map[ return out } +const workerDirectionOverviewLimit = 300 +const workerDirectionMessageRunes = 1200 + +// compactWorkerDirections keeps the latest durable human direction for each +// intent small enough for every Planner/Main Agent turn. The full message stays +// available in the Worker activity trace when an unusually long direction is +// truncated here. +func compactWorkerDirections(items []db.WorkerDirection) []map[string]any { + out := make([]map[string]any, 0, len(items)) + for _, item := range items { + message := item.Message + truncated := false + if runes := []rune(message); len(runes) > workerDirectionMessageRunes { + message = string(runes[:workerDirectionMessageRunes]) + "…" + truncated = true + } + row := map[string]any{ + "activity_id": item.ActivityID, + "intent_id": item.IntentID, + "intent_state": item.IntentState, + "intent_summary": item.IntentSummary, + "message": message, + "updated_at": item.CreatedAt, + } + if truncated { + row["truncated"] = true + } + out = append(out, row) + } + return out +} + // ToolSet exposes the PG-backed dual graph (asset + exploration) to an LLM agent. // One ToolSet is created per planner/worker run; per-run signals live here. type ToolSet struct { @@ -360,6 +392,14 @@ func (t *ToolSet) graphOverview() actool.CoreTool { // an empty context and always needs this first). func (t *ToolSet) graphOverviewData() map[string]any { out := map[string]any{} + // worker_directions is the persistent cross-agent steering channel. It has + // one latest accepted human message per local intent, so Planner and Main + // Agent do not have to infer a changed direction from the original summary. + if directions, err := t.ts.LatestWorkerDirections(workerDirectionOverviewLimit); err == nil { + out["worker_directions"] = compactWorkerDirections(directions) + } else { + out["worker_directions"] = []map[string]any{} + } // goals summary folded in so the planner needn't call list_goals each round. goals, _ := t.ts.ListByKind(db.KindGoal, 100) gsum := make([]map[string]any, 0, len(goals)) diff --git a/agent/worker.go b/agent/worker.go index 2c77722..84f693b 100644 --- a/agent/worker.go +++ b/agent/worker.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "strconv" + "strings" "time" "github.com/Autumn-27/artex/db" @@ -68,6 +69,30 @@ type Worker struct { nonStreamingFn func() bool } +// WorkerSessionID returns the stable transcript key used by a worker intent. +// Worker slots are reusable, so the intent id (rather than work#N) is the +// session identity. Keep this helper public so the Worker message API and UI +// can refer to exactly the conversation that will be resumed. +func WorkerSessionID(explorationID, intentID int64) string { + return fmt.Sprintf("exp%d-worker-i%d", explorationID, intentID) +} + +const workerChatMarkerPrefix = "" +} + +func hasWorkerChatMessage(messages []llm.Message, requestID string) bool { + marker := workerChatMarker(requestID) + for _, message := range messages { + if message.Role == llm.RoleUser && strings.Contains(message.Text(), marker) { + return true + } + } + return false +} + // SetNonStreaming wires a resolver deciding whether runs use the non-streaming // model path (true = non-streaming). nil/unset = streaming (default). Read per // run so a profile or task-chain toggle takes effect without rebuilding. @@ -274,6 +299,18 @@ func renderWorkerGraphOverview(data map[string]any) string { // explored but persisted nothing isn't mistaken for done, and the engine can log // facts/assets/findings separately instead of lumping them under "facts"). func (w *Worker) Execute(ctx context.Context, name string, taskID int64, as *db.AssetStore, ts *db.ExplorationStore, intent *db.Node, hooks harness.HookRunner, emit func(db.Activity), enr EnrichTrigger, notifyFinding func(int64, string)) (harness.TerminalReason, WriteCounts, error) { + return w.execute(ctx, name, taskID, as, ts, intent, hooks, emit, enr, notifyFinding, "", "") +} + +// ExecuteWithMessage runs the next turn in the same intent conversation with a +// human-authored message. The HTTP handler does not edit the transcript; +// agentcore records the message as a normal user turn when this Worker starts. +// This keeps Worker continuation identical to the regular agent chat flow. +func (w *Worker) ExecuteWithMessage(ctx context.Context, name string, taskID int64, as *db.AssetStore, ts *db.ExplorationStore, intent *db.Node, hooks harness.HookRunner, emit func(db.Activity), enr EnrichTrigger, notifyFinding func(int64, string), requestID, message string) (harness.TerminalReason, WriteCounts, error) { + return w.execute(ctx, name, taskID, as, ts, intent, hooks, emit, enr, notifyFinding, strings.TrimSpace(requestID), strings.TrimSpace(message)) +} + +func (w *Worker) execute(ctx context.Context, name string, taskID int64, as *db.AssetStore, ts *db.ExplorationStore, intent *db.Node, hooks harness.HookRunner, emit func(db.Activity), enr EnrichTrigger, notifyFinding func(int64, string), requestID, message string) (harness.TerminalReason, WriteCounts, error) { tsx := NewToolSet(ts, name) tsx.SetTaskID(taskID) coverageEnabled := as == nil || as.CoverageEnabled(taskID) @@ -358,7 +395,7 @@ func (w *Worker) Execute(ctx context.Context, name string, taskID int64, as *db. } if w.tx != nil { // persist raw LLM conversation; one file per worked intent opts.Transcript = w.tx - opts.SessionID = fmt.Sprintf("exp%d-worker-i%d", ts.ID(), intent.ID) + opts.SessionID = WorkerSessionID(ts.ID(), intent.ID) } intentID := intent.ID emitWrap := func(r db.Activity) { @@ -398,11 +435,26 @@ func (w *Worker) Execute(ctx context.Context, name string, taskID int64, as *db. // being re-run. The transcript ID is deterministic per intent, so if a prior // session exists the worker continues from where it left off instead of // restarting from scratch. + alreadyRecorded := false if w.tx != nil { _ = s.Resume(opts.SessionID) - if len(s.Messages()) > 0 { + alreadyRecorded = requestID != "" && hasWorkerChatMessage(s.Messages(), requestID) + if len(s.Messages()) > 0 && message == "" { seedUnlockFromHistory(s.Messages(), def.UnlockSkill) input = "继续执行。" + } else if len(s.Messages()) > 0 { + seedUnlockFromHistory(s.Messages(), def.UnlockSkill) + } + } + if message != "" { + if alreadyRecorded { + input = "继续执行上一次人工对话输入的新意图。不要重复已经完成的动作。" + } else if len(s.Messages()) > 0 { + input = workerChatMarker(requestID) + "\n【人工对话输入的新意图】\n" + message + + "\n\n请立即按这条人工输入执行，完成后再根据上下文决定原任务是否需要继续。" + } else { + input += "\n\n" + workerChatMarker(requestID) + "\n【人工对话输入的新意图】\n" + message + + "\n\n请优先执行这条人工输入。" } } diff --git a/agent/worker_intervention_test.go b/agent/worker_intervention_test.go new file mode 100644 index 0000000..ab8977e --- /dev/null +++ b/agent/worker_intervention_test.go @@ -0,0 +1,28 @@ +package agent + +import ( + "testing" + + "github.com/Autumn-27/norma/llm" +) + +func TestWorkerSessionIDIsStablePerIntent(t *testing.T) { + if got := WorkerSessionID(12, 34); got != "exp12-worker-i34" { + t.Fatalf("session id = %q, want exp12-worker-i34", got) + } +} + +func TestWorkerChatMarkerOnlyMatchesItsNormalUserTurn(t *testing.T) { + requestID := "worker-message-123" + messages := []llm.Message{ + {Role: llm.RoleAssistant, Content: []llm.ContentBlock{llm.TextBlock(workerChatMarker(requestID))}}, + llm.UserText(workerChatMarker("worker-message-other") + "\nother"), + llm.UserText(workerChatMarker(requestID) + "\nnew intent"), + } + if !hasWorkerChatMessage(messages, requestID) { + t.Fatal("expected the matching user turn to be detected") + } + if hasWorkerChatMessage(messages, "worker-message-missing") { + t.Fatal("unrelated request id matched a Worker user turn") + } +} diff --git a/db/exploration.go b/db/exploration.go index eb0e677..7c40e85 100644 --- a/db/exploration.go +++ b/db/exploration.go @@ -14,6 +14,11 @@ import ( // it to distinguish an expected control race (HTTP 409) from a storage failure. var ErrIntentStateConflict = errors.New("intent state changed concurrently") +// ErrIntentInterventionConflict marks a rejected or conflicting worker +// intervention request. It covers state races, a reused request id with a +// different payload, and a second request trying to overtake a pending one. +var ErrIntentInterventionConflict = errors.New("intent intervention conflict") + // utf8Clean makes a string safe for a PostgreSQL text column. It (1) replaces // invalid UTF-8 byte sequences with U+FFFD and (2) strips NUL (0x00) bytes. // Tool output (nmap/curl/… stdout) can carry raw/truncated bytes that PostgreSQL's @@ -330,6 +335,437 @@ WHERE id=$2 AND exploration_id=$3 AND kind='intent' AND state=$4`, state, id, s. return n == 1, err } +const ( + IntentInterventionPending = "pending" + IntentInterventionAccepted = "accepted" + IntentInterventionRejected = "rejected" + PlannerEventWorkerMessage = "worker_message" +) + +// IntentIntervention is the durable control record for one human interruption. +// It deliberately lives in activity rather than a new table: activity already +// participates in task archive/restore, and the accepted row is also the user +// turn rendered in the intent transcript. +type IntentIntervention struct { + RequestID string + Message string + Status string + ActivityID int64 + IntentID int64 + IntentState string + CreatedAt time.Time +} + +// WorkerDirection is the latest accepted human direction for one Worker +// intent. The accepted activity is the durable planning event, so this view +// survives restarts and is shared by Planner and Main Agent context builders. +type WorkerDirection struct { + ActivityID int64 + IntentID int64 + RequestID string + Message string + IntentState string + IntentSummary string + CreatedAt time.Time +} + +// lockIntentIntervention serializes request-id reservation/finalization across +// processes. exploration_nodes.id is a global primary key, so the intent id is +// sufficient input to the namespaced advisory key. +func lockIntentIntervention(tx *sql.Tx, intentID int64) error { + _, err := tx.Exec(`SELECT pg_advisory_xact_lock(hashtextextended('artex-intent-intervention:' || $1::text, 0))`, intentID) + return err +} + +func scanIntentIntervention(row *sql.Row) (*IntentIntervention, error) { + var it IntentIntervention + err := row.Scan(&it.ActivityID, &it.RequestID, &it.Message, &it.Status, + &it.IntentID, &it.IntentState, &it.CreatedAt) + if err == sql.ErrNoRows { + return nil, nil + } + if err != nil { + return nil, err + } + return &it, nil +} + +// IntentInterventionByRequest finds a prior reservation/acceptance. Callers do +// this before ordinary lifecycle validation so retrying an already accepted +// request remains idempotent even if the worker has since finished naturally. +func (s *ExplorationStore) IntentInterventionByRequest(intentID int64, requestID string) (*IntentIntervention, error) { + return scanIntentIntervention(s.db.QueryRow(` +SELECT COALESCE(NULLIF(a.metadata->>'accepted_activity_id','')::bigint, a.id), + COALESCE(a.metadata->>'intervention_request_id',''), + COALESCE(a.metadata->>'intervention_message', a.detail, ''), + COALESCE(a.metadata->>'intervention_status',''), + n.id, n.state, a.created_at +FROM activity a +JOIN exploration_nodes n ON n.id=a.node_id AND n.exploration_id=a.exploration_id +WHERE a.exploration_id=$1 AND a.node_id=$2 + AND a.metadata->>'intervention_request_id'=$3 +ORDER BY a.id +LIMIT 1`, s.expID, intentID, requestID)) +} + +// ReserveIntentIntervention durably claims a request id while the intent is +// still running. A hidden intervention_control activity is the write-ahead +// record; acceptance inserts a newer visible kind='user' row so SSE cursors can +// never skip a message whose control reservation had an older id. +func (s *ExplorationStore) ReserveIntentIntervention(intentID int64, requestID, message string) (*IntentIntervention, error) { + tx, err := s.db.Begin() + if err != nil { + return nil, err + } + defer tx.Rollback() + if err := lockIntentIntervention(tx, intentID); err != nil { + return nil, err + } + + existing, err := scanIntentIntervention(tx.QueryRow(` +SELECT COALESCE(NULLIF(a.metadata->>'accepted_activity_id','')::bigint, a.id), + COALESCE(a.metadata->>'intervention_request_id',''), + COALESCE(a.metadata->>'intervention_message', a.detail, ''), + COALESCE(a.metadata->>'intervention_status',''), + n.id, n.state, a.created_at +FROM activity a +JOIN exploration_nodes n ON n.id=a.node_id AND n.exploration_id=a.exploration_id +WHERE a.exploration_id=$1 AND a.node_id=$2 + AND a.metadata->>'intervention_request_id'=$3 +ORDER BY a.id +LIMIT 1`, s.expID, intentID, requestID)) + if err != nil { + return nil, err + } + if existing != nil { + if existing.Message != message { + return nil, fmt.Errorf("%w: request_id 已用于不同的 Worker 消息", ErrIntentInterventionConflict) + } + if err := tx.Commit(); err != nil { + return nil, err + } + return existing, nil + } + + var state string + if err := tx.QueryRow(`SELECT state FROM exploration_nodes + WHERE id=$1 AND exploration_id=$2 AND kind='intent' FOR UPDATE`, intentID, s.expID).Scan(&state); err != nil { + if err == sql.ErrNoRows { + return nil, fmt.Errorf("%w: intent not found", ErrIntentInterventionConflict) + } + return nil, err + } + if state != "running" && state != "paused" { + return nil, fmt.Errorf("%w: intent state %s cannot receive a message", ErrIntentInterventionConflict, state) + } + var otherRequest string + err = tx.QueryRow(`SELECT COALESCE(metadata->>'intervention_request_id','') + FROM activity + WHERE exploration_id=$1 AND node_id=$2 + AND metadata->>'intervention_status'=$3 + ORDER BY id LIMIT 1`, s.expID, intentID, IntentInterventionPending).Scan(&otherRequest) + if err != nil && err != sql.ErrNoRows { + return nil, err + } + if err == nil { + return nil, fmt.Errorf("%w: intent 已有待处理的 Worker 消息 %s", ErrIntentInterventionConflict, otherRequest) + } + + metadata, err := json.Marshal(map[string]any{ + "intervention": true, + "intervention_handoff": "pending", + "intervention_request_id": requestID, + "intervention_message": message, + "intervention_status": IntentInterventionPending, + "source": "user", + }) + if err != nil { + return nil, err + } + var out IntentIntervention + err = tx.QueryRow(` +INSERT INTO activity(exploration_id,node_id,worker,kind,metadata) +VALUES ($1,$2,'system','intervention_control',$3) +RETURNING id, created_at`, s.expID, intentID, string(metadata)).Scan(&out.ActivityID, &out.CreatedAt) + if err != nil { + return nil, err + } + out.RequestID, out.Message, out.Status = requestID, message, IntentInterventionPending + out.IntentID, out.IntentState = intentID, state + if err := tx.Commit(); err != nil { + return nil, err + } + return &out, nil +} + +// AcceptIntentIntervention is the DB commit point: the hidden reservation +// becomes an audited user turn in the same transaction that moves paused back +// to open. Thus no Worker can claim the intent before its pending message and +// visible activity are both durable. +func (s *ExplorationStore) AcceptIntentIntervention(intentID int64, requestID, message string) (Activity, bool, error) { + var activity Activity + tx, err := s.db.Begin() + if err != nil { + return activity, false, err + } + defer tx.Rollback() + if err := lockIntentIntervention(tx, intentID); err != nil { + return activity, false, err + } + + var controlID int64 + var storedMessage, status, state, taskStatus string + var taskPaused, taskQueued bool + var taskDeletedAt sql.NullTime + var createdAt time.Time + err = tx.QueryRow(` +SELECT a.id, + COALESCE(a.metadata->>'intervention_message', a.detail, ''), + COALESCE(a.metadata->>'intervention_status',''), + n.state, a.created_at, COALESCE(t.status,''), t.paused, t.queued, t.deleted_at +FROM activity a +JOIN exploration_nodes n ON n.id=a.node_id AND n.exploration_id=a.exploration_id +JOIN tasks t ON t.exploration_id=a.exploration_id +WHERE a.exploration_id=$1 AND a.node_id=$2 + AND a.metadata->>'intervention_request_id'=$3 + ORDER BY a.id LIMIT 1 + FOR UPDATE OF a,n,t`, s.expID, intentID, requestID). + Scan(&controlID, &storedMessage, &status, &state, &createdAt, + &taskStatus, &taskPaused, &taskQueued, &taskDeletedAt) + if err == sql.ErrNoRows { + return activity, false, fmt.Errorf("%w: intervention reservation not found", ErrIntentInterventionConflict) + } + if err != nil { + return activity, false, err + } + if storedMessage != message { + return activity, false, fmt.Errorf("%w: request_id 已用于不同的 Worker 消息", ErrIntentInterventionConflict) + } + if status == IntentInterventionAccepted { + var acceptedID int64 + if err := tx.QueryRow(`SELECT COALESCE((metadata->>'accepted_activity_id')::bigint,0), + COALESCE((SELECT created_at FROM activity accepted + WHERE accepted.id=(activity.metadata->>'accepted_activity_id')::bigint), created_at) + FROM activity WHERE id=$1`, controlID).Scan(&acceptedID, &createdAt); err != nil { + return activity, false, err + } + if acceptedID <= 0 { + return activity, false, fmt.Errorf("accepted intervention has no activity id") + } + if err := tx.Commit(); err != nil { + return activity, false, err + } + nid := intentID + activity.ID = acceptedID + activity.NodeID, activity.Worker, activity.Kind = &nid, "user", "user" + activity.Summary, activity.Detail, activity.CreatedAt = message, message, createdAt + return activity, true, nil + } + if status != IntentInterventionPending { + return activity, false, fmt.Errorf("%w: intervention status %s cannot be accepted", ErrIntentInterventionConflict, status) + } + if state != "paused" { + return activity, false, fmt.Errorf("%w: intent state %s cannot be reopened", ErrIntentInterventionConflict, state) + } + if taskPaused || taskQueued || taskDeletedAt.Valid || IsTerminal(taskStatus) { + return activity, false, fmt.Errorf("%w: task lifecycle changed (status=%s paused=%v queued=%v deleted=%v)", + ErrIntentInterventionConflict, taskStatus, taskPaused, taskQueued, taskDeletedAt.Valid) + } + + metadata, err := json.Marshal(map[string]any{ + "intervention": true, + "intervention_request_id": requestID, + "intervention_message": message, + "intervention_status": IntentInterventionAccepted, + "planner_event": PlannerEventWorkerMessage, + "source": "user", + }) + if err != nil { + return activity, false, err + } + if err := tx.QueryRow(`INSERT INTO activity( + exploration_id,node_id,worker,kind,summary,detail,metadata) + VALUES ($1,$2,'user','user',$3,$3,$4) + RETURNING id,created_at`, s.expID, intentID, utf8Clean(message), string(metadata)). + Scan(&activity.ID, &createdAt); err != nil { + return activity, false, err + } + var acceptedMeta map[string]any + if err := json.Unmarshal(metadata, &acceptedMeta); err != nil { + return activity, false, err + } + acceptedMeta["accepted_activity_id"] = activity.ID + controlMetadata, err := json.Marshal(acceptedMeta) + if err != nil { + return activity, false, err + } + if _, err := tx.Exec(`UPDATE activity SET metadata=$1 WHERE id=$2`, string(controlMetadata), controlID); err != nil { + return activity, false, err + } + res, err := tx.Exec(`UPDATE exploration_nodes + SET state='open', owner=NULL, completed_at=NULL, blocked_reason=NULL + WHERE id=$1 AND exploration_id=$2 AND kind='intent' AND state='paused'`, intentID, s.expID) + if err != nil { + return activity, false, err + } + if n, err := res.RowsAffected(); err != nil || n != 1 { + if err != nil { + return activity, false, err + } + return activity, false, fmt.Errorf("%w: intent is no longer paused", ErrIntentInterventionConflict) + } + if err := tx.Commit(); err != nil { + return activity, false, err + } + nid := intentID + activity.NodeID, activity.Worker, activity.Kind = &nid, "user", "user" + activity.Summary, activity.Detail, activity.CreatedAt = message, message, createdAt + activity.Metadata = metadata + return activity, false, nil +} + +// LatestWorkerDirections returns at most one accepted human direction per +// local intent, newest directions first. It reads the visible user activity +// rather than the hidden hand-off row so archive/restore and activity history +// share one authoritative record. +func (s *ExplorationStore) LatestWorkerDirections(limit int) ([]WorkerDirection, error) { + if limit <= 0 { + limit = 300 + } + rows, err := s.db.Query(` +SELECT activity_id,intent_id,request_id,message,intent_state,intent_summary,created_at +FROM ( + SELECT DISTINCT ON (a.node_id) + a.id AS activity_id, + a.node_id AS intent_id, + COALESCE(a.metadata->>'intervention_request_id','') AS request_id, + COALESCE(a.metadata->>'intervention_message',a.detail,'') AS message, + n.state AS intent_state, + COALESCE(n.payload->>'summary','') AS intent_summary, + a.created_at + FROM activity a + JOIN exploration_nodes n ON n.id=a.node_id AND n.exploration_id=a.exploration_id + WHERE a.exploration_id=$1 + AND a.kind='user' + AND n.kind='intent' + AND a.metadata->>'intervention_status'=$2 + AND a.metadata->>'planner_event'=$3 + ORDER BY a.node_id,a.id DESC +) latest +ORDER BY activity_id DESC +LIMIT $4`, s.expID, IntentInterventionAccepted, PlannerEventWorkerMessage, limit) + if err != nil { + return nil, err + } + defer rows.Close() + out := make([]WorkerDirection, 0) + for rows.Next() { + var item WorkerDirection + if err := rows.Scan(&item.ActivityID, &item.IntentID, &item.RequestID, &item.Message, + &item.IntentState, &item.IntentSummary, &item.CreatedAt); err != nil { + return nil, err + } + out = append(out, item) + } + return out, rows.Err() +} + +// RecoverableIntentInterventions returns control operations that must be +// reconstructed before ordinary frontier claiming after a process restart: +// pending transcript commits, plus accepted hand-offs not yet claimed once. +func (s *ExplorationStore) RecoverableIntentInterventions() ([]IntentIntervention, error) { + rows, err := s.db.Query(` +SELECT COALESCE(NULLIF(a.metadata->>'accepted_activity_id','')::bigint, a.id), + COALESCE(a.metadata->>'intervention_request_id',''), + COALESCE(a.metadata->>'intervention_message',''), + COALESCE(a.metadata->>'intervention_status',''), + n.id,n.state,a.created_at +FROM activity a +JOIN exploration_nodes n ON n.id=a.node_id AND n.exploration_id=a.exploration_id +WHERE a.exploration_id=$1 AND a.kind='intervention_control' + AND (a.metadata->>'intervention_status'=$2 OR ( + a.metadata->>'intervention_status'=$3 + AND COALESCE(a.metadata->>'intervention_handoff','pending')='pending')) +ORDER BY a.id`, s.expID, IntentInterventionPending, IntentInterventionAccepted) + if err != nil { + return nil, err + } + defer rows.Close() + out := []IntentIntervention{} + for rows.Next() { + var item IntentIntervention + if err := rows.Scan(&item.ActivityID, &item.RequestID, &item.Message, &item.Status, + &item.IntentID, &item.IntentState, &item.CreatedAt); err != nil { + return nil, err + } + out = append(out, item) + } + return out, rows.Err() +} + +// MarkIntentInterventionHandoffClaimed records that the accepted instruction +// has been preferentially claimed once. Idempotency status remains accepted. +func (s *ExplorationStore) MarkIntentInterventionHandoffClaimed(intentID int64) error { + _, err := s.db.Exec(`UPDATE activity SET metadata=jsonb_set( + metadata,'{intervention_handoff}',to_jsonb('claimed'::text)) + WHERE id=(SELECT id FROM activity + WHERE exploration_id=$1 AND node_id=$2 AND kind='intervention_control' + AND metadata->>'intervention_status'=$3 + AND COALESCE(metadata->>'intervention_handoff','pending')='pending' + ORDER BY id DESC LIMIT 1)`, s.expID, intentID, IntentInterventionAccepted) + return err +} + +// PendingIntentInterventionMessage returns the accepted human chat turn that +// should be consumed by the next Worker run. The handoff remains pending until +// the run has actually started, so a process restart can reconstruct the +// directed claim without losing the user's message. +func (s *ExplorationStore) PendingIntentInterventionMessage(intentID int64) (string, string, bool, error) { + var requestID, message string + err := s.db.QueryRow(`SELECT + COALESCE(metadata->>'intervention_request_id',''), + COALESCE(metadata->>'intervention_message','') + FROM activity + WHERE exploration_id=$1 AND node_id=$2 AND kind='intervention_control' + AND metadata->>'intervention_status'=$3 + AND COALESCE(metadata->>'intervention_handoff','pending')='pending' + ORDER BY id DESC LIMIT 1`, s.expID, intentID, IntentInterventionAccepted). + Scan(&requestID, &message) + if err == sql.ErrNoRows { + return "", "", false, nil + } + if err != nil { + return "", "", false, err + } + return requestID, message, true, nil +} + +// RejectIntentIntervention closes a pending control reservation without making +// it visible in task activity. It is used when lifecycle validation changes or +// the transcript could not be made durable; a different future request is then +// free to proceed after the user explicitly resumes the paused intent. +func (s *ExplorationStore) RejectIntentIntervention(intentID int64, requestID, reason string) error { + tx, err := s.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + if err := lockIntentIntervention(tx, intentID); err != nil { + return err + } + _, err = tx.Exec(`UPDATE activity + SET metadata=jsonb_set(jsonb_set(metadata,'{intervention_status}',to_jsonb($1::text)), + '{intervention_reject_reason}',to_jsonb($2::text)) + WHERE exploration_id=$3 AND node_id=$4 + AND metadata->>'intervention_request_id'=$5 + AND metadata->>'intervention_status'=$6`, + IntentInterventionRejected, utf8Clean(reason), s.expID, intentID, requestID, IntentInterventionPending) + if err != nil { + return err + } + return tx.Commit() +} + // IntentCleanup summarizes the blackboard records removed when a user cancels // one worker intent. Global assets and traffic are deliberately not part of this // operation; exploration anchors disappear only because their owning nodes do. @@ -1006,7 +1442,8 @@ func (d *DB) TokenTotalsAll() (map[int64]TokenUsage, error) { // Persisted (unlike Engine.LastActivity's in-memory map), so it survives restarts // and gives终态任务 a stable "ran until" time for computing run duration. func (d *DB) LastActivityAll() (map[int64]int64, error) { - rows, err := d.Query(`SELECT exploration_id, EXTRACT(EPOCH FROM MAX(created_at))::bigint FROM activity GROUP BY exploration_id`) + rows, err := d.Query(`SELECT exploration_id, EXTRACT(EPOCH FROM MAX(created_at))::bigint + FROM activity WHERE kind IS DISTINCT FROM 'intervention_control' GROUP BY exploration_id`) if err != nil { return nil, err } @@ -1057,7 +1494,7 @@ func (d *DB) TaskListMetricsAll() (map[int64]TaskListMetrics, error) { LEFT JOIN LATERAL ( SELECT EXTRACT(EPOCH FROM created_at)::bigint AS created_at FROM activity - WHERE exploration_id=task.exploration_id + WHERE exploration_id=task.exploration_id AND kind IS DISTINCT FROM 'intervention_control' ORDER BY created_at DESC LIMIT 1 ) latest_activity ON true @@ -1188,10 +1625,10 @@ func (s *ExplorationStore) ActivityList(nodeID *int64, sinceID int64, limit int) const cols = `id, node_id, COALESCE(worker,''), COALESCE(kind,''), COALESCE(tool,''), COALESCE(tool_use_id,''), is_error, COALESCE(summary,''), metadata, created_at, input_tokens, output_tokens, cache_read_tokens, cache_write_tokens` if nodeID != nil { rows, err = s.db.Query(`SELECT `+cols+` -FROM activity WHERE exploration_id=$1 AND node_id=$2 AND id>$3 ORDER BY id LIMIT $4`, s.expID, *nodeID, sinceID, limit) +FROM activity WHERE exploration_id=$1 AND node_id=$2 AND kind IS DISTINCT FROM 'intervention_control' AND id>$3 ORDER BY id LIMIT $4`, s.expID, *nodeID, sinceID, limit) } else { rows, err = s.db.Query(`SELECT `+cols+` -FROM activity WHERE exploration_id=$1 AND id>$2 ORDER BY id LIMIT $3`, s.expID, sinceID, limit) +FROM activity WHERE exploration_id=$1 AND kind IS DISTINCT FROM 'intervention_control' AND id>$2 ORDER BY id LIMIT $3`, s.expID, sinceID, limit) } if err != nil { return nil, sinceID, err @@ -1228,7 +1665,7 @@ func (s *ExplorationStore) ActivityListForTerminalIntent(nodeID, sinceID int64, JOIN exploration_nodes n ON n.id=a.node_id AND n.exploration_id=a.exploration_id WHERE a.exploration_id=$1 AND a.node_id=$2 AND a.id>$3 AND n.kind='intent' AND n.state IN ('done','blocked','exhausted','stopped') - AND a.kind NOT IN ('thinking','usage') + AND a.kind NOT IN ('thinking','usage','intervention_control') ORDER BY a.id LIMIT $4`, s.expID, nodeID, sinceID, limit) if err != nil { return nil, sinceID, err @@ -1291,7 +1728,7 @@ func (s *ExplorationStore) ActivityPage(f ActivitySessionFilter, before int64, l limitArg := len(args) + 1 args = append(args, limit+1) // one extra row to detect older history q := fmt.Sprintf(`SELECT `+cols+` -FROM activity WHERE exploration_id=$1%s AND ($%d <= 0 OR id < $%d) +FROM activity WHERE exploration_id=$1 AND kind IS DISTINCT FROM 'intervention_control'%s AND ($%d <= 0 OR id < $%d) ORDER BY id DESC LIMIT $%d`, cond, beforeArg, beforeArg, limitArg) rows, err := s.db.Query(q, args...) if err != nil { @@ -1336,7 +1773,7 @@ func (s *ExplorationStore) ActivityPageForTerminalIntent(nodeID, before int64, l JOIN exploration_nodes n ON n.id=a.node_id AND n.exploration_id=a.exploration_id WHERE a.exploration_id=$1 AND a.node_id=$2 AND n.kind='intent' AND n.state IN ('done','blocked','exhausted','stopped') - AND a.kind NOT IN ('thinking','usage') + AND a.kind NOT IN ('thinking','usage','intervention_control') AND ($3 <= 0 OR a.id < $3) ORDER BY a.id DESC LIMIT $4`, s.expID, nodeID, before, limit+1) if err != nil { @@ -1372,7 +1809,7 @@ func (s *ExplorationStore) ActivityPageForTerminalIntent(nodeID, before int64, l // session, since one SSE covers the whole task. func (s *ExplorationStore) ActivityMaxID() (int64, error) { var max sql.NullInt64 - err := s.db.QueryRow(`SELECT MAX(id) FROM activity WHERE exploration_id=$1`, s.expID).Scan(&max) + err := s.db.QueryRow(`SELECT MAX(id) FROM activity WHERE exploration_id=$1 AND kind IS DISTINCT FROM 'intervention_control'`, s.expID).Scan(&max) if err != nil { return 0, err } @@ -1382,7 +1819,7 @@ func (s *ExplorationStore) ActivityMaxID() (int64, error) { // ActivityDetail lazily returns the full detail blob for one step. func (s *ExplorationStore) ActivityDetail(id int64) (string, error) { var d sql.NullString - err := s.db.QueryRow(`SELECT detail FROM activity WHERE id=$1 AND exploration_id=$2`, id, s.expID).Scan(&d) + err := s.db.QueryRow(`SELECT detail FROM activity WHERE id=$1 AND exploration_id=$2 AND kind IS DISTINCT FROM 'intervention_control'`, id, s.expID).Scan(&d) if err == sql.ErrNoRows { return "", nil } @@ -1415,7 +1852,7 @@ func (s *ExplorationStore) ActivityTrace(nodeID int64, limit int) ([]Activity, e limit = 500 } rows, err := s.db.Query(`SELECT `+traceCols+` -FROM activity WHERE exploration_id=$1 AND node_id=$2 AND kind NOT IN ('thinking','usage') +FROM activity WHERE exploration_id=$1 AND node_id=$2 AND kind NOT IN ('thinking','usage','intervention_control') ORDER BY id LIMIT $3`, s.expID, nodeID, limit) if err != nil { return nil, err @@ -1435,7 +1872,7 @@ func (s *ExplorationStore) ActivityTraceForTerminalIntent(nodeID int64, limit in JOIN exploration_nodes n ON n.id=a.node_id AND n.exploration_id=a.exploration_id WHERE a.exploration_id=$1 AND a.node_id=$2 AND n.kind='intent' AND n.state IN ('done','blocked','exhausted','stopped') - AND a.kind NOT IN ('thinking','usage') + AND a.kind NOT IN ('thinking','usage','intervention_control') ORDER BY a.id LIMIT $3`, s.expID, nodeID, limit) if err != nil { return nil, err @@ -1457,11 +1894,11 @@ func (s *ExplorationStore) ActivityTraceSearch(nodeID *int64, q string, limit in var err error if nodeID != nil { rows, err = s.db.Query(`SELECT `+traceCols+` -FROM activity WHERE exploration_id=$1 AND node_id=$2 AND kind NOT IN ('thinking','usage') +FROM activity WHERE exploration_id=$1 AND node_id=$2 AND kind NOT IN ('thinking','usage','intervention_control') AND (summary ILIKE $3 OR detail ILIKE $3) ORDER BY id LIMIT $4`, s.expID, *nodeID, like, limit) } else { rows, err = s.db.Query(`SELECT `+traceCols+` -FROM activity WHERE exploration_id=$1 AND node_id IS NOT NULL AND kind NOT IN ('thinking','usage') +FROM activity WHERE exploration_id=$1 AND node_id IS NOT NULL AND kind NOT IN ('thinking','usage','intervention_control') AND (summary ILIKE $2 OR detail ILIKE $2) ORDER BY id LIMIT $3`, s.expID, like, limit) } if err != nil { @@ -1483,7 +1920,7 @@ func (s *ExplorationStore) ActivityTraceSearchForTerminalIntent(nodeID int64, q JOIN exploration_nodes n ON n.id=a.node_id AND n.exploration_id=a.exploration_id WHERE a.exploration_id=$1 AND a.node_id=$2 AND n.kind='intent' AND n.state IN ('done','blocked','exhausted','stopped') - AND a.kind NOT IN ('thinking','usage') + AND a.kind NOT IN ('thinking','usage','intervention_control') AND (a.summary ILIKE $3 OR a.detail ILIKE $3) ORDER BY a.id LIMIT $4`, s.expID, nodeID, like, limit) if err != nil { @@ -1505,7 +1942,7 @@ func (s *ExplorationStore) ActivityTraceSearchTerminalIntents(q string, limit in JOIN exploration_nodes n ON n.id=a.node_id AND n.exploration_id=a.exploration_id WHERE a.exploration_id=$1 AND n.kind='intent' AND n.state IN ('done','blocked','exhausted','stopped') - AND a.kind NOT IN ('thinking','usage') + AND a.kind NOT IN ('thinking','usage','intervention_control') AND (a.summary ILIKE $2 OR a.detail ILIKE $2) ORDER BY a.id LIMIT $3`, s.expID, like, limit) if err != nil { @@ -1576,7 +2013,7 @@ func (s *ExplorationStore) ActivityTraceSearchExcluding(excludeNodeID int64, q s like := "%" + q + "%" rows, err := s.db.Query(`SELECT `+traceCols+` FROM activity WHERE exploration_id=$1 AND node_id IS NOT NULL AND node_id <> $2 -AND kind NOT IN ('thinking','usage') +AND kind NOT IN ('thinking','usage','intervention_control') AND (summary ILIKE $3 OR detail ILIKE $3) ORDER BY id LIMIT $4`, s.expID, excludeNodeID, like, limit) if err != nil { return nil, err @@ -1600,7 +2037,7 @@ func (s *ExplorationStore) ActivityByIDs(ids []int64) ([]Activity, error) { args[i+1] = id } rows, err := s.db.Query(`SELECT id, node_id, COALESCE(kind,''), COALESCE(tool,''), is_error, COALESCE(detail,'') -FROM activity WHERE exploration_id=$1 AND kind<>'thinking' AND id IN (`+strings.Join(ph, ",")+`) ORDER BY id`, args...) +FROM activity WHERE exploration_id=$1 AND kind NOT IN ('thinking','intervention_control') AND id IN (`+strings.Join(ph, ",")+`) ORDER BY id`, args...) if err != nil { return nil, err } @@ -1634,7 +2071,7 @@ func (s *ExplorationStore) ActivityByIDsForTerminalIntents(ids []int64) ([]Activ rows, err := s.db.Query(`SELECT a.id, a.node_id, COALESCE(a.kind,''), COALESCE(a.tool,''), a.is_error, COALESCE(a.detail,'') FROM activity a JOIN exploration_nodes n ON n.id=a.node_id AND n.exploration_id=a.exploration_id - WHERE a.exploration_id=$1 AND a.kind NOT IN ('thinking','usage') AND n.kind='intent' + WHERE a.exploration_id=$1 AND a.kind NOT IN ('thinking','usage','intervention_control') AND n.kind='intent' AND n.state IN ('done','blocked','exhausted','stopped') AND a.id IN (`+strings.Join(ph, ",")+`) ORDER BY a.id`, args...) if err != nil { diff --git a/db/intent_intervention_lifecycle_test.go b/db/intent_intervention_lifecycle_test.go new file mode 100644 index 0000000..c27a746 --- /dev/null +++ b/db/intent_intervention_lifecycle_test.go @@ -0,0 +1,204 @@ +package db + +import ( + "encoding/json" + "errors" + "sync" + "testing" +) + +func newRunningInterventionIntent(t *testing.T, d *DB, label string) (*ExplorationStore, int64, func()) { + t.Helper() + task, err := d.CreateTask("worker intervention "+label, "verify intervention protocol", nil, 0, 0) + if err != nil { + t.Fatal(err) + } + store := d.Exploration(task.ExplorationID) + intentID, err := store.AddIntent(map[string]any{"summary": label}, 1, nil, "planner") + if err != nil { + _ = d.DeleteTask(task.ID) + t.Fatal(err) + } + if claimed, err := store.ClaimIntent(intentID, "work#1"); err != nil || !claimed { + _ = d.DeleteTask(task.ID) + t.Fatalf("claim intervention intent: claimed=%v err=%v", claimed, err) + } + return store, intentID, func() { _ = d.DeleteTask(task.ID) } +} + +func TestIntentInterventionReservationAndAcceptanceAreIdempotent(t *testing.T) { + d, err := Open(testDSN(t)) + if err != nil { + t.Skipf("postgres unavailable (%v) - skipping", err) + } + defer d.Close() + store, intentID, cleanup := newRunningInterventionIntent(t, d, "idempotent") + defer cleanup() + + const requestID = "intervention-idempotent-1" + const message = "停止枚举，优先验证登录越权" + reserved, err := store.ReserveIntentIntervention(intentID, requestID, message) + if err != nil { + t.Fatal(err) + } + if reserved.Status != IntentInterventionPending || reserved.ActivityID <= 0 || reserved.IntentState != "running" { + t.Fatalf("unexpected reservation: %+v", reserved) + } + + replayed, err := store.ReserveIntentIntervention(intentID, requestID, message) + if err != nil { + t.Fatal(err) + } + if replayed.ActivityID != reserved.ActivityID || replayed.Status != IntentInterventionPending { + t.Fatalf("reservation retry created a different operation: first=%+v retry=%+v", reserved, replayed) + } + if _, err := store.ReserveIntentIntervention(intentID, requestID, "改成另一条 Worker 消息"); !errors.Is(err, ErrIntentInterventionConflict) { + t.Fatalf("same request_id with different message error=%v, want intervention conflict", err) + } + if _, err := store.ReserveIntentIntervention(intentID, "intervention-other", message); !errors.Is(err, ErrIntentInterventionConflict) { + t.Fatalf("second pending request error=%v, want intervention conflict", err) + } + + if changed, err := store.CompareAndSetIntentState(intentID, "running", "paused"); err != nil || !changed { + t.Fatalf("settle intent before acceptance: changed=%v err=%v", changed, err) + } + activity, duplicate, err := store.AcceptIntentIntervention(intentID, requestID, message) + if err != nil || duplicate { + t.Fatalf("accept intervention: duplicate=%v err=%v", duplicate, err) + } + if activity.ID <= reserved.ActivityID || activity.NodeID == nil || *activity.NodeID != intentID || + activity.Worker != "user" || activity.Kind != "user" || activity.Summary != message || activity.Detail != message { + t.Fatalf("accepted audit mismatch: %+v", activity) + } + var metadata map[string]any + if err := json.Unmarshal(activity.Metadata, &metadata); err != nil { + t.Fatal(err) + } + if metadata["intervention_request_id"] != requestID || metadata["intervention_status"] != IntentInterventionAccepted { + t.Fatalf("accepted metadata=%v", metadata) + } + node, err := store.GetNode(intentID) + if err != nil || node == nil || node.State != "open" || node.Owner != "" { + t.Fatalf("accepted intent=%+v err=%v, want reopened and unowned", node, err) + } + + again, duplicate, err := store.AcceptIntentIntervention(intentID, requestID, message) + if err != nil || !duplicate || again.ID != activity.ID { + t.Fatalf("accept retry: activity=%+v duplicate=%v err=%v", again, duplicate, err) + } + lookup, err := store.IntentInterventionByRequest(intentID, requestID) + if err != nil || lookup == nil || lookup.Status != IntentInterventionAccepted || lookup.ActivityID != activity.ID { + t.Fatalf("accepted lookup=%+v err=%v", lookup, err) + } + var rows int + if err := d.QueryRow(`SELECT COUNT(*) FROM activity + WHERE exploration_id=$1 AND node_id=$2 + AND metadata->>'intervention_request_id'=$3`, store.ID(), intentID, requestID).Scan(&rows); err != nil { + t.Fatal(err) + } + if rows != 2 { + t.Fatalf("Worker message rows=%d, want one hidden control row and one visible user row", rows) + } + activities, cursor, err := store.ActivityList(&intentID, 0, 10) + if err != nil { + t.Fatal(err) + } + if len(activities) != 1 || activities[0].ID != activity.ID || cursor != activity.ID { + t.Fatalf("public activities leaked control row: activities=%+v cursor=%d", activities, cursor) + } + maxID, err := store.ActivityMaxID() + if err != nil || maxID != activity.ID { + t.Fatalf("public activity cursor=%d err=%v, want visible user row %d", maxID, err, activity.ID) + } + if detail, err := store.ActivityDetail(reserved.ActivityID); err != nil || detail != "" { + t.Fatalf("hidden control detail=%q err=%v", detail, err) + } +} + +func TestIntentInterventionConcurrentReservationHasSingleDurableWinner(t *testing.T) { + d, err := Open(testDSN(t)) + if err != nil { + t.Skipf("postgres unavailable (%v) - skipping", err) + } + defer d.Close() + store, intentID, cleanup := newRunningInterventionIntent(t, d, "concurrent") + defer cleanup() + + const contenders = 12 + const requestID = "intervention-concurrent-1" + const message = "只保留被动请求并验证当前假设" + start := make(chan struct{}) + results := make(chan *IntentIntervention, contenders) + errs := make(chan error, contenders) + var wg sync.WaitGroup + for range contenders { + wg.Add(1) + go func() { + defer wg.Done() + <-start + reservation, reserveErr := store.ReserveIntentIntervention(intentID, requestID, message) + results <- reservation + errs <- reserveErr + }() + } + close(start) + wg.Wait() + close(results) + close(errs) + for err := range errs { + if err != nil { + t.Fatalf("concurrent reservation: %v", err) + } + } + var activityID int64 + for result := range results { + if result == nil || result.ActivityID <= 0 { + t.Fatalf("invalid concurrent reservation: %+v", result) + } + if activityID == 0 { + activityID = result.ActivityID + } else if result.ActivityID != activityID { + t.Fatalf("concurrent requests produced activity ids %d and %d", activityID, result.ActivityID) + } + } + var rows int + if err := d.QueryRow(`SELECT COUNT(*) FROM activity + WHERE exploration_id=$1 AND node_id=$2 + AND metadata->>'intervention_request_id'=$3`, store.ID(), intentID, requestID).Scan(&rows); err != nil { + t.Fatal(err) + } + if rows != 1 { + t.Fatalf("concurrent intervention rows=%d, want 1", rows) + } +} + +func TestIntentInterventionAcceptanceCanRetryAfterStateSettlement(t *testing.T) { + d, err := Open(testDSN(t)) + if err != nil { + t.Skipf("postgres unavailable (%v) - skipping", err) + } + defer d.Close() + store, intentID, cleanup := newRunningInterventionIntent(t, d, "settlement retry") + defer cleanup() + + const requestID = "intervention-settle-1" + const message = "立刻停止当前动作并检查权限边界" + reserved, err := store.ReserveIntentIntervention(intentID, requestID, message) + if err != nil { + t.Fatal(err) + } + if _, _, err := store.AcceptIntentIntervention(intentID, requestID, message); !errors.Is(err, ErrIntentInterventionConflict) { + t.Fatalf("accept before worker settlement error=%v, want conflict", err) + } + pending, err := store.IntentInterventionByRequest(intentID, requestID) + if err != nil || pending == nil || pending.Status != IntentInterventionPending || pending.ActivityID != reserved.ActivityID { + t.Fatalf("failed acceptance lost reservation: pending=%+v err=%v", pending, err) + } + if changed, err := store.CompareAndSetIntentState(intentID, "running", "paused"); err != nil || !changed { + t.Fatalf("settle intent: changed=%v err=%v", changed, err) + } + activity, duplicate, err := store.AcceptIntentIntervention(intentID, requestID, message) + if err != nil || duplicate || activity.ID <= reserved.ActivityID { + t.Fatalf("retry acceptance: activity=%+v duplicate=%v err=%v", activity, duplicate, err) + } +} diff --git a/server/engine.go b/server/engine.go index f3c7074..2754c09 100644 --- a/server/engine.go +++ b/server/engine.go @@ -75,6 +75,7 @@ const ( ) var errWorkControlConflict = errors.New("work control conflict") +var errWorkInterventionConflict = errors.New("work intervention conflict") // retryableWorkerModelError excludes errors already handled by the task router. // In particular, a quota error after partial streaming advances the task cursor @@ -130,6 +131,17 @@ type Engine struct { // runWorkerStep has stopped writing and committed its final state. workMu sync.Mutex work map[int64]*workExecution + // interventionLocks serializes reservation, cancellation and message hand-off + // per intent. PostgreSQL separately serializes the durable transaction steps. + interventionLocks sync.Map // intentID -> *sync.Mutex + + // directedClaims keeps intents with a new human message ahead of the normal + // frontier without mutating their persisted priority. runWorkerStep installs + // the entry before releasing ControlWork, so a single worker slot cannot grab + // unrelated work in the paused->open hand-off gap. + directedMu sync.Mutex + directedClaims map[string][]int64 + directedReady map[int64]bool // steerBox queues planner course-corrections for a running work (keyed by intent // id). The worker's PreToolUse hook drains it before its next tool call and hands @@ -162,7 +174,14 @@ type taskRuntime struct { type workExecution struct { cancel context.CancelCauseFunc done chan error - action string // user action: pause | cancel + action string // user action: pause | cancel | intervene +} + +type WorkInterventionResult struct { + RequestID string + ActivityID int64 + State string + Duplicate bool } // nextPlannerRound returns the next planner round number for a task (1-based). @@ -285,6 +304,12 @@ func (e *Engine) StopTask(taskID string) { e.stamped.Delete(taskID) e.inflight.Delete(taskID) e.coordStarted.Delete(taskID) + e.directedMu.Lock() + for _, intentID := range e.directedClaims[taskID] { + delete(e.directedReady, intentID) + } + delete(e.directedClaims, taskID) + e.directedMu.Unlock() e.deleteMu.Lock() e.deleting.Delete(taskID) e.deleteMu.Unlock() @@ -390,7 +415,7 @@ func NewEngine(m *Manager) *Engine { return &Engine{m: m, debounce: 800 * time.Millisecond, bc: NewBroadcaster(), execCancel: map[string]context.CancelCauseFunc{}, execCtx: map[string]context.Context{}, work: map[int64]*workExecution{}, steerBox: map[int64][]string{}, - runtimes: map[string]*taskRuntime{}} + directedClaims: map[string][]int64{}, directedReady: map[int64]bool{}, runtimes: map[string]*taskRuntime{}} } // registerWork records the cancel for the work currently running intentID. @@ -425,7 +450,7 @@ func (e *Engine) detachWork(intentID int64) (action string, complete func(error) // worker has fully stopped writing. Cancellation cleanup is performed by the API // handler after this returns; pause state is committed by runWorkerStep itself. func (e *Engine) ControlWork(ctx context.Context, intentID int64, action string) error { - if action != "pause" && action != "cancel" { + if action != "pause" && action != "cancel" && action != "intervene" { return fmt.Errorf("unsupported work action %q", action) } if ctx == nil { @@ -444,8 +469,11 @@ func (e *Engine) ControlWork(ctx context.Context, intentID int64, action string) run.action = action done := run.done cause := error(agent.AbortWorkPausedByUser) - if action == "cancel" { + switch action { + case "cancel": cause = agent.AbortWorkCancelledByUser + case "intervene": + cause = agent.AbortWorkIntervenedByUser } run.cancel(cause) e.workMu.Unlock() @@ -464,6 +492,287 @@ func (e *Engine) ControlWork(ctx context.Context, intentID int64, action string) } } +func (e *Engine) lockWorkIntervention(intentID int64) func() { + value, _ := e.interventionLocks.LoadOrStore(intentID, &sync.Mutex{}) + mu := value.(*sync.Mutex) + mu.Lock() + return mu.Unlock +} + +func (e *Engine) validateWorkInterventionTask(t *Task, allowAdmissionPause bool) error { + if t == nil { + return fmt.Errorf("%w: task not found", errWorkInterventionConflict) + } + lifecycle := t.lifecycleSnapshot() + switch { + case e.IsDeleting(t.ID): + return fmt.Errorf("%w: 任务正在删除，无法向 Worker 发送消息", errWorkInterventionConflict) + case lifecycle.Paused || (!allowAdmissionPause && e.IsPaused(t.ID)): + return fmt.Errorf("%w: 任务已暂停，无法向 Worker 发送消息", errWorkInterventionConflict) + case lifecycle.Queued: + return fmt.Errorf("%w: 排队中的任务无法向 Worker 发送消息", errWorkInterventionConflict) + case isTerminalStatus(lifecycle.Status): + return fmt.Errorf("%w: 终态任务无法向 Worker 发送消息", errWorkInterventionConflict) + case e.isSettling(t.ID): + return fmt.Errorf("%w: 任务正在收尾，无法向 Worker 发送消息", errWorkInterventionConflict) + default: + return nil + } +} + +func (e *Engine) abandonWorkIntervention(t *Task, intentID int64, requestID string, reason error) { + e.removeDirectedClaim(t.ID, intentID) + target := "open" + lifecycle := t.lifecycleSnapshot() + if e.IsDeleting(t.ID) || isTerminalStatus(lifecycle.Status) { + target = "stopped" + } else if e.isSettling(t.ID) { + target = "exhausted" + } + if changed, err := t.Store.CompareAndSetIntentState(intentID, "paused", target); err != nil { + log.Printf("[worker intervention] task %s 意图 #%d 补偿 paused->%s 失败: %v", t.ID, intentID, target, err) + } else if changed && target == "open" { + t.NotifyWorker() + } + if err := t.Store.RejectIntentIntervention(intentID, requestID, reason.Error()); err != nil { + log.Printf("[worker intervention] task %s 意图 #%d 标记请求 %s rejected 失败: %v", t.ID, intentID, requestID, err) + } +} + +// InterveneWork performs the complete Worker message protocol. Once the request +// has a durable pending reservation it no longer depends on the HTTP request +// context: the caller supplies a server-owned bounded context so disconnects do +// not strand the intent. The accepted DB transition is running -> paused -> +// open, with the visible user message and reopen committed atomically. +func (e *Engine) InterveneWork(ctx context.Context, t *Task, intentID int64, requestID, message string) (WorkInterventionResult, error) { + return e.interveneWork(ctx, t, intentID, requestID, message, false) +} + +func (e *Engine) interveneWork(ctx context.Context, t *Task, intentID int64, requestID, message string, allowAdmissionPause bool) (WorkInterventionResult, error) { + out := WorkInterventionResult{RequestID: requestID} + if t == nil || t.Store == nil { + return out, fmt.Errorf("%w: task not found", errWorkInterventionConflict) + } + if ctx == nil { + ctx = context.Background() + } + unlock := e.lockWorkIntervention(intentID) + defer unlock() + + existing, err := t.Store.IntentInterventionByRequest(intentID, requestID) + if err != nil { + return out, err + } + if existing != nil { + if existing.Message != message { + return out, fmt.Errorf("%w: request_id 已用于不同的 Worker 消息", errWorkInterventionConflict) + } + if existing.Status == db.IntentInterventionAccepted { + out.ActivityID, out.State, out.Duplicate = existing.ActivityID, existing.IntentState, true + if out.State == "open" && e.validateWorkInterventionTask(t, allowAdmissionPause) == nil { + e.queueDirectedClaim(t.ID, intentID) + e.readyDirectedClaim(intentID) + t.NotifyWorker() + } + return out, nil + } + if existing.Status != db.IntentInterventionPending { + return out, fmt.Errorf("%w: Worker 消息请求状态为 %s", errWorkInterventionConflict, existing.Status) + } + } + if err := e.validateWorkInterventionTask(t, allowAdmissionPause); err != nil { + lifecycle := t.lifecycleSnapshot() + if existing != nil && (e.IsDeleting(t.ID) || e.isSettling(t.ID) || isTerminalStatus(lifecycle.Status)) { + e.abandonWorkIntervention(t, intentID, requestID, err) + } + return out, err + } + _, worker := e.snapshotFor(t) + if worker == nil { + return out, fmt.Errorf("%w: worker 尚未就绪", errWorkInterventionConflict) + } + if existing == nil { + existing, err = t.Store.ReserveIntentIntervention(intentID, requestID, message) + if err != nil { + if errors.Is(err, db.ErrIntentInterventionConflict) { + return out, fmt.Errorf("%w: %v", errWorkInterventionConflict, err) + } + return out, err + } + } + out.ActivityID = existing.ActivityID + + // A pending reservation can be recovered after a process restart. An open + // row is fenced back to paused before publishing the user turn; a running row + // is stopped through the normal live-work protocol so no late writer races us. + for { + node, getErr := t.Store.GetNode(intentID) + if getErr != nil { + return out, getErr + } + if node == nil { + return out, fmt.Errorf("%w: intent not found", errWorkInterventionConflict) + } + switch node.State { + case "running": + controlErr := e.ControlWork(ctx, intentID, "intervene") + if controlErr != nil { + // The per-work 30s wait may expire before the server-owned operation + // context. Keep waiting for the named cancellation to settle; timeout + // release marks the run no-handoff so a late exit cannot install a + // permanent directed-claim barrier behind our back. A freshly claimed + // intent can also be running just before registerWork; retrying closes + // that small claim-to-registration window. + for { + settled, readErr := t.Store.GetNode(intentID) + if readErr != nil { + return out, readErr + } + if settled != nil && settled.State == "paused" { + break + } + if ctx.Err() != nil || settled == nil || settled.State != "running" { + e.removeDirectedClaim(t.ID, intentID) + return out, fmt.Errorf("%w: %v", errWorkInterventionConflict, controlErr) + } + if sleepCtx(ctx, 50*time.Millisecond) { + e.removeDirectedClaim(t.ID, intentID) + return out, fmt.Errorf("%w: %v", errWorkInterventionConflict, controlErr) + } + if errors.Is(controlErr, errWorkControlConflict) { + controlErr = e.ControlWork(ctx, intentID, "intervene") + } + } + } + case "open": + changed, setErr := t.Store.CompareAndSetIntentState(intentID, "open", "paused") + if setErr != nil { + return out, setErr + } + if !changed { + continue + } + e.queueDirectedClaim(t.ID, intentID) + case "paused": + e.queueDirectedClaim(t.ID, intentID) + default: + e.abandonWorkIntervention(t, intentID, requestID, + fmt.Errorf("意图已进入 %s，无法发送新消息", node.State)) + return out, fmt.Errorf("%w: 意图已进入 %s，无法发送新消息", errWorkInterventionConflict, node.State) + } + break + } + e.queueDirectedClaim(t.ID, intentID) + + // Close lifecycle races that began while cancellation was settling. A task + // pause or queue preserves the pending message and paused intent for a later + // admission; terminal, settling and deleting transitions close the request. + if err := e.validateWorkInterventionTask(t, allowAdmissionPause); err != nil { + // The user message has not entered the transcript yet. Preserve the + // reservation and paused intent so the same request can be retried after a + // transient task pause; do not reopen it behind a terminal transition. + e.removeDirectedClaim(t.ID, intentID) + lifecycle := t.lifecycleSnapshot() + if e.IsDeleting(t.ID) || e.isSettling(t.ID) || isTerminalStatus(lifecycle.Status) { + e.abandonWorkIntervention(t, intentID, requestID, err) + } + return out, err + } + activity, duplicate, err := t.Store.AcceptIntentIntervention(intentID, requestID, message) + if err != nil { + // Keep this intent paused so a retry of the same request can safely finish; + // remove only the task-wide claim barrier so unrelated workers keep moving. + e.removeDirectedClaim(t.ID, intentID) + return out, err + } + if !duplicate { + e.publishStoredActivity(t, activity) + t.NotifyWorkerMessage(intentID, activity.ID, message) + } + e.readyDirectedClaim(intentID) + t.NotifyWorker() + node, err := t.Store.GetNode(intentID) + if err != nil { + return out, err + } + out.Duplicate = duplicate + if node != nil { + out.State = node.State + } + return out, nil +} + +// RecoverWorkerMessages reconstructs unfinished Worker chat handoffs before +// ordinary frontier claiming. Pending messages are fenced in paused until an +// agent runtime is available; accepted messages regain their directed priority +// and are consumed by ExecuteWithMessage on the next claim. +func (e *Engine) RecoverWorkerMessages(ctx context.Context, t *Task) error { + return e.recoverWorkerMessages(ctx, t, false) +} + +// recoverWorkerMessagesForAdmission runs while the scheduler still owns the +// Engine pause barrier but has already committed a runnable task lifecycle. It +// restores directed hand-offs before Resume wakes any worker, so ordinary +// frontier work cannot overtake the human message. +func (e *Engine) recoverWorkerMessagesForAdmission(ctx context.Context, t *Task) error { + return e.recoverWorkerMessages(ctx, t, true) +} + +func (e *Engine) recoverWorkerMessages(ctx context.Context, t *Task, allowAdmissionPause bool) error { + if t == nil || t.Store == nil { + return nil + } + items, err := t.Store.RecoverableIntentInterventions() + if err != nil { + return err + } + lifecycle := t.lifecycleSnapshot() + runnable := !e.IsDeleting(t.ID) && !e.isSettling(t.ID) && !isTerminalStatus(lifecycle.Status) && + !lifecycle.Paused && !lifecycle.Queued && (allowAdmissionPause || !e.IsPaused(t.ID)) + if !runnable { + for _, item := range items { + e.removeDirectedClaim(t.ID, item.IntentID) + } + return nil + } + for _, item := range items { + switch item.Status { + case db.IntentInterventionPending: + if item.IntentState == "open" { + if changed, setErr := t.Store.CompareAndSetIntentState(item.IntentID, "open", "paused"); setErr != nil { + return setErr + } else if changed { + item.IntentState = "paused" + } + } + if _, worker := e.snapshotFor(t); worker == nil { + continue + } + if _, recoverErr := e.interveneWork(ctx, t, item.IntentID, item.RequestID, item.Message, allowAdmissionPause); recoverErr != nil { + log.Printf("[worker chat] task %s 恢复意图 #%d 待处理消息失败: %v", t.ID, item.IntentID, recoverErr) + } + case db.IntentInterventionAccepted: + // Reconstruct the Planner-facing event while the Worker hand-off is still + // pending. The durable graph_overview view remains authoritative even if + // this notification was already consumed before a later restart. + t.NotifyWorkerMessage(item.IntentID, item.ActivityID, item.Message) + if item.IntentState == "paused" { + if changed, setErr := t.Store.CompareAndSetIntentState(item.IntentID, "paused", "open"); setErr != nil { + return setErr + } else if changed { + item.IntentState = "open" + } + } + if item.IntentState == "open" { + e.queueDirectedClaim(t.ID, item.IntentID) + e.readyDirectedClaim(item.IntentID) + t.NotifyWorker() + } + } + } + return nil +} + // releaseWorkControl drops only this caller's reservation after its wait is // cancelled. The work context stays cancelled; runWorkerStep recognizes the // named cancellation cause and settles the intent into the recoverable paused @@ -471,7 +780,11 @@ func (e *Engine) ControlWork(ctx context.Context, intentID int64, action string) func (e *Engine) releaseWorkControl(intentID int64, run *workExecution, action string) { e.workMu.Lock() if current := e.work[intentID]; current == run && current.action == action { - current.action = "" + if action == "intervene" { + current.action = "intervene_no_handoff" + } else { + current.action = "" + } } e.workMu.Unlock() } @@ -610,6 +923,19 @@ func (e *Engine) emitActivity(t *Task, r db.Activity) db.Activity { return r } +// publishStoredActivity broadcasts an activity inserted by a larger DB +// transaction, avoiding a second INSERT through emitActivity. +func (e *Engine) publishStoredActivity(t *Task, r db.Activity) { + if t == nil || r.ID <= 0 { + return + } + if r.CreatedAt.IsZero() { + r.CreatedAt = time.Now() + } + e.bc.Publish(t.ID, r) + e.touch(t.ID) +} + // appendActivity persists one activity row, retrying briefly on write failure. // Concurrent planner + worker inserts into the same exploration's activity log // occasionally fail; a couple of quick retries recover most. Crucially, every @@ -876,13 +1202,13 @@ func (e *Engine) workerLoop(ctx context.Context, t *Task, name string) { } _, worker := e.snapshotFor(t) if worker == nil { - if sleepCtx(ctx, 1500*time.Millisecond) { + if waitWorker(ctx, t, 1500*time.Millisecond) { return } continue } if e.IsPaused(t.ID) { - if sleepCtx(ctx, 1000*time.Millisecond) { + if waitWorker(ctx, t, 1000*time.Millisecond) { return } continue // user-paused: don't claim/execute intents @@ -891,13 +1217,13 @@ func (e *Engine) workerLoop(ctx context.Context, t *Task, name string) { return } if e.isSettling(t.ID) { - if sleepCtx(ctx, 1000*time.Millisecond) { + if waitWorker(ctx, t, 1000*time.Millisecond) { return } continue // 任务超时收尾中:不再领新意图(在跑的自行收尾,协调器等其 drain) } if isTerminalStatus(e.m.TaskStatus(t.ID)) { - if sleepCtx(ctx, 1000*time.Millisecond) { + if waitWorker(ctx, t, 1000*time.Millisecond) { return } continue // 任务已终态(done/failed/timeout):停止领取遗留意图,别在完成后空跑 frontier @@ -907,7 +1233,7 @@ func (e *Engine) workerLoop(ctx context.Context, t *Task, name string) { } claimed := e.runWorkerStep(ctx, t, name, worker) e.decInflight(t.ID) - if !claimed && sleepCtx(ctx, 800*time.Millisecond) { + if !claimed && waitWorker(ctx, t, 800*time.Millisecond) { return } } @@ -946,9 +1272,31 @@ func (e *Engine) runWorkerStep(ctx context.Context, t *Task, name string, worker workCtx = intercept.WithTaskContext(workCtx, t.ID, fmt.Sprintf("%s · #%d", name, iid), taskEmit) hooks := steerHooks{inner: t.Guard.Hooks(), drain: func() (string, bool) { return e.drainSteer(iid) }} wTaskID, _ := strconv.ParseInt(t.ID, 10, 64) + chatRequestID, chatMessage, hasChatMessage, chatErr := t.Store.PendingIntentInterventionMessage(intent.ID) + if chatErr != nil { + log.Printf("[worker %s] task %s 读取意图 #%d 待处理人工对话失败: %v", name, t.ID, intent.ID, chatErr) + if err := transitionIntentState(t.Store, intent.ID, "running", "open"); err != nil { + log.Printf("[worker %s] task %s 意图 #%d 人工对话读取失败后回退失败: %v", name, t.ID, intent.ID, err) + } + _, completeWork := e.detachWork(intent.ID) + completeWork(chatErr) + return true + } e.BeginLLMCall(t.ID) - reason, wrote, err := worker.Execute(workCtx, name, wTaskID, e.m.assets, t.Store, intent, hooks, emit, e.m.enrich, t.NotifyFinding) + var reason harness.TerminalReason + var wrote agent.WriteCounts + var err error + if hasChatMessage { + reason, wrote, err = worker.ExecuteWithMessage(workCtx, name, wTaskID, e.m.assets, t.Store, intent, hooks, emit, e.m.enrich, t.NotifyFinding, chatRequestID, chatMessage) + } else { + reason, wrote, err = worker.Execute(workCtx, name, wTaskID, e.m.assets, t.Store, intent, hooks, emit, e.m.enrich, t.NotifyFinding) + } e.EndLLMCall(t.ID) + if hasChatMessage { + if markErr := t.Store.MarkIntentInterventionHandoffClaimed(intent.ID); markErr != nil { + log.Printf("[worker %s] task %s 标记意图 #%d 人工对话已消费失败: %v", name, t.ID, intent.ID, markErr) + } + } // model_error 收场 → 额外重跑几次（退避后再试）。仅在意图仍属本 work、任务 // 未暂停/未终止/未取消【且未进入收尾】时重试；否则让位给对应分支处理(收尾期不 // 再重试,避免退避挤占其他 worker 的优雅收尾窗口)。 @@ -961,7 +1309,11 @@ func (e *Engine) runWorkerStep(ctx context.Context, t *Task, name string, worker break // 退避期间被取消（终止/暂停）→ 交给下方分支处理 } e.BeginLLMCall(t.ID) - reason, wrote, err = worker.Execute(workCtx, name, wTaskID, e.m.assets, t.Store, intent, hooks, emit, e.m.enrich, t.NotifyFinding) + if hasChatMessage { + reason, wrote, err = worker.ExecuteWithMessage(workCtx, name, wTaskID, e.m.assets, t.Store, intent, hooks, emit, e.m.enrich, t.NotifyFinding, chatRequestID, chatMessage) + } else { + reason, wrote, err = worker.Execute(workCtx, name, wTaskID, e.m.assets, t.Store, intent, hooks, emit, e.m.enrich, t.NotifyFinding) + } e.EndLLMCall(t.ID) } // Capture kill state before detachWork cancels workCtx. kill = this work's @@ -979,6 +1331,8 @@ func (e *Engine) runWorkerStep(ctx context.Context, t *Task, name string, worker switch { case errors.Is(workCause, agent.AbortWorkPausedByUser): action = "pause" + case errors.Is(workCause, agent.AbortWorkIntervenedByUser): + action = "intervene" case errors.Is(workCause, agent.AbortWorkCancelledByUser): action = "cancel" } @@ -995,6 +1349,22 @@ func (e *Engine) runWorkerStep(ctx context.Context, t *Task, name string, worker e.touch(t.ID) return true } + if action == "intervene" || action == "intervene_no_handoff" { + controlErr = transitionIntentState(t.Store, intent.ID, "running", "paused") + if controlErr != nil { + log.Printf("[worker %s] task %s 意图 #%d 对话暂停落库失败: %v", name, t.ID, intent.ID, controlErr) + return true + } + // Install the directed barrier before completeWork releases ControlWork. + // With one worker slot this prevents its next loop from taking unrelated + // frontier work during the message hand-off. + if action == "intervene" { + e.queueDirectedClaim(t.ID, intent.ID) + } + log.Printf("[worker %s] task %s 意图 #%d 已暂停，等待新的对话意图", name, t.ID, intent.ID) + e.touch(t.ID) + return true + } if action == "cancel" { // Park the stopped run in paused before handing cleanup to the API. If the // request disconnects after cancellation, the intent remains recoverable and @@ -1099,7 +1469,114 @@ func sleepCtx(ctx context.Context, d time.Duration) (done bool) { } } +func waitWorker(ctx context.Context, t *Task, d time.Duration) (done bool) { + timer := time.NewTimer(d) + defer timer.Stop() + var wake <-chan struct{} + if t != nil { + wake = t.workerWake + } + select { + case <-ctx.Done(): + return true + case <-wake: + return false + case <-timer.C: + return false + } +} + +func (e *Engine) queueDirectedClaim(taskID string, intentID int64) { + if taskID == "" || intentID <= 0 { + return + } + e.directedMu.Lock() + defer e.directedMu.Unlock() + for _, id := range e.directedClaims[taskID] { + if id == intentID { + return + } + } + e.directedClaims[taskID] = append(e.directedClaims[taskID], intentID) + e.directedReady[intentID] = false +} + +func (e *Engine) readyDirectedClaim(intentID int64) { + e.directedMu.Lock() + if _, exists := e.directedReady[intentID]; exists { + e.directedReady[intentID] = true + } + e.directedMu.Unlock() +} + +func (e *Engine) removeDirectedClaim(taskID string, intentID int64) { + e.directedMu.Lock() + defer e.directedMu.Unlock() + items := e.directedClaims[taskID] + for i, id := range items { + if id != intentID { + continue + } + items = append(items[:i], items[i+1:]...) + delete(e.directedReady, intentID) + if len(items) == 0 { + delete(e.directedClaims, taskID) + } else { + e.directedClaims[taskID] = items + } + return + } +} + +// claimDirected tries every pending human-message hand-off before ordinary work +// frontier. blocked=true means at least one hand-off is still paused/running; +// callers must not let a just-released single worker slot overtake it. +func (e *Engine) claimDirected(t *Task, name string) (intent *db.Node, blocked bool) { + e.directedMu.Lock() + ids := append([]int64(nil), e.directedClaims[t.ID]...) + ready := make(map[int64]bool, len(ids)) + for _, id := range ids { + ready[id] = e.directedReady[id] + } + e.directedMu.Unlock() + for _, id := range ids { + node, err := t.Store.GetNode(id) + if err != nil { + return nil, true + } + if node == nil { + e.removeDirectedClaim(t.ID, id) + continue + } + switch node.State { + case "open": + if !ready[id] { + blocked = true + continue + } + ok, err := t.Store.ClaimIntent(id, name) + if err != nil { + return nil, true + } + if ok { + e.removeDirectedClaim(t.ID, id) + node.State, node.Owner = "running", name + return node, false + } + blocked = true + case "paused", "running": + blocked = true + default: + e.removeDirectedClaim(t.ID, id) + } + } + return nil, blocked +} + func (e *Engine) claimNext(t *Task, name string) *db.Node { + if intent, blocked := e.claimDirected(t, name); intent != nil || blocked { + return intent + } fr, _ := t.Store.Frontier(20) for _, in := range fr { if ok, _ := t.Store.ClaimIntent(in.ID, name); ok { diff --git a/server/engine_cancelcause_test.go b/server/engine_cancelcause_test.go index d1f4f7a..63c7324 100644 --- a/server/engine_cancelcause_test.go +++ b/server/engine_cancelcause_test.go @@ -45,6 +45,7 @@ func TestControlWorkCarriesUserActionCause(t *testing.T) { }{ {action: "pause", code: "work_paused_by_user"}, {action: "cancel", code: "work_cancelled_by_user"}, + {action: "intervene", code: "work_intervened_by_user"}, } { t.Run(tc.action, func(t *testing.T) { e := NewEngine(nil) diff --git a/server/goals.go b/server/goals.go index c8e6d98..05f2c7e 100644 --- a/server/goals.go +++ b/server/goals.go @@ -225,6 +225,13 @@ func (s *Server) hasReadyQueuedTask(excludeID string) bool { } func (s *Server) startAdmittedTask(t *Task, mode string) { + recoverMessages := s.engine.RecoverWorkerMessages + if s.engine.IsPaused(t.ID) { + recoverMessages = s.engine.recoverWorkerMessagesForAdmission + } + if err := recoverMessages(s.ctx, t); err != nil { + log.Printf("[worker chat] task %s 恢复待处理人工消息失败: %v", t.ID, err) + } if mode == "bootstrap" { if !s.engine.beginTaskOperation(t.ID) { return diff --git a/server/intent_intervention.go b/server/intent_intervention.go new file mode 100644 index 0000000..6df99fa --- /dev/null +++ b/server/intent_intervention.go @@ -0,0 +1,125 @@ +package server + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "strconv" + "strings" + "time" + + "github.com/Autumn-27/artex/db" +) + +const maxWorkerMessageBytes = 64 << 10 + +func validWorkerMessageRequestID(id string) bool { + if id == "" || len(id) > 128 { + return false + } + for _, r := range id { + if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || + (r >= '0' && r <= '9') || r == '-' || r == '_' || r == '.' || r == ':' { + continue + } + return false + } + return true +} + +// sendWorkerMessage pauses a running Worker when necessary, records one user +// turn for its existing conversation, and immediately schedules that same +// intent to continue. A disconnected browser cannot strand an accepted message. +func (s *Server) sendWorkerMessage(w http.ResponseWriter, r *http.Request) { + t, ok := s.m.Task(r.PathValue("id")) + if !ok { + writeErr(w, http.StatusNotFound, "task not found") + return + } + iid, err := strconv.ParseInt(r.PathValue("iid"), 10, 64) + if err != nil || iid <= 0 { + writeErr(w, http.StatusBadRequest, "bad intent id") + return + } + r.Body = http.MaxBytesReader(w, r.Body, maxWorkerMessageBytes) + var req struct { + Message string `json:"message"` + RequestID string `json:"request_id"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + var tooLarge *http.MaxBytesError + if errors.As(err, &tooLarge) { + writeErr(w, http.StatusRequestEntityTooLarge, "请求体过大") + return + } + writeErr(w, http.StatusBadRequest, "bad json: "+err.Error()) + return + } + message := strings.TrimSpace(req.Message) + requestID := strings.TrimSpace(req.RequestID) + if message == "" { + writeErr(w, http.StatusBadRequest, "消息不能为空") + return + } + if len([]rune(message)) > 4000 { + writeErr(w, http.StatusBadRequest, "消息不能超过 4000 个字符") + return + } + if !validWorkerMessageRequestID(requestID) { + writeErr(w, http.StatusBadRequest, "request_id 必须是 1-128 位字母、数字、-、_、. 或 :") + return + } + + if !s.engine.beginTaskOperation(t.ID) { + writeErr(w, http.StatusConflict, "任务正在删除，无法向 Worker 发送消息") + return + } + defer s.engine.decInflight(t.ID) + + node, err := t.Store.GetNode(iid) + if err != nil { + writeErr(w, http.StatusInternalServerError, err.Error()) + return + } + if node == nil { + if inherited, sourceErr := t.Store.GetNodeWithSources(iid); sourceErr == nil && inherited != nil && inherited.Inherited { + writeErr(w, http.StatusConflict, "继承意图为只读，不能发送 Worker 消息") + return + } + writeErr(w, http.StatusNotFound, "intent not found") + return + } + if node.Kind != db.KindIntent { + writeErr(w, http.StatusConflict, "node is not an intent") + return + } + base := s.ctx + if base == nil { + base = context.Background() + } + opCtx, cancel := context.WithTimeout(base, workControlWaitTimeout+15*time.Second) + defer cancel() + result, err := s.engine.InterveneWork(opCtx, t, iid, requestID, message) + if err != nil { + switch { + case errors.Is(err, context.DeadlineExceeded), errors.Is(err, context.Canceled): + writeErr(w, http.StatusGatewayTimeout, err.Error()) + case errors.Is(err, errWorkInterventionConflict), errors.Is(err, errWorkControlConflict), + errors.Is(err, db.ErrIntentInterventionConflict), errors.Is(err, db.ErrIntentStateConflict): + writeErr(w, http.StatusConflict, err.Error()) + default: + writeErr(w, http.StatusInternalServerError, err.Error()) + } + return + } + + writeJSON(w, http.StatusOK, map[string]any{ + "id": iid, + "state": result.State, + "accepted": true, + "activity_seq": result.ActivityID, + "request_id": result.RequestID, + "duplicate": result.Duplicate, + }) +} diff --git a/server/intent_intervention_protocol_test.go b/server/intent_intervention_protocol_test.go new file mode 100644 index 0000000..8347eca --- /dev/null +++ b/server/intent_intervention_protocol_test.go @@ -0,0 +1,378 @@ +package server + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "iter" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + "time" + + "github.com/Autumn-27/artex/agent" + "github.com/Autumn-27/artex/db" + "github.com/Autumn-27/norma/llm" + "github.com/Autumn-27/norma/transcript" +) + +type workerMessageTestProvider struct{} + +func (workerMessageTestProvider) Stream(_ context.Context, _ llm.CompletionRequest) iter.Seq2[llm.StreamEvent, error] { + return func(yield func(llm.StreamEvent, error) bool) { + yield(llm.StreamEvent{Type: llm.SETextDelta, Text: "已按新的对话意图继续执行"}, nil) + yield(llm.StreamEvent{Type: llm.SEMessageDelta, StopReason: "end_turn"}, nil) + yield(llm.StreamEvent{Type: llm.SEMessageStop}, nil) + } +} + +func (p workerMessageTestProvider) Complete(ctx context.Context, req llm.CompletionRequest) (llm.Message, string, llm.Usage, error) { + acc := llm.NewAccumulator() + for event, err := range p.Stream(ctx, req) { + if err != nil { + return llm.Message{}, "", llm.Usage{}, err + } + acc.Add(event) + } + return acc.Message(), acc.StopReason, acc.Usage, nil +} + +type interventionProtocolFixture struct { + m *Manager + task *Task + engine *Engine + worker *agent.Worker + tx *transcript.Store + intentID int64 +} + +func newInterventionProtocolFixture(t *testing.T, targetPriority int) *interventionProtocolFixture { + t.Helper() + m, err := NewManager(t.TempDir(), "") + if err != nil { + t.Skipf("postgres unavailable (%v) - skipping", err) + } + task, err := m.CreateTask("worker message protocol", "verify immediate conversation handoff", nil, 0, 0) + if err != nil { + m.Close() + t.Fatal(err) + } + intentID, err := task.Store.AddIntent(map[string]any{"summary": "target worker intent"}, targetPriority, nil, "planner") + if err != nil { + m.Close() + t.Fatal(err) + } + if claimed, err := task.Store.ClaimIntent(intentID, "work#1"); err != nil || !claimed { + m.Close() + t.Fatalf("claim target intent: claimed=%v err=%v", claimed, err) + } + tx := transcript.NewStore(m.dir + "/transcripts") + worker := agent.NewWorker(workerMessageTestProvider{}, "test-model", m.dir, tx, 0, 1) + engine := NewEngine(m) + engine.UseLLM(nil, worker) + fixture := &interventionProtocolFixture{m: m, task: task, engine: engine, worker: worker, tx: tx, intentID: intentID} + t.Cleanup(func() { + taskID, _ := strconv.ParseInt(task.ID, 10, 64) + _ = m.pg.DeleteTask(taskID) + _ = m.Close() + }) + return fixture +} + +// settleRegisteredIntervention emulates the ordering inside runWorkerStep after +// a cancellation-aware provider returns: detach the live work handle, commit the +// running->paused final state, install the directed hand-off, then release the +// controller waiting on workExecution.done. +func settleRegisteredIntervention(t *testing.T, f *interventionProtocolFixture) string { + t.Helper() + action, complete := f.engine.detachWork(f.intentID) + if action != "intervene" { + t.Fatalf("settled action=%q, want intervene", action) + } + if changed, err := f.task.Store.CompareAndSetIntentState(f.intentID, "running", "paused"); err != nil || !changed { + t.Fatalf("settle running->paused: changed=%v err=%v", changed, err) + } + f.engine.queueDirectedClaim(f.task.ID, f.intentID) + complete(nil) + return action +} + +func countTranscriptWorkerMessages(t *testing.T, tx *transcript.Store, sessionID, requestID string) (int, []transcript.Record) { + t.Helper() + records, err := tx.Load(tx.MainPath(sessionID)) + if err != nil { + t.Fatal(err) + } + marker := "ARTEX_WORKER_CHAT:" + requestID + count := 0 + for _, record := range records { + if record.Message != nil && strings.Contains(record.Message.Text(), marker) { + count++ + } + } + return count, records +} + +func TestWorkerMessageCancelsBeforeAcceptAndContinuesSameSessionFirst(t *testing.T) { + f := newInterventionProtocolFixture(t, 1) + competitorID, err := f.task.Store.AddIntent(map[string]any{"summary": "higher priority competitor"}, 100, nil, "planner") + if err != nil { + t.Fatal(err) + } + + // Pre-seed the same stable session so the message must continue its UUID + // chain rather than creating a worker-slot-specific conversation. + sessionID := agent.WorkerSessionID(f.task.ExpID, f.intentID) + seed := f.tx.NewWriter(sessionID, "") + seed.RecordMessage(llm.UserText("original intent turn"), llm.Usage{}) + if err := seed.Err(); err != nil { + t.Fatal(err) + } + before, err := f.tx.Load(f.tx.MainPath(sessionID)) + if err != nil || len(before) != 1 { + t.Fatalf("seed transcript records=%d err=%v", len(before), err) + } + + workCtx, cancelWork := context.WithCancelCause(context.Background()) + f.engine.registerWork(f.intentID, cancelWork) + live, unsubscribe := f.engine.Broadcaster().Subscribe(f.task.ID) + defer unsubscribe() + + const requestID = "worker-message-engine-1" + const message = "停止主动枚举，立即验证登录越权" + type outcome struct { + result WorkInterventionResult + err error + } + done := make(chan outcome, 1) + go func() { + result, err := f.engine.InterveneWork(context.Background(), f.task, f.intentID, requestID, message) + done <- outcome{result: result, err: err} + }() + + select { + case <-workCtx.Done(): + if code, _, _, _ := agent.AbortReason(workCtx); code != "work_intervened_by_user" { + t.Fatalf("old execution cancel code=%q, want work_intervened_by_user", code) + } + case <-time.After(time.Second): + t.Fatal("Worker message did not immediately cancel the old execution") + } + select { + case early := <-done: + t.Fatalf("Worker message accepted before old worker settled: %+v", early) + case <-time.After(25 * time.Millisecond): + } + pending, err := f.task.Store.IntentInterventionByRequest(f.intentID, requestID) + if err != nil || pending == nil || pending.Status != db.IntentInterventionPending { + t.Fatalf("durable pending reservation=%+v err=%v", pending, err) + } + settleRegisteredIntervention(t, f) + + var accepted outcome + select { + case accepted = <-done: + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for accepted Worker message") + } + if accepted.err != nil || accepted.result.ActivityID <= pending.ActivityID || accepted.result.Duplicate || accepted.result.State != "open" { + t.Fatalf("accepted Worker message=%+v control=%+v err=%v", accepted.result, pending, accepted.err) + } + node, err := f.task.Store.GetNode(f.intentID) + if err != nil || node == nil || node.State != "open" { + t.Fatalf("reopened target=%+v err=%v", node, err) + } + + // The API publishes the visible user activity but never edits transcript files. + // agentcore appends the standard user turn only after the same Worker is claimed. + count, records := countTranscriptWorkerMessages(t, f.tx, sessionID, requestID) + if count != 0 || len(records) != 1 { + t.Fatalf("HTTP handler edited Worker transcript: count=%d records=%+v", count, records) + } + select { + case activity := <-live: + if activity.ID != accepted.result.ActivityID || activity.NodeID == nil || *activity.NodeID != f.intentID || + activity.Worker != "user" || activity.Kind != "user" || activity.Summary != message { + t.Fatalf("broadcast activity=%+v", activity) + } + case <-time.After(time.Second): + t.Fatal("accepted Worker message was not broadcast") + } + select { + case duplicate := <-live: + t.Fatalf("accepted Worker message broadcast twice: %+v", duplicate) + default: + } + + // The directed hand-off must outrank the normal frontier without changing the + // persisted priority. This catches the one-worker race where the released slot + // used to take an unrelated, higher-priority intent before the target reopened. + claimed := f.engine.claimNext(f.task, "work#2") + if claimed == nil || claimed.ID != f.intentID { + t.Fatalf("directed claim=%+v, want target %d ahead of competitor %d", claimed, f.intentID, competitorID) + } + competitor, err := f.task.Store.GetNode(competitorID) + if err != nil || competitor == nil || competitor.State != "open" { + t.Fatalf("competitor was claimed before Worker message target: node=%+v err=%v", competitor, err) + } + request, nextMessage, hasMessage, err := f.task.Store.PendingIntentInterventionMessage(f.intentID) + if err != nil || !hasMessage || request != requestID || nextMessage != message { + t.Fatalf("pending Worker message request=%q message=%q present=%v err=%v", request, nextMessage, hasMessage, err) + } + taskID, _ := strconv.ParseInt(f.task.ID, 10, 64) + if _, _, err := f.worker.ExecuteWithMessage(context.Background(), "work#2", taskID, nil, f.task.Store, + claimed, nil, nil, nil, nil, request, nextMessage); err != nil { + t.Fatalf("continue Worker conversation: %v", err) + } + if err := f.task.Store.MarkIntentInterventionHandoffClaimed(f.intentID); err != nil { + t.Fatal(err) + } + count, records = countTranscriptWorkerMessages(t, f.tx, sessionID, requestID) + if count != 1 || len(records) != 3 || records[1].ParentUUID != records[0].UUID || records[2].ParentUUID != records[1].UUID { + t.Fatalf("Worker message did not form one normal chained turn: count=%d records=%+v", count, records) + } + if records[1].Message == nil || records[1].Message.Role != llm.RoleUser || !strings.Contains(records[1].Message.Text(), message) { + t.Fatalf("Worker message was not recorded as the next user turn: %+v", records[1]) + } + if _, _, pendingAfterRun, err := f.task.Store.PendingIntentInterventionMessage(f.intentID); err != nil || pendingAfterRun { + t.Fatalf("Worker message handoff remains pending=%v err=%v", pendingAfterRun, err) + } + + retry, err := f.engine.InterveneWork(context.Background(), f.task, f.intentID, requestID, message) + if err != nil || !retry.Duplicate || retry.ActivityID != accepted.result.ActivityID { + t.Fatalf("idempotent engine retry=%+v err=%v", retry, err) + } + if _, err := f.engine.InterveneWork(context.Background(), f.task, f.intentID, requestID, "同 request_id 的另一条消息"); !errors.Is(err, errWorkInterventionConflict) { + t.Fatalf("same request_id different message error=%v, want conflict", err) + } + count, _ = countTranscriptWorkerMessages(t, f.tx, sessionID, requestID) + if count != 1 { + t.Fatalf("engine retries appended %d Worker messages, want 1", count) + } +} + +func TestSendWorkerMessageContinuesAfterRequestContextCancellation(t *testing.T) { + f := newInterventionProtocolFixture(t, 1) + workCtx, cancelWork := context.WithCancelCause(context.Background()) + f.engine.registerWork(f.intentID, cancelWork) + s := &Server{m: f.m, engine: f.engine, ctx: context.Background()} + + requestBody, _ := json.Marshal(map[string]string{ + "message": "中止当前动作，只验证权限边界", "request_id": "worker-message-disconnect-1", + }) + requestCtx, disconnect := context.WithCancel(context.Background()) + disconnect() + req := httptest.NewRequest(http.MethodPost, "/api/tasks/1/intents/1/messages", bytes.NewReader(requestBody)).WithContext(requestCtx) + req.SetPathValue("id", f.task.ID) + req.SetPathValue("iid", strconv.FormatInt(f.intentID, 10)) + recorder := httptest.NewRecorder() + done := make(chan struct{}) + go func() { + s.sendWorkerMessage(recorder, req) + close(done) + }() + + select { + case <-workCtx.Done(): + case <-time.After(time.Second): + t.Fatal("disconnected request did not continue to cancel the worker") + } + settleRegisteredIntervention(t, f) + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("server-owned Worker message did not finish after browser disconnect") + } + if recorder.Code != http.StatusOK { + t.Fatalf("disconnected Worker message status=%d body=%s", recorder.Code, recorder.Body.String()) + } + accepted, err := f.task.Store.IntentInterventionByRequest(f.intentID, "worker-message-disconnect-1") + if err != nil || accepted == nil || accepted.Status != db.IntentInterventionAccepted { + t.Fatalf("disconnected request acceptance=%+v err=%v", accepted, err) + } +} + +func TestWorkerMessageRejectsUnavailableTaskStatesWithoutReservation(t *testing.T) { + cases := []struct { + name string + block func(*interventionProtocolFixture) + }{ + {name: "paused", block: func(f *interventionProtocolFixture) { + f.task.updateLifecycle(func(state *taskLifecycleState) { state.Paused = true }) + f.engine.Pause(f.task.ID, agent.AbortPausedByUser) + }}, + {name: "queued", block: func(f *interventionProtocolFixture) { + f.task.updateLifecycle(func(state *taskLifecycleState) { state.Queued = true }) + }}, + {name: "terminal", block: func(f *interventionProtocolFixture) { + f.task.updateLifecycle(func(state *taskLifecycleState) { state.Status = "done" }) + }}, + {name: "settling", block: func(f *interventionProtocolFixture) { f.engine.markSettling(f.task.ID) }}, + {name: "deleting", block: func(f *interventionProtocolFixture) { f.engine.BeginDelete(f.task.ID) }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f := newInterventionProtocolFixture(t, 1) + tc.block(f) + requestID := "worker-message-unavailable-" + tc.name + if _, err := f.engine.InterveneWork(context.Background(), f.task, f.intentID, requestID, "must not persist"); !errors.Is(err, errWorkInterventionConflict) { + t.Fatalf("unavailable state error=%v, want conflict", err) + } + reservation, err := f.task.Store.IntentInterventionByRequest(f.intentID, requestID) + if err != nil || reservation != nil { + t.Fatalf("unavailable state persisted reservation=%+v err=%v", reservation, err) + } + node, err := f.task.Store.GetNode(f.intentID) + if err != nil || node == nil || node.State != "running" { + t.Fatalf("unavailable state mutated intent=%+v err=%v", node, err) + } + count, _ := countTranscriptWorkerMessages(t, f.tx, agent.WorkerSessionID(f.task.ExpID, f.intentID), requestID) + if count != 0 { + t.Fatalf("unavailable state appended %d transcript interventions", count) + } + }) + } +} + +func TestWorkerWakeInterruptsLongIdlePoll(t *testing.T) { + task := &Task{workerWake: make(chan struct{}, 1)} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + started := make(chan struct{}) + returned := make(chan bool, 1) + go func() { + close(started) + returned <- waitWorker(ctx, task, 5*time.Second) + }() + <-started + start := time.Now() + task.NotifyWorker() + select { + case stopped := <-returned: + if stopped { + t.Fatal("worker wake was mistaken for context shutdown") + } + if elapsed := time.Since(start); elapsed > 250*time.Millisecond { + t.Fatalf("worker wake took %s, want immediate wake", elapsed) + } + case <-time.After(time.Second): + t.Fatal("worker wake waited for the idle polling interval") + } +} + +func TestSendWorkerMessageRejectsOversizedBodyAsEntityTooLarge(t *testing.T) { + f := newInterventionProtocolFixture(t, 1) + s := &Server{m: f.m, engine: f.engine, ctx: context.Background()} + body := `{"message":"` + strings.Repeat("x", maxWorkerMessageBytes) + `","request_id":"oversized"}` + req := httptest.NewRequest(http.MethodPost, "/api/tasks/1/intents/1/messages", strings.NewReader(body)) + req.SetPathValue("id", f.task.ID) + req.SetPathValue("iid", strconv.FormatInt(f.intentID, 10)) + recorder := httptest.NewRecorder() + s.sendWorkerMessage(recorder, req) + if recorder.Code != http.StatusRequestEntityTooLarge { + t.Fatalf("oversized Worker message status=%d body=%s", recorder.Code, recorder.Body.String()) + } +} diff --git a/server/manager.go b/server/manager.go index cb198c2..d5c20e8 100644 --- a/server/manager.go +++ b/server/manager.go @@ -63,6 +63,7 @@ type Task struct { Store *pgdb.ExplorationStore `json:"-"` Guard *guard.Guard `json:"-"` notify chan struct{} + workerWake chan struct{} lifecycleMu sync.RWMutex llmMu sync.RWMutex @@ -737,7 +738,7 @@ func taskFromPG(pt *pgdb.Task, store *pgdb.ExplorationStore, ic *intercept.Inter TimeoutSeconds: pt.TimeoutSeconds, PlanHeartbeatSeconds: pt.PlanHeartbeatSeconds, CoverageEnabled: pt.CoverageEnabled, FirstRunAt: unixOrZero(pt.FirstRunAt), DeadlineAt: unixOrZero(pt.DeadlineAt), - Store: store, Guard: guard.NewWithInterceptor(ic), notify: make(chan struct{}, 1), + Store: store, Guard: guard.NewWithInterceptor(ic), notify: make(chan struct{}, 1), workerWake: make(chan struct{}, 64), } } @@ -1638,6 +1639,42 @@ func (t *Task) Notify() { } } +// NotifyWorker wakes one idle execution slot without itself scheduling a Planner +// round. Worker-message acceptance separately records a persistent planning event +// and calls NotifyWorkerMessage; keeping execution wake-up independent prevents +// Planner debounce from delaying the directed Worker claim. +func (t *Task) NotifyWorker() { + if t == nil || t.workerWake == nil { + return + } + select { + case t.workerWake <- struct{}{}: + default: + } +} + +// NotifyWorkerMessage immediately schedules a Planner round for an accepted, +// persistent Worker direction. ActivityID identifies the durable kind=user row +// and prevents retries in the same debounce window from duplicating the prompt. +func (t *Task) NotifyWorkerMessage(intentID, activityID int64, message string) { + message = strings.TrimSpace(message) + if intentID <= 0 || activityID <= 0 || message == "" { + return + } + t.trigMu.Lock() + for _, event := range t.pendingTriggers { + if event.Kind == agent.TriggerWorkerMessage && event.ActivityID == activityID { + t.trigMu.Unlock() + return + } + } + t.pendingTriggers = append(t.pendingTriggers, agent.TriggerEvent{ + Kind: agent.TriggerWorkerMessage, ActivityID: activityID, IntentID: intentID, Detail: message, + }) + t.trigMu.Unlock() + t.Notify() +} + // NotifyDone is Notify plus a hint: a worker just finished intentID and that is // what triggered this wake-up. The planner reads the accumulated triggers next // round so it can spell out which intent finished (+ its output). Events pile up diff --git a/server/server.go b/server/server.go index f9ed5f4..b6a4182 100644 --- a/server/server.go +++ b/server/server.go @@ -249,6 +249,9 @@ func New(ctx context.Context, m *Manager, skillDir string, dataDir string, keyDi if n, _ := t.Store.ResetRunningIntents(); n > 0 { log.Printf("[engine] task %s 重置 %d 个残留 running 意图为 open", t.ID, n) } + if err := s.engine.RecoverWorkerMessages(ctx, t); err != nil { + log.Printf("[worker chat] task %s 恢复待处理人工消息失败: %v", t.ID, err) + } if lifecycle.Paused { s.engine.Pause(t.ID, agent.AbortPausedOnReload) } @@ -471,6 +474,11 @@ func (s *Server) applyLLM(cfg agent.Config) error { s.llmOn = true s.cfgMu.Unlock() s.invalidateTaskAgents() + for _, task := range s.m.List() { + if err := s.engine.RecoverWorkerMessages(s.ctx, task); err != nil { + log.Printf("[worker chat] task %s 在 LLM 就绪后恢复待处理人工消息失败: %v", task.ID, err) + } + } // wake the active task so a task created while idle starts exploring. if t := s.m.ActiveTask(); t != nil { @@ -749,6 +757,7 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("PUT /api/tasks/{id}/llm", s.updateTaskLLMProfiles) mux.HandleFunc("GET /api/tasks/{id}/llm/resolution", s.taskLLMResolutionHandler) mux.HandleFunc("POST /api/tasks/{id}/intents/{iid}/control", s.controlIntent) + mux.HandleFunc("POST /api/tasks/{id}/intents/{iid}/messages", s.sendWorkerMessage) mux.HandleFunc("POST /api/tasks/{id}/intents/{iid}/rerun", s.rerunIntent) // 重跑单条 blocked/exhausted/stopped 意图 mux.HandleFunc("POST /api/tasks/{id}/intents/rerun-blocked", s.rerunBlocked) // 批量重跑本任务全部 blocked 意图 mux.HandleFunc("POST /api/active", s.setActive) diff --git a/web/src/app/(main)/function/tasks/detail/_tabs/sessions-tab.tsx b/web/src/app/(main)/function/tasks/detail/_tabs/sessions-tab.tsx index 937b97e..46ed163 100644 --- a/web/src/app/(main)/function/tasks/detail/_tabs/sessions-tab.tsx +++ b/web/src/app/(main)/function/tasks/detail/_tabs/sessions-tab.tsx @@ -14,7 +14,6 @@ import { Loader2Icon, PaperclipIcon, PauseIcon, - PlayIcon, RadioIcon, RotateCwIcon, ShieldAlertIcon, @@ -43,6 +42,7 @@ import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { InputGroup, InputGroupAddon, InputGroupButton, InputGroupTextarea } from "@/components/ui/input-group"; import { ScrollArea } from "@/components/ui/scroll-area"; +import { Spinner } from "@/components/ui/spinner"; import { Textarea } from "@/components/ui/textarea"; import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; import { api, sseUrl } from "@/lib/api"; @@ -93,6 +93,18 @@ const PAGE = 200; // history page size const SYSTEM_SCAN_PAGE = 500; // generic activity pages scanned to recover sparse system audit events const MAX_KEEP = 4000; // per-session in-memory cap; older pages re-fetched on scroll-up const STREAM_WINDOW_MS = 5000; // "live" = activity seen within this window +const MAX_WORKER_MESSAGE_CHARS = 4000; + +function newWorkerMessageRequestID(): string { + return ( + globalThis.crypto?.randomUUID?.() ?? + `worker-message-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 12)}` + ); +} + +function workerMessageCharCount(value: string): number { + return Array.from(value).length; +} // One session's lazily-loaded, reverse-paginated cache. `lastTs`/`unread` are kept // live even for sessions that were never opened, so the list shows liveness + unread @@ -313,8 +325,6 @@ function SessionItem({ hasPending, unread, onClick, - onPause, - onResume, onCancel, controlling, deleted, @@ -325,8 +335,6 @@ function SessionItem({ hasPending?: boolean; unread?: number; onClick: () => void; - onPause?: () => void; - onResume?: () => void; onCancel?: () => void; controlling?: boolean; deleted?: boolean; @@ -339,7 +347,8 @@ function SessionItem({ ) : null; - const controllable = s.role === "worker" && !s.inherited && (s.status === "running" || s.status === "paused"); + const cancellable = + s.role === "worker" && !s.inherited && !deleted && (s.status === "running" || s.status === "paused"); return (
)} - {controllable && ( + {cancellable && (
- {s.status === "running" ? ( - - ) : ( - - )}
+ ) : active.role === "worker" && + !active.inherited && + (active.status === "running" || active.status === "paused" || active.status === "pending") ? ( +
+ + { + setWorkerMessage(event.target.value); + setWorkerMessageRequestId(""); + }} + onKeyDown={(event) => { + if (!shouldSubmitOnKey(event, sendMode)) return; + event.preventDefault(); + sendWorkerChat(); + }} + disabled={active.status === "running" || workerMessageSending} + aria-invalid={workerMessageCharCount(workerMessage) > MAX_WORKER_MESSAGE_CHARS} + className="max-h-36 min-h-9 overflow-y-auto" + /> + + MAX_WORKER_MESSAGE_CHARS && "text-destructive", + )} + > + {workerMessageCharCount(workerMessage)}/{MAX_WORKER_MESSAGE_CHARS} + + {active.status === "running" ? ( + void controlWorker(active, "pause")} + disabled={controllingIntent === active.intent_id} + title="暂停当前 Worker" + aria-label="暂停当前 Worker" + > + {controllingIntent === active.intent_id ? ( + + ) : ( + + )} + + ) : ( + MAX_WORKER_MESSAGE_CHARS + } + title="发送消息" + aria-label="发送消息" + > + {workerMessageSending ? : } + + )} + + +
) : (
(`/tasks/${taskId}/intents/${intentId}/control`, { action, reason: reason ?? "" }), + sendWorkerMessage: (taskId: string, intentId: string, message: string, requestId: string) => + post<{ + id: number; + state: "open" | "running" | "paused"; + accepted: true; + activity_seq: number; + request_id: string; + }>(`/tasks/${taskId}/intents/${intentId}/messages`, { message, request_id: requestId }), taskLLMResolution: (id: string) => get(`/tasks/${id}/llm/resolution`), // 重跑一条没跑成功的意图(blocked/exhausted/stopped)：置回 open，worker 会重新认领、从头再跑。 rerunIntent: (taskId: string, intentId: string) => diff --git a/web/src/lib/mock/handler.ts b/web/src/lib/mock/handler.ts index 407fe4e..98b6c44 100644 --- a/web/src/lib/mock/handler.ts +++ b/web/src/lib/mock/handler.ts @@ -10,6 +10,7 @@ import { normalizeCompanyScopeValue, } from "../company-scope"; import type { + Activity, ArchiveBatchItem, Asset, BatchControlItem, @@ -42,6 +43,7 @@ const mockConversations = structuredClone(D.conversations); const mockIntents = structuredClone(D.intents); const mockCompanies = structuredClone(D.companies); const mockAssets = structuredClone(D.assets); +const mockActivity = structuredClone(D.activity); type MockTaskArchiveSnapshot = { task: Task; numericTaskID: number; @@ -420,6 +422,15 @@ function bodyIDs(value: unknown): string[] { } type MockIntentControlResult = { ok: boolean; state?: string; error?: string }; +type MockWorkerMessage = { + intentId: string; + message: string; + state: "open" | "running"; + activitySeq: number; +}; + +const mockWorkerMessages = new Map(); +let nextMockWorkerMessageActivitySeq = mockActivity.reduce((maximum, item) => Math.max(maximum, item.seq), 0) + 1; function controlMockIntent(id: string, action: "pause" | "resume"): MockIntentControlResult { const intent = mockIntents.find((item) => item.id === id); @@ -432,6 +443,71 @@ function controlMockIntent(id: string, action: "pause" | "resume"): MockIntentCo return { ok: true, state: intent.state }; } +function sendMockWorkerMessage( + id: string, + message: string, + requestId: string, +): MockIntentControlResult & { activitySeq?: number; requestId?: string } { + const normalizedMessage = message.trim(); + const normalizedRequestId = requestId.trim(); + if (!normalizedRequestId) return { ok: false, error: "request_id 不能为空" }; + const previous = mockWorkerMessages.get(normalizedRequestId); + if (previous) { + if (previous.intentId !== id || previous.message !== normalizedMessage) { + return { ok: false, error: "request_id 已用于其他 Worker 消息" }; + } + return { + ok: true, + state: previous.state, + activitySeq: previous.activitySeq, + requestId: normalizedRequestId, + }; + } + const intent = mockIntents.find((item) => item.id === id); + if (!intent) return { ok: false, error: "意图不存在" }; + if (intent.inherited || (intent.state !== "running" && intent.state !== "paused")) { + return { ok: false, state: intent.state, error: "Worker 状态已变化" }; + } + if (!normalizedMessage) return { ok: false, state: intent.state, error: "消息不能为空" }; + if (Array.from(normalizedMessage).length > 4000) { + return { ok: false, state: intent.state, error: "消息不能超过 4000 个字符" }; + } + + // The real endpoint stops the active turn, persists a normal user message, + // and immediately reopens the same intent. Keep the mock's transition in sync, + // then emulate a Worker reclaiming it shortly afterward. + intent.state = "open"; + const activitySeq = nextMockWorkerMessageActivitySeq++; + const workerMessage: MockWorkerMessage = { + intentId: id, + message: normalizedMessage, + state: "open", + activitySeq, + }; + mockWorkerMessages.set(normalizedRequestId, workerMessage); + mockActivity.push({ + seq: activitySeq, + intent_id: id, + worker: "user", + ts: new Date().toISOString(), + kind: "user", + summary: normalizedMessage, + detail: normalizedMessage, + } satisfies Activity); + setTimeout(() => { + if (intent.state === "open") { + intent.state = "running"; + workerMessage.state = "running"; + } + }, 250); + return { + ok: true, + state: workerMessage.state, + activitySeq: workerMessage.activitySeq, + requestId: normalizedRequestId, + }; +} + function controlMockTask(id: string, action: "pause" | "resume"): BatchControlItem { const task = mockTasks.find((item) => item.id === id); if (!task) return { id, ok: false, error: "任务不存在" }; @@ -917,7 +993,7 @@ function route(m: string, path: string, seg: string[], q: URLSearchParams, b: Re intents: 1, facts: 1, findings: 1, - activities: D.activity.filter((item) => item.intent_id === id).length, + activities: mockActivity.filter((item) => item.intent_id === id).length, }, }; } @@ -926,6 +1002,18 @@ function route(m: string, path: string, seg: string[], q: URLSearchParams, b: Re if (!result.ok) throw new Error(result.error ?? "Worker 状态已变化"); return { id: Number(id.replace(/\D/g, "")) || 0, state: result.state }; } + if (seg[0] === "tasks" && seg[2] === "intents" && seg[4] === "messages" && m === "POST") { + const id = seg[3]; + const result = sendMockWorkerMessage(id, String(b.message ?? ""), String(b.request_id ?? "")); + if (!result.ok) throw new Error(result.error ?? "Worker 状态已变化"); + return { + id: Number(id.replace(/\D/g, "")) || 0, + state: result.state ?? "open", + accepted: true, + activity_seq: result.activitySeq ?? 0, + request_id: result.requestId ?? "", + }; + } if (seg[0] === "tasks" && seg.length === 3 && seg[2] === "control" && m === "POST") { const action = b.action === "resume" ? "resume" : "pause"; const result = controlMockTask(seg[1], action); @@ -1464,13 +1552,11 @@ function route(m: string, path: string, seg: string[], q: URLSearchParams, b: Re if (path === "/exploration/activity" && seg.length === 2) { const since = Number(q.get("since") ?? 0); const limit = Math.max(1, Number(q.get("limit") ?? 300)); - const items = D.activityForTask() - .filter((item) => item.seq > since) - .slice(0, limit); + const items = mockActivity.filter((item) => item.seq > since).slice(0, limit); return { items, cursor: items.length ? items[items.length - 1].seq : since }; } if (seg[0] === "exploration" && seg[1] === "activity" && seg.length === 3) { - const a = D.activity.find((x) => x.seq === Number(seg[2])); + const a = mockActivity.find((x) => x.seq === Number(seg[2])); return { detail: a?.detail ?? a?.summary ?? "" }; } if (path === "/tokens/daily") return D.dailyTokens; From 7e633b2056745f297856900850639609579217b1 Mon Sep 17 00:00:00 2001 From: pphua Date: Sat, 29 Aug 2026 21:44:39 +0800 Subject: [PATCH 22/23] fix: sync worker directions across agents --- agent/mainagent.go | 24 ++- agent/worker_direction_sync_test.go | 155 ++++++++++++++++++++ db/exploration.go | 12 +- db/intent_intervention_lifecycle_test.go | 69 +++++++++ server/intent_intervention_protocol_test.go | 32 ++++ 5 files changed, 285 insertions(+), 7 deletions(-) create mode 100644 agent/worker_direction_sync_test.go diff --git a/agent/mainagent.go b/agent/mainagent.go index 9c40eca..2962252 100644 --- a/agent/mainagent.go +++ b/agent/mainagent.go @@ -2,6 +2,7 @@ package agent import ( "context" + "encoding/json" "fmt" "github.com/Autumn-27/artex/db" @@ -85,6 +86,27 @@ func mainAgentSystem(goal, dataDir, workDir string) string { return body + artifactSpec(workDir) } +// mainAgentInput keeps Worker-side human steering visible to the Main Agent on +// every fresh Chat session. The direction records remain separate from the +// current user turn so the model does not mistake persisted context for a new +// instruction that must be sent to the Worker again. +func mainAgentInput(ts *db.ExplorationStore, message string) string { + if ts == nil { + return message + } + directions, err := ts.LatestWorkerDirections(workerDirectionOverviewLimit) + if err != nil || len(directions) == 0 { + return message + } + raw, err := json.Marshal(compactWorkerDirections(directions)) + if err != nil { + return message + } + return "【系统同步的 Worker 最新人工方向】\n" + + "以下内容来自持久化的 Worker 对话。回答、观察和调度时必须纳入，但不要把它当作本轮新消息重复下发。\n" + + string(raw) + "\n\n【用户本轮消息】\n" + message +} + // Chat handles one human message and returns the assistant reply. emit, if // non-nil, receives each execution step (thinking / tool_use / tool_result / // text / result) so the main-agent session shows its work — exactly like the @@ -153,7 +175,7 @@ func (m *MainAgent) Chat(ctx context.Context, taskID int64, as *db.AssetStore, t // C2: this session is fresh each turn; re-unlock skill-gated MCPs from prior // Skill() calls in the reloaded history so revealed tools stay callable. seedUnlockFromHistory(s.Messages(), def.UnlockSkill) - text, _, err := captureRunSession(ctx, s, message, func(r db.Activity) { + text, _, err := captureRunSession(ctx, s, mainAgentInput(ts, message), func(r db.Activity) { if emit != nil { r.Worker = "mainagent" emit(r) diff --git a/agent/worker_direction_sync_test.go b/agent/worker_direction_sync_test.go new file mode 100644 index 0000000..a1cf7f8 --- /dev/null +++ b/agent/worker_direction_sync_test.go @@ -0,0 +1,155 @@ +package agent + +import ( + "context" + "iter" + "strings" + "sync" + "testing" + + "github.com/Autumn-27/artex/db" + "github.com/Autumn-27/norma/llm" +) + +type directionCaptureProvider struct { + mu sync.Mutex + requests []llm.CompletionRequest +} + +func (p *directionCaptureProvider) Stream(_ context.Context, req llm.CompletionRequest) iter.Seq2[llm.StreamEvent, error] { + p.mu.Lock() + p.requests = append(p.requests, req) + p.mu.Unlock() + return func(yield func(llm.StreamEvent, error) bool) { + if !yield(llm.StreamEvent{Type: llm.SETextDelta, Text: "同步完成"}, nil) { + return + } + if !yield(llm.StreamEvent{Type: llm.SEMessageDelta, StopReason: "end_turn"}, nil) { + return + } + yield(llm.StreamEvent{Type: llm.SEMessageStop}, nil) + } +} + +func (p *directionCaptureProvider) Complete(_ context.Context, req llm.CompletionRequest) (llm.Message, string, llm.Usage, error) { + p.mu.Lock() + p.requests = append(p.requests, req) + p.mu.Unlock() + return llm.Message{Role: llm.RoleAssistant, Content: []llm.ContentBlock{llm.TextBlock("同步完成")}}, "end_turn", llm.Usage{}, nil +} + +func (p *directionCaptureProvider) lastRequest(t *testing.T) llm.CompletionRequest { + t.Helper() + p.mu.Lock() + defer p.mu.Unlock() + if len(p.requests) == 0 { + t.Fatal("provider received no completion request") + } + return p.requests[len(p.requests)-1] +} + +func lastUserText(t *testing.T, req llm.CompletionRequest) string { + t.Helper() + for i := len(req.Messages) - 1; i >= 0; i-- { + if req.Messages[i].Role == llm.RoleUser { + return req.Messages[i].Text() + } + } + t.Fatal("completion request has no user message") + return "" +} + +type directionAgentFixture struct { + d *db.DB + task *db.Task + store *db.ExplorationStore + intentID int64 + activity db.Activity +} + +func newDirectionAgentFixture(t *testing.T) *directionAgentFixture { + t.Helper() + d := testDB(t) + task, err := d.CreateTask("agent direction sync", "synchronize worker steering", nil, 0, 0) + if err != nil { + d.Close() + t.Fatal(err) + } + t.Cleanup(func() { + _ = d.DeleteTask(task.ID) + _ = d.Close() + }) + store := d.Exploration(task.ExplorationID) + intentID, err := store.AddIntent(map[string]any{"summary": "原始接口枚举方向"}, 1, nil, "planner") + if err != nil { + t.Fatal(err) + } + if claimed, err := store.ClaimIntent(intentID, "work#direction"); err != nil || !claimed { + t.Fatalf("claim direction fixture: claimed=%v err=%v", claimed, err) + } + const requestID = "agent-direction-sync-1" + const direction = "停止接口枚举，立即聚焦登录后的水平越权" + if _, err := store.ReserveIntentIntervention(intentID, requestID, direction); err != nil { + t.Fatal(err) + } + if changed, err := store.CompareAndSetIntentState(intentID, "running", "paused"); err != nil || !changed { + t.Fatalf("pause direction fixture: changed=%v err=%v", changed, err) + } + activity, duplicate, err := store.AcceptIntentIntervention(intentID, requestID, direction) + if err != nil || duplicate { + t.Fatalf("accept direction fixture: activity=%+v duplicate=%v err=%v", activity, duplicate, err) + } + return &directionAgentFixture{d: d, task: task, store: store, intentID: intentID, activity: activity} +} + +func TestGraphOverviewIncludesLatestWorkerDirection(t *testing.T) { + f := newDirectionAgentFixture(t) + overview := NewToolSet(f.store, "planner").graphOverviewData() + directions, ok := overview["worker_directions"].([]map[string]any) + if !ok || len(directions) != 1 { + t.Fatalf("worker_directions=%T %+v", overview["worker_directions"], overview["worker_directions"]) + } + if directions[0]["activity_id"] != f.activity.ID || directions[0]["intent_id"] != f.intentID || + directions[0]["intent_summary"] != "原始接口枚举方向" || + directions[0]["message"] != "停止接口枚举，立即聚焦登录后的水平越权" { + t.Fatalf("worker direction overview=%+v", directions[0]) + } +} + +func TestPlannerRequestIncludesWorkerMessageTriggerAndPersistedDirection(t *testing.T) { + f := newDirectionAgentFixture(t) + provider := &directionCaptureProvider{} + planner := NewPlanner(provider, "test-model", t.TempDir(), nil, 0, 1) + triggerDetail := "本轮只验证登录后的对象级授权" + _, _, err := planner.Plan(context.Background(), f.task.ID, nil, f.store, f.task.Goal, []TriggerEvent{{ + Kind: TriggerWorkerMessage, ActivityID: f.activity.ID, IntentID: f.intentID, Detail: triggerDetail, + }}, nil) + if err != nil { + t.Fatal(err) + } + input := lastUserText(t, provider.lastRequest(t)) + for _, want := range []string{"【本次触发本轮的实际变动】", triggerDetail, `"worker_directions"`, "停止接口枚举，立即聚焦登录后的水平越权"} { + if !strings.Contains(input, want) { + t.Fatalf("Planner input missing %q:\n%s", want, input) + } + } +} + +func TestMainAgentNextTurnIncludesPersistedWorkerDirection(t *testing.T) { + f := newDirectionAgentFixture(t) + provider := &directionCaptureProvider{} + mainAgent := NewMainAgent(provider, "test-model", t.TempDir(), nil, 0, 1) + const userMessage = "汇报当前执行方向" + if _, err := mainAgent.Chat(context.Background(), f.task.ID, nil, f.store, f.task.Goal, userMessage, nil, nil, nil, nil); err != nil { + t.Fatal(err) + } + input := lastUserText(t, provider.lastRequest(t)) + for _, want := range []string{"【系统同步的 Worker 最新人工方向】", "停止接口枚举，立即聚焦登录后的水平越权", "【用户本轮消息】", userMessage} { + if !strings.Contains(input, want) { + t.Fatalf("Main Agent input missing %q:\n%s", want, input) + } + } + if strings.Index(input, "【系统同步的 Worker 最新人工方向】") > strings.Index(input, "【用户本轮消息】") { + t.Fatalf("Main Agent context is not separated before current user message:\n%s", input) + } +} diff --git a/db/exploration.go b/db/exploration.go index 7c40e85..6a05f9a 100644 --- a/db/exploration.go +++ b/db/exploration.go @@ -360,13 +360,13 @@ type IntentIntervention struct { // intent. The accepted activity is the durable planning event, so this view // survives restarts and is shared by Planner and Main Agent context builders. type WorkerDirection struct { - ActivityID int64 - IntentID int64 - RequestID string - Message string - IntentState string + ActivityID int64 + IntentID int64 + RequestID string + Message string + IntentState string IntentSummary string - CreatedAt time.Time + CreatedAt time.Time } // lockIntentIntervention serializes request-id reservation/finalization across diff --git a/db/intent_intervention_lifecycle_test.go b/db/intent_intervention_lifecycle_test.go index c27a746..9363e7c 100644 --- a/db/intent_intervention_lifecycle_test.go +++ b/db/intent_intervention_lifecycle_test.go @@ -202,3 +202,72 @@ func TestIntentInterventionAcceptanceCanRetryAfterStateSettlement(t *testing.T) t.Fatalf("retry acceptance: activity=%+v duplicate=%v err=%v", activity, duplicate, err) } } + +func acceptTestWorkerDirection(t *testing.T, store *ExplorationStore, intentID int64, requestID, message string) Activity { + t.Helper() + if claimed, err := store.ClaimIntent(intentID, "direction-test-worker"); err != nil || !claimed { + t.Fatalf("claim direction intent: claimed=%v err=%v", claimed, err) + } + if _, err := store.ReserveIntentIntervention(intentID, requestID, message); err != nil { + t.Fatal(err) + } + if changed, err := store.CompareAndSetIntentState(intentID, "running", "paused"); err != nil || !changed { + t.Fatalf("pause direction intent: changed=%v err=%v", changed, err) + } + activity, duplicate, err := store.AcceptIntentIntervention(intentID, requestID, message) + if err != nil || duplicate { + t.Fatalf("accept direction: activity=%+v duplicate=%v err=%v", activity, duplicate, err) + } + return activity +} + +func TestLatestWorkerDirectionsKeepsLatestPerIntentAcrossStoreReopen(t *testing.T) { + d, err := Open(testDSN(t)) + if err != nil { + t.Skipf("postgres unavailable (%v) - skipping", err) + } + defer d.Close() + task, err := d.CreateTask("worker direction persistence", "keep latest directions", nil, 0, 0) + if err != nil { + t.Fatal(err) + } + defer d.DeleteTask(task.ID) //nolint:errcheck + store := d.Exploration(task.ExplorationID) + firstIntent, err := store.AddIntent(map[string]any{"summary": "first route"}, 1, nil, "planner") + if err != nil { + t.Fatal(err) + } + secondIntent, err := store.AddIntent(map[string]any{"summary": "second route"}, 1, nil, "planner") + if err != nil { + t.Fatal(err) + } + + acceptTestWorkerDirection(t, store, firstIntent, "direction-old", "先检查旧方向") + secondActivity := acceptTestWorkerDirection(t, store, secondIntent, "direction-second", "保留第二条意图方向") + latestActivity := acceptTestWorkerDirection(t, store, firstIntent, "direction-latest", "改为验证最新权限边界") + + d2, err := Open(testDSN(t)) + if err != nil { + t.Fatal(err) + } + defer d2.Close() + directions, err := d2.Exploration(task.ExplorationID).LatestWorkerDirections(10) + if err != nil { + t.Fatal(err) + } + if len(directions) != 2 { + t.Fatalf("directions=%+v, want one latest row per intent", directions) + } + byIntent := make(map[int64]WorkerDirection, len(directions)) + for _, direction := range directions { + byIntent[direction.IntentID] = direction + } + if got := byIntent[firstIntent]; got.ActivityID != latestActivity.ID || got.RequestID != "direction-latest" || + got.Message != "改为验证最新权限边界" || got.IntentSummary != "first route" { + t.Fatalf("latest first direction=%+v", got) + } + if got := byIntent[secondIntent]; got.ActivityID != secondActivity.ID || got.Message != "保留第二条意图方向" || + got.IntentSummary != "second route" { + t.Fatalf("second direction=%+v", got) + } +} diff --git a/server/intent_intervention_protocol_test.go b/server/intent_intervention_protocol_test.go index 8347eca..77ac2e7 100644 --- a/server/intent_intervention_protocol_test.go +++ b/server/intent_intervention_protocol_test.go @@ -181,6 +181,11 @@ func TestWorkerMessageCancelsBeforeAcceptAndContinuesSameSessionFirst(t *testing if accepted.err != nil || accepted.result.ActivityID <= pending.ActivityID || accepted.result.Duplicate || accepted.result.State != "open" { t.Fatalf("accepted Worker message=%+v control=%+v err=%v", accepted.result, pending, accepted.err) } + triggers := f.task.drainTriggers() + if len(triggers) != 1 || triggers[0].Kind != agent.TriggerWorkerMessage || + triggers[0].ActivityID != accepted.result.ActivityID || triggers[0].IntentID != f.intentID || triggers[0].Detail != message { + t.Fatalf("accepted Worker message triggers=%+v, want one matching persistent event", triggers) + } node, err := f.task.Store.GetNode(f.intentID) if err != nil || node == nil || node.State != "open" { t.Fatalf("reopened target=%+v err=%v", node, err) @@ -248,12 +253,39 @@ func TestWorkerMessageCancelsBeforeAcceptAndContinuesSameSessionFirst(t *testing if _, err := f.engine.InterveneWork(context.Background(), f.task, f.intentID, requestID, "同 request_id 的另一条消息"); !errors.Is(err, errWorkInterventionConflict) { t.Fatalf("same request_id different message error=%v, want conflict", err) } + if triggers := f.task.drainTriggers(); len(triggers) != 0 { + t.Fatalf("idempotent retries emitted Planner triggers: %+v", triggers) + } count, _ = countTranscriptWorkerMessages(t, f.tx, sessionID, requestID) if count != 1 { t.Fatalf("engine retries appended %d Worker messages, want 1", count) } } +func TestRecoverAcceptedWorkerMessageReconstructsPlannerTrigger(t *testing.T) { + f := newInterventionProtocolFixture(t, 1) + const requestID = "worker-message-recovery-1" + const message = "恢复后优先验证人工指定的权限边界" + if _, err := f.task.Store.ReserveIntentIntervention(f.intentID, requestID, message); err != nil { + t.Fatal(err) + } + if changed, err := f.task.Store.CompareAndSetIntentState(f.intentID, "running", "paused"); err != nil || !changed { + t.Fatalf("pause before recovery acceptance: changed=%v err=%v", changed, err) + } + activity, duplicate, err := f.task.Store.AcceptIntentIntervention(f.intentID, requestID, message) + if err != nil || duplicate { + t.Fatalf("accept before recovery: activity=%+v duplicate=%v err=%v", activity, duplicate, err) + } + if err := f.engine.RecoverWorkerMessages(context.Background(), f.task); err != nil { + t.Fatal(err) + } + triggers := f.task.drainTriggers() + if len(triggers) != 1 || triggers[0].Kind != agent.TriggerWorkerMessage || triggers[0].ActivityID != activity.ID || + triggers[0].IntentID != f.intentID || triggers[0].Detail != message { + t.Fatalf("recovered Planner triggers=%+v", triggers) + } +} + func TestSendWorkerMessageContinuesAfterRequestContextCancellation(t *testing.T) { f := newInterventionProtocolFixture(t, 1) workCtx, cancelWork := context.WithCancelCause(context.Background()) From 76210b99336a1a7d10fa1283e386c837b2568870 Mon Sep 17 00:00:00 2001 From: pphua Date: Sat, 29 Aug 2026 22:05:28 +0800 Subject: [PATCH 23/23] revert: remove cross-agent worker direction synchronization --- agent/mainagent.go | 24 +-- agent/planner.go | 20 +-- agent/tools.go | 40 ----- agent/worker_direction_sync_test.go | 155 -------------------- db/exploration.go | 61 -------- db/intent_intervention_lifecycle_test.go | 69 --------- server/engine.go | 5 - server/intent_intervention_protocol_test.go | 32 ---- server/manager.go | 30 +--- 9 files changed, 11 insertions(+), 425 deletions(-) delete mode 100644 agent/worker_direction_sync_test.go diff --git a/agent/mainagent.go b/agent/mainagent.go index 2962252..9c40eca 100644 --- a/agent/mainagent.go +++ b/agent/mainagent.go @@ -2,7 +2,6 @@ package agent import ( "context" - "encoding/json" "fmt" "github.com/Autumn-27/artex/db" @@ -86,27 +85,6 @@ func mainAgentSystem(goal, dataDir, workDir string) string { return body + artifactSpec(workDir) } -// mainAgentInput keeps Worker-side human steering visible to the Main Agent on -// every fresh Chat session. The direction records remain separate from the -// current user turn so the model does not mistake persisted context for a new -// instruction that must be sent to the Worker again. -func mainAgentInput(ts *db.ExplorationStore, message string) string { - if ts == nil { - return message - } - directions, err := ts.LatestWorkerDirections(workerDirectionOverviewLimit) - if err != nil || len(directions) == 0 { - return message - } - raw, err := json.Marshal(compactWorkerDirections(directions)) - if err != nil { - return message - } - return "【系统同步的 Worker 最新人工方向】\n" + - "以下内容来自持久化的 Worker 对话。回答、观察和调度时必须纳入，但不要把它当作本轮新消息重复下发。\n" + - string(raw) + "\n\n【用户本轮消息】\n" + message -} - // Chat handles one human message and returns the assistant reply. emit, if // non-nil, receives each execution step (thinking / tool_use / tool_result / // text / result) so the main-agent session shows its work — exactly like the @@ -175,7 +153,7 @@ func (m *MainAgent) Chat(ctx context.Context, taskID int64, as *db.AssetStore, t // C2: this session is fresh each turn; re-unlock skill-gated MCPs from prior // Skill() calls in the reloaded history so revealed tools stay callable. seedUnlockFromHistory(s.Messages(), def.UnlockSkill) - text, _, err := captureRunSession(ctx, s, mainAgentInput(ts, message), func(r db.Activity) { + text, _, err := captureRunSession(ctx, s, message, func(r db.Activity) { if emit != nil { r.Worker = "mainagent" emit(r) diff --git a/agent/planner.go b/agent/planner.go index 3f63166..95802f8 100644 --- a/agent/planner.go +++ b/agent/planner.go @@ -123,7 +123,6 @@ func renderPlannerTodos(items []actool.Todo) string { // // "done" — a worker finished intent IntentID (its output conclusion is fetched). // "finding" — a worker reported a finding on intent IntentID (Detail = 摘要). -// "worker_message" — the human changed one Worker's direction (Detail = 新方向). // "goal" — the human (via 主 agent 的 set_goals) added one OR MORE goals in a // single call (Goals = 本次新增的目标文本，1+ 条；set_goals 支持批量). // "goal_deleted" — the human deleted a goal from 总览的目标管理 (Detail = 被删目标文本). @@ -131,17 +130,14 @@ func renderPlannerTodos(items []actool.Todo) string { // "cancelled" — the human deleted intent IntentID (Detail = 删除原因). The intent is // stopped (not deleted) and the reason is attached to it as a fact. type TriggerEvent struct { - Kind string - ActivityID int64 // persistent activity id; worker_message notifications use it for de-duplication - IntentID int64 - Detail string - Goals []string // Kind=="goal" 专用：本次 set_goals 新增的目标文本（1 条或多条） - OldGoal string // Kind=="goal_edited" 专用：修改前的目标文本 - NewGoal string // Kind=="goal_edited" 专用：修改后的目标文本 + Kind string + IntentID int64 + Detail string + Goals []string // Kind=="goal" 专用：本次 set_goals 新增的目标文本（1 条或多条） + OldGoal string // Kind=="goal_edited" 专用：修改前的目标文本 + NewGoal string // Kind=="goal_edited" 专用：修改后的目标文本 } -const TriggerWorkerMessage = db.PlannerEventWorkerMessage - // renderTriggers spells out the change(s) that fired this round: for a finished // worker — which intent + its output conclusion; for a finding — which intent + // what was found. Empty for time/heartbeat wakes. Reads the store (best-effort; @@ -166,9 +162,6 @@ func renderTriggers(ts *db.ExplorationStore, evs []TriggerEvent) string { b.WriteString(fmt.Sprintf("\n- 人修改了目标，由「%s」变为「%s」—— 请据新目标调整探索方向（原方向若已不适用请停派）。", ev.OldGoal, ev.NewGoal)) case "finding": b.WriteString(fmt.Sprintf("\n- 意图 #%d（%s）的 worker 报告了一个 finding：%s", ev.IntentID, intentSummary(ts, ev.IntentID), ev.Detail)) - case TriggerWorkerMessage: - b.WriteString(fmt.Sprintf("\n- 用户向意图 #%d（%s）的 Worker 下发了新的人工方向：%s —— 该方向已优先交给同一 Worker；后续规划必须以它替代或补充原意图描述，避免生成冲突、回退或重复方向。", - ev.IntentID, intentSummary(ts, ev.IntentID), truncOutput(ev.Detail, 2000))) case "cancelled": b.WriteString(fmt.Sprintf("\n- 意图 #%d 由用户删除，意图内容是：%s、删除原因是：%s。该意图已停止（不再执行），其原因已作为事实挂在该意图上；请据此重新规划。", ev.IntentID, intentSummary(ts, ev.IntentID), ev.Detail)) default: // "done" @@ -257,7 +250,6 @@ const plannerDefaultTmpl = `你是一个授权渗透测试系统的"规划者" 决策流程（每次唤醒）： 1. **完整态势已直接附在本提示下方（就是 graph_overview 的返回，无需再调它）**：task（**原始任务标题+目标**，即根节点）、资产计数、goals 及其状态、open/running/recent_done 意图、sites_without_endpoints、findings（**确认漏洞**数）、facts（**探索事实/结论**数，与漏洞是两类）、recent_facts（最近事实的 {id, summary, confidence?}，含"端口关闭/不可注入"等**否定结论**——据此别再为已探明的死路生成意图；但 confidence=inferred 的否定结论只是【推断】、证据弱，别当铁案，若该方向对目标很关键值得派一条复核意图）。**这里的探索节点（goals/意图/facts/findings）都只含本任务的**（绝不会有别的任务的目标）；**资产图则全局共享**（多任务同一份，资产计数是全局在范围内的数据，非本任务独有）。需要更深的细节时才按需调：list_facts 分页列事实（最新在前，默认 20 条，可传 q 关键词过滤、before 翻页，返回带 total/has_more）、list_findings 列全部漏洞、**node_detail(id)** 取某条的完整证据/详情（列表/recent_facts 只给摘要）。 - open_intents / running_intents / recent_done_intents——"哪些方向已经有意图在覆盖/已尝试"。每个意图还带 **parents（上游：它派生自哪些事实/意图）和 yields（下游：它产生了哪些事实/发现）**——这就是探索图的**血缘关系**，据此理解"哪些事实来自哪个方向、能否综合成新方向"。recent_facts 里每个事实带 **from_intent**（由哪个意图产生）。 - - worker_directions——用户在 Worker 对话里对每个意图最近一次下发的人工方向（持久化、重启后仍存在）。它比该意图最初的 summary 更新；判断在跑方向、补意图和使用 steer/kill 时必须以它为准，禁止重新派发与人工方向冲突的工作。 - sites_without_endpoints / findings——"哪些方向【可能】需要探索"。 - 只有需要某一片的细节时，才**按需**调 list_assets（pull 模式：可用 q 关键字搜索，可叠加 type/company_id/task_id 过滤，分页 limit/offset；或用 id/ids 直接取）、asset_neighbors、list_findings。资产图全局共享，别默认拉全量。资产中可能包含非本次任务涉及到的资产，所以需要主要出现非本次任务相关的资产时忽略这些资产。 2. 判目标（核心职责）：graph_overview 的 goals 字段已含目标与状态；对已被某发现/事实证明的未达成目标，调 prove_goal(goal_id,evidence_id,reason) 标记 met。**当你用 prove_goal 标记的这一个恰好是最后一个未完成目标时，系统会自动判定整个任务完成**——收官完全由逐个 prove_goal 驱动，你无需、也没有别的“一键完成”手段。 diff --git a/agent/tools.go b/agent/tools.go index 3a34ca1..ebed6d8 100644 --- a/agent/tools.go +++ b/agent/tools.go @@ -48,38 +48,6 @@ func compactIntents(ns []*db.Node, parentsOf, yieldsOf map[int64][]int64) []map[ return out } -const workerDirectionOverviewLimit = 300 -const workerDirectionMessageRunes = 1200 - -// compactWorkerDirections keeps the latest durable human direction for each -// intent small enough for every Planner/Main Agent turn. The full message stays -// available in the Worker activity trace when an unusually long direction is -// truncated here. -func compactWorkerDirections(items []db.WorkerDirection) []map[string]any { - out := make([]map[string]any, 0, len(items)) - for _, item := range items { - message := item.Message - truncated := false - if runes := []rune(message); len(runes) > workerDirectionMessageRunes { - message = string(runes[:workerDirectionMessageRunes]) + "…" - truncated = true - } - row := map[string]any{ - "activity_id": item.ActivityID, - "intent_id": item.IntentID, - "intent_state": item.IntentState, - "intent_summary": item.IntentSummary, - "message": message, - "updated_at": item.CreatedAt, - } - if truncated { - row["truncated"] = true - } - out = append(out, row) - } - return out -} - // ToolSet exposes the PG-backed dual graph (asset + exploration) to an LLM agent. // One ToolSet is created per planner/worker run; per-run signals live here. type ToolSet struct { @@ -392,14 +360,6 @@ func (t *ToolSet) graphOverview() actool.CoreTool { // an empty context and always needs this first). func (t *ToolSet) graphOverviewData() map[string]any { out := map[string]any{} - // worker_directions is the persistent cross-agent steering channel. It has - // one latest accepted human message per local intent, so Planner and Main - // Agent do not have to infer a changed direction from the original summary. - if directions, err := t.ts.LatestWorkerDirections(workerDirectionOverviewLimit); err == nil { - out["worker_directions"] = compactWorkerDirections(directions) - } else { - out["worker_directions"] = []map[string]any{} - } // goals summary folded in so the planner needn't call list_goals each round. goals, _ := t.ts.ListByKind(db.KindGoal, 100) gsum := make([]map[string]any, 0, len(goals)) diff --git a/agent/worker_direction_sync_test.go b/agent/worker_direction_sync_test.go deleted file mode 100644 index a1cf7f8..0000000 --- a/agent/worker_direction_sync_test.go +++ /dev/null @@ -1,155 +0,0 @@ -package agent - -import ( - "context" - "iter" - "strings" - "sync" - "testing" - - "github.com/Autumn-27/artex/db" - "github.com/Autumn-27/norma/llm" -) - -type directionCaptureProvider struct { - mu sync.Mutex - requests []llm.CompletionRequest -} - -func (p *directionCaptureProvider) Stream(_ context.Context, req llm.CompletionRequest) iter.Seq2[llm.StreamEvent, error] { - p.mu.Lock() - p.requests = append(p.requests, req) - p.mu.Unlock() - return func(yield func(llm.StreamEvent, error) bool) { - if !yield(llm.StreamEvent{Type: llm.SETextDelta, Text: "同步完成"}, nil) { - return - } - if !yield(llm.StreamEvent{Type: llm.SEMessageDelta, StopReason: "end_turn"}, nil) { - return - } - yield(llm.StreamEvent{Type: llm.SEMessageStop}, nil) - } -} - -func (p *directionCaptureProvider) Complete(_ context.Context, req llm.CompletionRequest) (llm.Message, string, llm.Usage, error) { - p.mu.Lock() - p.requests = append(p.requests, req) - p.mu.Unlock() - return llm.Message{Role: llm.RoleAssistant, Content: []llm.ContentBlock{llm.TextBlock("同步完成")}}, "end_turn", llm.Usage{}, nil -} - -func (p *directionCaptureProvider) lastRequest(t *testing.T) llm.CompletionRequest { - t.Helper() - p.mu.Lock() - defer p.mu.Unlock() - if len(p.requests) == 0 { - t.Fatal("provider received no completion request") - } - return p.requests[len(p.requests)-1] -} - -func lastUserText(t *testing.T, req llm.CompletionRequest) string { - t.Helper() - for i := len(req.Messages) - 1; i >= 0; i-- { - if req.Messages[i].Role == llm.RoleUser { - return req.Messages[i].Text() - } - } - t.Fatal("completion request has no user message") - return "" -} - -type directionAgentFixture struct { - d *db.DB - task *db.Task - store *db.ExplorationStore - intentID int64 - activity db.Activity -} - -func newDirectionAgentFixture(t *testing.T) *directionAgentFixture { - t.Helper() - d := testDB(t) - task, err := d.CreateTask("agent direction sync", "synchronize worker steering", nil, 0, 0) - if err != nil { - d.Close() - t.Fatal(err) - } - t.Cleanup(func() { - _ = d.DeleteTask(task.ID) - _ = d.Close() - }) - store := d.Exploration(task.ExplorationID) - intentID, err := store.AddIntent(map[string]any{"summary": "原始接口枚举方向"}, 1, nil, "planner") - if err != nil { - t.Fatal(err) - } - if claimed, err := store.ClaimIntent(intentID, "work#direction"); err != nil || !claimed { - t.Fatalf("claim direction fixture: claimed=%v err=%v", claimed, err) - } - const requestID = "agent-direction-sync-1" - const direction = "停止接口枚举，立即聚焦登录后的水平越权" - if _, err := store.ReserveIntentIntervention(intentID, requestID, direction); err != nil { - t.Fatal(err) - } - if changed, err := store.CompareAndSetIntentState(intentID, "running", "paused"); err != nil || !changed { - t.Fatalf("pause direction fixture: changed=%v err=%v", changed, err) - } - activity, duplicate, err := store.AcceptIntentIntervention(intentID, requestID, direction) - if err != nil || duplicate { - t.Fatalf("accept direction fixture: activity=%+v duplicate=%v err=%v", activity, duplicate, err) - } - return &directionAgentFixture{d: d, task: task, store: store, intentID: intentID, activity: activity} -} - -func TestGraphOverviewIncludesLatestWorkerDirection(t *testing.T) { - f := newDirectionAgentFixture(t) - overview := NewToolSet(f.store, "planner").graphOverviewData() - directions, ok := overview["worker_directions"].([]map[string]any) - if !ok || len(directions) != 1 { - t.Fatalf("worker_directions=%T %+v", overview["worker_directions"], overview["worker_directions"]) - } - if directions[0]["activity_id"] != f.activity.ID || directions[0]["intent_id"] != f.intentID || - directions[0]["intent_summary"] != "原始接口枚举方向" || - directions[0]["message"] != "停止接口枚举，立即聚焦登录后的水平越权" { - t.Fatalf("worker direction overview=%+v", directions[0]) - } -} - -func TestPlannerRequestIncludesWorkerMessageTriggerAndPersistedDirection(t *testing.T) { - f := newDirectionAgentFixture(t) - provider := &directionCaptureProvider{} - planner := NewPlanner(provider, "test-model", t.TempDir(), nil, 0, 1) - triggerDetail := "本轮只验证登录后的对象级授权" - _, _, err := planner.Plan(context.Background(), f.task.ID, nil, f.store, f.task.Goal, []TriggerEvent{{ - Kind: TriggerWorkerMessage, ActivityID: f.activity.ID, IntentID: f.intentID, Detail: triggerDetail, - }}, nil) - if err != nil { - t.Fatal(err) - } - input := lastUserText(t, provider.lastRequest(t)) - for _, want := range []string{"【本次触发本轮的实际变动】", triggerDetail, `"worker_directions"`, "停止接口枚举，立即聚焦登录后的水平越权"} { - if !strings.Contains(input, want) { - t.Fatalf("Planner input missing %q:\n%s", want, input) - } - } -} - -func TestMainAgentNextTurnIncludesPersistedWorkerDirection(t *testing.T) { - f := newDirectionAgentFixture(t) - provider := &directionCaptureProvider{} - mainAgent := NewMainAgent(provider, "test-model", t.TempDir(), nil, 0, 1) - const userMessage = "汇报当前执行方向" - if _, err := mainAgent.Chat(context.Background(), f.task.ID, nil, f.store, f.task.Goal, userMessage, nil, nil, nil, nil); err != nil { - t.Fatal(err) - } - input := lastUserText(t, provider.lastRequest(t)) - for _, want := range []string{"【系统同步的 Worker 最新人工方向】", "停止接口枚举，立即聚焦登录后的水平越权", "【用户本轮消息】", userMessage} { - if !strings.Contains(input, want) { - t.Fatalf("Main Agent input missing %q:\n%s", want, input) - } - } - if strings.Index(input, "【系统同步的 Worker 最新人工方向】") > strings.Index(input, "【用户本轮消息】") { - t.Fatalf("Main Agent context is not separated before current user message:\n%s", input) - } -} diff --git a/db/exploration.go b/db/exploration.go index 6a05f9a..1ffb92a 100644 --- a/db/exploration.go +++ b/db/exploration.go @@ -339,7 +339,6 @@ const ( IntentInterventionPending = "pending" IntentInterventionAccepted = "accepted" IntentInterventionRejected = "rejected" - PlannerEventWorkerMessage = "worker_message" ) // IntentIntervention is the durable control record for one human interruption. @@ -356,19 +355,6 @@ type IntentIntervention struct { CreatedAt time.Time } -// WorkerDirection is the latest accepted human direction for one Worker -// intent. The accepted activity is the durable planning event, so this view -// survives restarts and is shared by Planner and Main Agent context builders. -type WorkerDirection struct { - ActivityID int64 - IntentID int64 - RequestID string - Message string - IntentState string - IntentSummary string - CreatedAt time.Time -} - // lockIntentIntervention serializes request-id reservation/finalization across // processes. exploration_nodes.id is a global primary key, so the intent id is // sufficient input to the namespaced advisory key. @@ -577,7 +563,6 @@ WHERE a.exploration_id=$1 AND a.node_id=$2 "intervention_request_id": requestID, "intervention_message": message, "intervention_status": IntentInterventionAccepted, - "planner_event": PlannerEventWorkerMessage, "source": "user", }) if err != nil { @@ -624,52 +609,6 @@ WHERE a.exploration_id=$1 AND a.node_id=$2 return activity, false, nil } -// LatestWorkerDirections returns at most one accepted human direction per -// local intent, newest directions first. It reads the visible user activity -// rather than the hidden hand-off row so archive/restore and activity history -// share one authoritative record. -func (s *ExplorationStore) LatestWorkerDirections(limit int) ([]WorkerDirection, error) { - if limit <= 0 { - limit = 300 - } - rows, err := s.db.Query(` -SELECT activity_id,intent_id,request_id,message,intent_state,intent_summary,created_at -FROM ( - SELECT DISTINCT ON (a.node_id) - a.id AS activity_id, - a.node_id AS intent_id, - COALESCE(a.metadata->>'intervention_request_id','') AS request_id, - COALESCE(a.metadata->>'intervention_message',a.detail,'') AS message, - n.state AS intent_state, - COALESCE(n.payload->>'summary','') AS intent_summary, - a.created_at - FROM activity a - JOIN exploration_nodes n ON n.id=a.node_id AND n.exploration_id=a.exploration_id - WHERE a.exploration_id=$1 - AND a.kind='user' - AND n.kind='intent' - AND a.metadata->>'intervention_status'=$2 - AND a.metadata->>'planner_event'=$3 - ORDER BY a.node_id,a.id DESC -) latest -ORDER BY activity_id DESC -LIMIT $4`, s.expID, IntentInterventionAccepted, PlannerEventWorkerMessage, limit) - if err != nil { - return nil, err - } - defer rows.Close() - out := make([]WorkerDirection, 0) - for rows.Next() { - var item WorkerDirection - if err := rows.Scan(&item.ActivityID, &item.IntentID, &item.RequestID, &item.Message, - &item.IntentState, &item.IntentSummary, &item.CreatedAt); err != nil { - return nil, err - } - out = append(out, item) - } - return out, rows.Err() -} - // RecoverableIntentInterventions returns control operations that must be // reconstructed before ordinary frontier claiming after a process restart: // pending transcript commits, plus accepted hand-offs not yet claimed once. diff --git a/db/intent_intervention_lifecycle_test.go b/db/intent_intervention_lifecycle_test.go index 9363e7c..c27a746 100644 --- a/db/intent_intervention_lifecycle_test.go +++ b/db/intent_intervention_lifecycle_test.go @@ -202,72 +202,3 @@ func TestIntentInterventionAcceptanceCanRetryAfterStateSettlement(t *testing.T) t.Fatalf("retry acceptance: activity=%+v duplicate=%v err=%v", activity, duplicate, err) } } - -func acceptTestWorkerDirection(t *testing.T, store *ExplorationStore, intentID int64, requestID, message string) Activity { - t.Helper() - if claimed, err := store.ClaimIntent(intentID, "direction-test-worker"); err != nil || !claimed { - t.Fatalf("claim direction intent: claimed=%v err=%v", claimed, err) - } - if _, err := store.ReserveIntentIntervention(intentID, requestID, message); err != nil { - t.Fatal(err) - } - if changed, err := store.CompareAndSetIntentState(intentID, "running", "paused"); err != nil || !changed { - t.Fatalf("pause direction intent: changed=%v err=%v", changed, err) - } - activity, duplicate, err := store.AcceptIntentIntervention(intentID, requestID, message) - if err != nil || duplicate { - t.Fatalf("accept direction: activity=%+v duplicate=%v err=%v", activity, duplicate, err) - } - return activity -} - -func TestLatestWorkerDirectionsKeepsLatestPerIntentAcrossStoreReopen(t *testing.T) { - d, err := Open(testDSN(t)) - if err != nil { - t.Skipf("postgres unavailable (%v) - skipping", err) - } - defer d.Close() - task, err := d.CreateTask("worker direction persistence", "keep latest directions", nil, 0, 0) - if err != nil { - t.Fatal(err) - } - defer d.DeleteTask(task.ID) //nolint:errcheck - store := d.Exploration(task.ExplorationID) - firstIntent, err := store.AddIntent(map[string]any{"summary": "first route"}, 1, nil, "planner") - if err != nil { - t.Fatal(err) - } - secondIntent, err := store.AddIntent(map[string]any{"summary": "second route"}, 1, nil, "planner") - if err != nil { - t.Fatal(err) - } - - acceptTestWorkerDirection(t, store, firstIntent, "direction-old", "先检查旧方向") - secondActivity := acceptTestWorkerDirection(t, store, secondIntent, "direction-second", "保留第二条意图方向") - latestActivity := acceptTestWorkerDirection(t, store, firstIntent, "direction-latest", "改为验证最新权限边界") - - d2, err := Open(testDSN(t)) - if err != nil { - t.Fatal(err) - } - defer d2.Close() - directions, err := d2.Exploration(task.ExplorationID).LatestWorkerDirections(10) - if err != nil { - t.Fatal(err) - } - if len(directions) != 2 { - t.Fatalf("directions=%+v, want one latest row per intent", directions) - } - byIntent := make(map[int64]WorkerDirection, len(directions)) - for _, direction := range directions { - byIntent[direction.IntentID] = direction - } - if got := byIntent[firstIntent]; got.ActivityID != latestActivity.ID || got.RequestID != "direction-latest" || - got.Message != "改为验证最新权限边界" || got.IntentSummary != "first route" { - t.Fatalf("latest first direction=%+v", got) - } - if got := byIntent[secondIntent]; got.ActivityID != secondActivity.ID || got.Message != "保留第二条意图方向" || - got.IntentSummary != "second route" { - t.Fatalf("second direction=%+v", got) - } -} diff --git a/server/engine.go b/server/engine.go index 2754c09..165972d 100644 --- a/server/engine.go +++ b/server/engine.go @@ -687,7 +687,6 @@ func (e *Engine) interveneWork(ctx context.Context, t *Task, intentID int64, req } if !duplicate { e.publishStoredActivity(t, activity) - t.NotifyWorkerMessage(intentID, activity.ID, message) } e.readyDirectedClaim(intentID) t.NotifyWorker() @@ -752,10 +751,6 @@ func (e *Engine) recoverWorkerMessages(ctx context.Context, t *Task, allowAdmiss log.Printf("[worker chat] task %s 恢复意图 #%d 待处理消息失败: %v", t.ID, item.IntentID, recoverErr) } case db.IntentInterventionAccepted: - // Reconstruct the Planner-facing event while the Worker hand-off is still - // pending. The durable graph_overview view remains authoritative even if - // this notification was already consumed before a later restart. - t.NotifyWorkerMessage(item.IntentID, item.ActivityID, item.Message) if item.IntentState == "paused" { if changed, setErr := t.Store.CompareAndSetIntentState(item.IntentID, "paused", "open"); setErr != nil { return setErr diff --git a/server/intent_intervention_protocol_test.go b/server/intent_intervention_protocol_test.go index 77ac2e7..8347eca 100644 --- a/server/intent_intervention_protocol_test.go +++ b/server/intent_intervention_protocol_test.go @@ -181,11 +181,6 @@ func TestWorkerMessageCancelsBeforeAcceptAndContinuesSameSessionFirst(t *testing if accepted.err != nil || accepted.result.ActivityID <= pending.ActivityID || accepted.result.Duplicate || accepted.result.State != "open" { t.Fatalf("accepted Worker message=%+v control=%+v err=%v", accepted.result, pending, accepted.err) } - triggers := f.task.drainTriggers() - if len(triggers) != 1 || triggers[0].Kind != agent.TriggerWorkerMessage || - triggers[0].ActivityID != accepted.result.ActivityID || triggers[0].IntentID != f.intentID || triggers[0].Detail != message { - t.Fatalf("accepted Worker message triggers=%+v, want one matching persistent event", triggers) - } node, err := f.task.Store.GetNode(f.intentID) if err != nil || node == nil || node.State != "open" { t.Fatalf("reopened target=%+v err=%v", node, err) @@ -253,39 +248,12 @@ func TestWorkerMessageCancelsBeforeAcceptAndContinuesSameSessionFirst(t *testing if _, err := f.engine.InterveneWork(context.Background(), f.task, f.intentID, requestID, "同 request_id 的另一条消息"); !errors.Is(err, errWorkInterventionConflict) { t.Fatalf("same request_id different message error=%v, want conflict", err) } - if triggers := f.task.drainTriggers(); len(triggers) != 0 { - t.Fatalf("idempotent retries emitted Planner triggers: %+v", triggers) - } count, _ = countTranscriptWorkerMessages(t, f.tx, sessionID, requestID) if count != 1 { t.Fatalf("engine retries appended %d Worker messages, want 1", count) } } -func TestRecoverAcceptedWorkerMessageReconstructsPlannerTrigger(t *testing.T) { - f := newInterventionProtocolFixture(t, 1) - const requestID = "worker-message-recovery-1" - const message = "恢复后优先验证人工指定的权限边界" - if _, err := f.task.Store.ReserveIntentIntervention(f.intentID, requestID, message); err != nil { - t.Fatal(err) - } - if changed, err := f.task.Store.CompareAndSetIntentState(f.intentID, "running", "paused"); err != nil || !changed { - t.Fatalf("pause before recovery acceptance: changed=%v err=%v", changed, err) - } - activity, duplicate, err := f.task.Store.AcceptIntentIntervention(f.intentID, requestID, message) - if err != nil || duplicate { - t.Fatalf("accept before recovery: activity=%+v duplicate=%v err=%v", activity, duplicate, err) - } - if err := f.engine.RecoverWorkerMessages(context.Background(), f.task); err != nil { - t.Fatal(err) - } - triggers := f.task.drainTriggers() - if len(triggers) != 1 || triggers[0].Kind != agent.TriggerWorkerMessage || triggers[0].ActivityID != activity.ID || - triggers[0].IntentID != f.intentID || triggers[0].Detail != message { - t.Fatalf("recovered Planner triggers=%+v", triggers) - } -} - func TestSendWorkerMessageContinuesAfterRequestContextCancellation(t *testing.T) { f := newInterventionProtocolFixture(t, 1) workCtx, cancelWork := context.WithCancelCause(context.Background()) diff --git a/server/manager.go b/server/manager.go index d5c20e8..6fca4dc 100644 --- a/server/manager.go +++ b/server/manager.go @@ -1639,10 +1639,10 @@ func (t *Task) Notify() { } } -// NotifyWorker wakes one idle execution slot without itself scheduling a Planner -// round. Worker-message acceptance separately records a persistent planning event -// and calls NotifyWorkerMessage; keeping execution wake-up independent prevents -// Planner debounce from delaying the directed Worker claim. +// NotifyWorker wakes one idle execution slot without also scheduling a Planner +// round. The channel is separate from notify because human intervention needs +// the reopened intent reclaimed immediately, while graph-change notifications +// are Planner-facing and debounced. A nil channel is tolerated by test fixtures. func (t *Task) NotifyWorker() { if t == nil || t.workerWake == nil { return @@ -1653,28 +1653,6 @@ func (t *Task) NotifyWorker() { } } -// NotifyWorkerMessage immediately schedules a Planner round for an accepted, -// persistent Worker direction. ActivityID identifies the durable kind=user row -// and prevents retries in the same debounce window from duplicating the prompt. -func (t *Task) NotifyWorkerMessage(intentID, activityID int64, message string) { - message = strings.TrimSpace(message) - if intentID <= 0 || activityID <= 0 || message == "" { - return - } - t.trigMu.Lock() - for _, event := range t.pendingTriggers { - if event.Kind == agent.TriggerWorkerMessage && event.ActivityID == activityID { - t.trigMu.Unlock() - return - } - } - t.pendingTriggers = append(t.pendingTriggers, agent.TriggerEvent{ - Kind: agent.TriggerWorkerMessage, ActivityID: activityID, IntentID: intentID, Detail: message, - }) - t.trigMu.Unlock() - t.Notify() -} - // NotifyDone is Notify plus a hint: a worker just finished intentID and that is // what triggered this wake-up. The planner reads the accumulated triggers next // round so it can spell out which intent finished (+ its output). Events pile up