From 21656cc2e31000803782bd532836f7b463bd304c Mon Sep 17 00:00:00 2001 From: Autumn-27 <53844290+Autumn-27@users.noreply.github.com> Date: Sat, 29 Aug 2026 01:37:46 -0400 Subject: [PATCH] fix: restore changes dropped by the upstream sync merge The manual v0.3.6 sync (22132a4) was incomplete, and the follow-up merge (d803ae2) resolved conflicts in favour of that copy, silently reverting three shipped changes. main is already an ancestor of this branch, so re-merging cannot bring them back; restore them explicitly. - schema.sql: re-add the idempotent llm_profiles_format_check migration. CREATE TABLE IF NOT EXISTS leaves an existing table alone, so databases created before openai-responses kept the old two-value CHECK and rejected every openai-responses profile. - orchestration.go: re-add the reseedPlannerPrompt/reseedWorkerPrompt calls, dropped by a comment-alignment hunk while the functions stayed behind as dead code. Without them existing installs never receive the quantified acceptance check or the negative-conclusion evidence threshold. - CHANGELOG.md: restore the 0.3.6 entries, the @neouks credit and the correct norma version range. Also close three recovery gaps in the new cold-archive path: - traffic: write the staging journal even when no legacy host directory exists. Modern installs keep traffic in SQLite only, so a crash between the PostgreSQL commit and the SQLite commit left the archived task's exclusive traffic in the hot store with nothing for recovery to act on. - recoverTaskArchiveStages: treat a missing journal.json as a discardable directory instead of an error, which used to set recoveryFailed and stop the archive worker from starting for the rest of the process lifetime. - stageTaskArchiveFiles: replay the journal's renames when resuming an existing staging root. The journal is written before the first rename, so its presence did not guarantee a complete payload, and a resumed round could package an archive missing transcripts or workspace files. --- CHANGELOG.md | 39 +++++++++++++++++++- db/schema.sql | 5 +++ server/orchestration.go | 2 ++ server/task_archive_package.go | 65 ++++++++++++++++++++++++++++------ traffic/traffic.go | 9 +++-- 5 files changed, 107 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d96ad57..b6e5616 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,7 +29,7 @@ #### 修改的功能 -- 升级 norma v0.2.9 → v0.3.2：修复 OpenAI 系响应里被静默丢弃的 reasoning/refusal（Responses 补 `reasoning_text`）、reasoning 字段三名去重、空响应（零内容块）有界重试、压缩边界切断工具配对导致的网关 400（同时修复已烤进 transcript 的存量孤儿）。 +- 升级 norma v0.3.1 → v0.3.2：修复 OpenAI 系响应里被静默丢弃的 reasoning/refusal（Responses 补 `reasoning_text`）、reasoning 字段三名去重、空响应（零内容块）有界重试、压缩边界切断工具配对导致的网关 400（同时修复已烤进 transcript 的存量孤儿）。 ### 贡献者 @@ -42,10 +42,47 @@ #### 新增的功能 - 每个 LLM 配置支持流式/非流式切换（默认流式）：开走流式 SSE；关走真·非流式（`stream:false`、一次性返回完整 JSON），可绕开部分网关糟糕的 SSE 实现（空帧、思考字段丢帧），代价是失去运行中的实时进度与实时 Token 计数。worker / planner / mainagent / chat / goals 均按当前激活配置动态取值；`llmpool` / `llmrec` / 任务运行时三层 Provider 包装均兼容非流式；`llm_profiles` 新增 `streaming` 列并 `ALTER` 补旧库（默认 `true`，旧配置无感）。 +- 支持 OpenAI Responses API 格式的 LLM 配置：每个配置新增第三种格式 `openai-responses`（打 `POST /v1/responses`），与 Chat Completions / Anthropic 并列；`BaseURL` 归一化、默认模型 `gpt-5`；`llm_profiles` 的 `format` 约束加入 `openai-responses` 并幂等迁移补旧库；前端格式下拉新增「OpenAI (Responses API)」。依赖升级 norma v0.3.1（含 `reasoning_content` 回传修复）。 +- 录制 LLM 请求/响应 HTTP 原文：在 HTTP transport 层捕获真实 wire body，保留归一化视图看不到的工具 schema、`tool_use` 块与原始 SSE 帧；norma 内部重试的多次尝试逐次保留；`llm_records` 新增 `raw_request` / `raw_response` 两列并 `ALTER` 补旧库；录制页详情面板增加「原文」视图切换与请求/响应复制按钮（兼容非安全上下文的 `execCommand` 回退）。 + +### 对话 + +#### 修改的功能 + +- 会话列表多选改为「多选」模式开关：默认列表不再常驻每行勾选框（观感更干净），表头改为「共 N 个 + 多选」按钮；点「多选」进入选择模式（勾选框、全选、批量删除），「完成」退出并清空选择，批量删除全部成功后自动退出（有失败则留在选择模式便于重试）；单条重命名 / 置顶 / 删除仍走每行 ⋯ 菜单。 + +### UI + +#### 修改的功能 + +- 任务管理、会话操作与流量查看体验增强（#57）：任务列表列排序偏好可持久化记忆、行内重命名改用图标直接触发（不再经菜单）、抽屉（sheet）交互与流量查看细节优化。 + +#### 修复的问题 + +- Worker 资产标签只显示域名与 IP，并正确处理空标签的场景。 + +### Agent + +#### 新增的功能 + +- planner 判目标时新增「量化验收核对」：目标含可量化验收条件（资产测试覆盖度达到 X%、拿到 N 个 flag、获得某权限）时，`prove_goal` 前必须核对 `graph_overview` 的实测值（`coverage.pct`、findings 计数等），实测未达标则禁止 `prove_goal`、改为派意图补足差距，不得以「主要部分已完成/大体达成」为由提前标 met。修复了目标要求覆盖度 100%、实测仅 40% 却被判完成的问题。 + +#### 修改的功能 + +- 重写 planner 提示词中「本轮 0 意图」的判定依据：原文将 0 意图描述为「最常见、最重要的原则」，会让规划者在目标未达成、范围内仍有未测面时过早收手。现改为只在两种具体情况下才应产出 0 意图——① 想到的方向都已被 open/running/recent_done 意图覆盖；② 下一步依赖当前正在运行的 work 的产出、而产出尚未出现（应等其跑完、图更新后的下一次唤醒再规划）。并补充反向约束：确有未被覆盖且不依赖在跑 work 的新方向、或目标未达成且仍有未测面时，不要因「0 意图常见」而收手。 +- 强化 worker 提示词中「否定结论的证据门槛」：对「不可注入/端口关闭/无登录入口」等可能让规划者放弃一整条方向的否定结论，要求下结论前先穷尽该意图内的合理手段（换编码/参数/路径/方法）；手段未走完或证据偏弱一律标 `confidence=inferred`，避免用轻率的 `observed` 否定把整条路线焊死（任务早期的错误否定尤其会把方向带偏且难以自愈）。 +- 上述 planner / worker 默认提示词变更通过一次性迁移（`reseedPlannerPrompt` / `reseedWorkerPrompt`，各带 settings flag 守卫）追加为新版本并切为当前版本；旧版本保留在版本历史中，自定义过提示词的用户可在版本记录里恢复。 + +### 运维 + +#### 新增的功能 + +- 新增 `reset-password.sh` 重置管理员（用户名固定 `ARTEX`）密码：支持 local / docker 两种部署，连接信息可显式指定或自动从 `--dsn`/`$ARTEX_PG_DSN`/`config.json` 读取；在库内用 `pgcrypto` 生成与后端登录兼容的 bcrypt 哈希并写回 `settings.auth.password_hash`，重置后无需重启服务。密码经环境变量传入、不进入进程 argv，并做转义防注入。 ### 贡献者 - [@Autumn-27](https://github.com/Autumn-27) +- [@neouks](https://github.com/neouks) ## [0.3.4] - 2026-08-24 diff --git a/db/schema.sql b/db/schema.sql index 88deae6..fe6924e 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -324,6 +324,11 @@ ALTER TABLE llm_profiles ADD COLUMN IF NOT EXISTS priority INTEGER NOT NULL ALTER TABLE llm_profiles ADD COLUMN IF NOT EXISTS pool_exclude BOOLEAN NOT NULL DEFAULT false; -- 流式开关；补旧库。默认 true = 保持既有的流式行为，旧配置无感升级。 ALTER TABLE llm_profiles ADD COLUMN IF NOT EXISTS streaming BOOLEAN NOT NULL DEFAULT true; +-- 放开 format 约束以容纳 openai-responses(OpenAI Responses API)；补旧库。 +-- 每次启动执行,幂等:先删旧 CHECK 再建含三值的新 CHECK。 +ALTER TABLE llm_profiles DROP CONSTRAINT IF EXISTS llm_profiles_format_check; +ALTER TABLE llm_profiles ADD CONSTRAINT llm_profiles_format_check + CHECK (format IN ('openai','anthropic','openai-responses')); -- 思考开关字段 thinking_type，从旧的单一 reasoning_effort 语义一次性拆分而来。 -- schema.sql 每次启动都执行，故迁移必须只跑一次：仅当该列尚不存在时才回填， diff --git a/server/orchestration.go b/server/orchestration.go index 4288d08..191bde2 100644 --- a/server/orchestration.go +++ b/server/orchestration.go @@ -452,6 +452,8 @@ func (s *Server) seedOrchestrationTools() { s.unbindGoalMetDefault() s.reseedGoalsPrompt() // goals 提示词加入「抽操作约束」步 → 旧库追加一版新默认(一次性) s.reseedMainAgentPrompt() // mainagent 提示词加入「目标达成后 add_intent 反问是否建目标」(一次性) + s.reseedPlannerPrompt() // planner 提示词:重写「0 意图」正当理由 + 加量化验收核对(一次性) + s.reseedWorkerPrompt() // worker 提示词:加否定结论证据门槛(一次性) s.seedReporterAgent() // 预置「报告撰写」agent + 工具绑定 + finding 触发器(一次性) // 注：pentest 的默认工具绑定无需迁移——BuiltinToolSeeds 在全新初始化时就把 // list_assets/insert_assets/report_finding/list_findings/list_companies 连同 diff --git a/server/task_archive_package.go b/server/task_archive_package.go index bc55617..a07631f 100644 --- a/server/task_archive_package.go +++ b/server/task_archive_package.go @@ -96,6 +96,17 @@ func stageTaskArchiveFiles(dataDir string, archiveID int64, taskID string, explo if err := json.Unmarshal(raw, &stage.journal); err != nil { return nil, fmt.Errorf("read task archive staging journal: %w", err) } + // The journal is written before the first rename, so its presence does not mean + // the payload is complete: a previous round may have failed mid-loop with a + // rollback that itself errored, which leaves the root in place. Replay the moves + // rather than packaging a payload that is missing transcripts or workspace files. + if err := os.MkdirAll(filepath.Join(stage.payload, "files"), archiveDirMode); err != nil { + return nil, err + } + if err := stage.applyMoves(); err != nil { + _ = stage.rollback() + return nil, err + } return stage, nil } else if !os.IsNotExist(err) { return nil, err @@ -133,20 +144,42 @@ func stageTaskArchiveFiles(dataDir string, archiveID int64, taskID string, explo _ = stage.rollback() return nil, err } - for _, move := range targets { - destination := filepath.Join(stage.payload, move.Relative) - if err := os.MkdirAll(filepath.Dir(destination), archiveDirMode); err != nil { - _ = stage.rollback() - return nil, err - } - if err := os.Rename(move.Source, destination); err != nil { - _ = stage.rollback() - return nil, fmt.Errorf("stage task archive path %s: %w", move.Source, err) - } + if err := stage.applyMoves(); err != nil { + _ = stage.rollback() + return nil, err } return stage, nil } +// applyMoves performs the renames recorded in the journal. Entries already sitting +// in the payload are skipped, so an interrupted staging round can be resumed in +// place without moving anything twice. +func (s *taskArchiveFileStage) applyMoves() error { + for _, move := range s.journal.Moves { + destination := filepath.Join(s.payload, move.Relative) + if _, err := os.Lstat(destination); err == nil { + continue + } else if !os.IsNotExist(err) { + return err + } + if _, err := os.Lstat(move.Source); err != nil { + if os.IsNotExist(err) { + // Neither side exists: the path was removed outside the archive flow + // after the journal was written. Nothing can be staged for it. + continue + } + return err + } + if err := os.MkdirAll(filepath.Dir(destination), archiveDirMode); err != nil { + return err + } + if err := os.Rename(move.Source, destination); err != nil { + return fmt.Errorf("stage task archive path %s: %w", move.Source, err) + } + } + return nil +} + func (s *taskArchiveFileStage) rollback() error { if s == nil || s.done { return nil @@ -368,6 +401,18 @@ func recoverTaskArchiveStages(dataDir string, pg *pgdb.DB) error { } root := filepath.Join(parent, entry.Name()) raw, err := os.ReadFile(filepath.Join(root, "journal.json")) + if os.IsNotExist(err) { + // Either staging died between creating the directory tree and writing the + // journal — no rename had run, so payload holds nothing — or commit/rollback + // failed part-way through removing the root, in which case payload only holds + // copies already inside the package. Without a journal there is nothing to + // roll back, and reporting an error here keeps the archive worker from ever + // starting, so discard the directory instead. + if err := os.RemoveAll(root); err != nil { + errs = append(errs, err) + } + continue + } if err != nil { errs = append(errs, err) continue diff --git a/traffic/traffic.go b/traffic/traffic.go index 6dfbce3..d7a2b7f 100644 --- a/traffic/traffic.go +++ b/traffic/traffic.go @@ -911,7 +911,12 @@ func (t *Traffic) stageTreesForArchive(dirs, hosts []string, archiveID, taskID i } planned = append(planned, stagedTrafficPath{source: source}) } - if len(planned) == 0 { + // 归档路径即使一个历史 host 目录都没有(新装机的流量只落在 SQLite + _blobs) + // 也必须留下 journal：崩溃点若落在 PostgreSQL 提交与 SQLite 提交之间，重启后 + // SQLite 事务被回滚，只有这份 journal 能让恢复流程补做 host 行的删除。少了它， + // 已转冷任务的独占流量会永久留在热库里。 + uniqueHosts := uniqueArchiveHosts(hosts) + if len(planned) == 0 && len(uniqueHosts) == 0 { return "", nil, nil } parent := filepath.Join(t.dir, "_delete_staging") @@ -921,7 +926,7 @@ func (t *Traffic) stageTreesForArchive(dirs, hosts []string, archiveID, taskID i if stageDir, err = os.MkdirTemp(parent, "hosts-"); err != nil { return "", nil, fmt.Errorf("创建流量暂存目录: %w", err) } - journal := hostDeleteStageJournal{Version: 1, ArchiveID: archiveID, TaskID: taskID, Hosts: uniqueArchiveHosts(hosts)} + journal := hostDeleteStageJournal{Version: 1, ArchiveID: archiveID, TaskID: taskID, Hosts: uniqueHosts} for i := range planned { planned[i].staged = filepath.Join(stageDir, fmt.Sprintf("%d-%s", i, filepath.Base(planned[i].source))) journal.Moves = append(journal.Moves, hostDeleteStageMove{Source: planned[i].source, Staged: planned[i].staged})