mirror of
https://github.com/RuoJi6/ARTEX.git
synced 2026-10-27 17:03:49 +08:00
默认关闭。开启后「未指定模型」的 agent 在当前配置不可用(402 余额不足 /
401 key 失效 / 429 限流 / 5xx)时自动切到下一个配置;agent 绑定或任务 pin
了配置的默认不轮询(绑定即独占,可由 llm_pool_bind_fallback 打开兜底)。
链序 = 激活配置 + 其余按 priority DESC,排除 pool_exclude 的。只给
llm_profiles 加两列,不引入关联表,所以 agents/tasks/conversations 对
llm_profile_id 的四处引用一行未改。
核心约束:只在本次调用尚未 yield 出任何事件时切换成员。已经吐出
text_delta / tool_use_start 后中途断流一律原样上抛(交给 agentcore 的
resume),否则重发同一 request 会重复输出、污染会话历史。SDK 的 doStream
保证 4xx/5xx 在读 body 前返回,所以目标场景都落在这个安全窗口内。
context.Canceled / DeadlineExceeded 绝不触发切换——那是用户取消或任务
超时,误判会白烧备用 key 并污染 a6bd180 的终止原因诊断。400 也不切
(请求本身有问题,换模型结果一样)。
熔断:402/401/403/404 一次即熔断,429/5xx/网络错误连续 3 次;冷却
60s→5min→30min 退避,到期半开试探,成功清零。Registry 是进程级的,独立
于 provider 缓存,重建链路不丢状态,且全进程共享(A 任务撞见的 402,B 任务
立刻知道)。状态旁路落库,重启只恢复尚未到期的冷却窗口。
上下文窗口装不下当前请求的成员直接跳过(必然 400),不把主力模型的窗口
压到全链最小值。全员熔断时仍探链首,不让引擎停摆。
Pool 包在 llmrec 外面、每个成员自带 recorder,失败调用会以 status=error
+ 正确的 profile 名进 llm_records;切换/熔断走 log.Printf,自动出现在
系统日志页。
顺带修两个既有问题:
- DecomposeGoals 原本自己 NewProvider(),绕过限速器/录制/轮询,且每个
任务新建一个实例——改为由 server 传入共享 provider
- saveLLMConfig 会把 priority/pool_exclude 重置为零值
前端:LLM 页新增轮询卡片(总开关/兜底开关/实际链路顺序/健康徽章/冷却
倒计时/立即恢复),配置编辑区加优先级与「不参与轮询」。
只有一个配置时链长度为 1,poolChain 返回 nil 走裸 provider,行为与开关
打开前完全一致。设计见 docs/LLM轮询设计.md。
243 lines
9.2 KiB
Go
243 lines
9.2 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"github.com/Autumn-27/artex/agent"
|
|
"github.com/Autumn-27/artex/db"
|
|
"github.com/Autumn-27/norma/llm"
|
|
)
|
|
|
|
type goalSpec struct {
|
|
Text string
|
|
VulnClass string
|
|
}
|
|
|
|
// launchTask runs the shared post-creation sequence for a task created via ANY
|
|
// path (HTTP createTask 或 orchestration spawn_task),避免两处复制粘贴:
|
|
// 1. seed 根资产,喂给事件驱动 loop;
|
|
// 2. 可选种子意图,worker 免等首轮 planner 直接开跑;
|
|
// 3. 后台异步做目标分解(发「第 0 轮目标拆解」round + LLM 分解步骤 + 逐条 goal,页面可见),
|
|
// 分解完再 engine.Run —— 引擎在 goal 节点就绪后才启动,避免 planner 抢在 goal 之前跑的竞态。
|
|
//
|
|
// 异步(goroutine)所以调用方立即返回,两条路径行为一致:秒建任务、后台拆目标。
|
|
func (s *Server) launchTask(t *Task, seedText string, seedFirstIntent bool) {
|
|
s.seed(t, seedText)
|
|
if seedFirstIntent {
|
|
s.seedFirstIntent(t)
|
|
}
|
|
// 并发上限:满了就挂起排队(不做目标拆解、不启动引擎),由 reconcileConcurrency 补位启动。
|
|
if s.queueIfAtCapacity(t) {
|
|
return
|
|
}
|
|
go s.startTaskEngine(t)
|
|
}
|
|
|
|
// startTaskEngine does the actual running work: 第0轮目标拆解 + 启动引擎循环。
|
|
// Shared by fresh launches and by the concurrency reconciler promoting a queued task.
|
|
func (s *Server) startTaskEngine(t *Task) {
|
|
s.engine.emitActivity(t, db.Activity{Worker: "planner", Kind: "round",
|
|
Summary: "第 0 轮目标拆解"})
|
|
goals := s.createGoals(s.ctx, t, func(r db.Activity) {
|
|
s.engine.emitActivity(t, r)
|
|
})
|
|
for _, g := range goals {
|
|
summary := g.Text
|
|
if g.VulnClass != "" {
|
|
summary = fmt.Sprintf("[%s] %s", g.VulnClass, g.Text)
|
|
}
|
|
s.engine.emitActivity(t, db.Activity{Worker: "planner", Kind: "text", Summary: summary})
|
|
}
|
|
s.engine.Run(s.ctx, t)
|
|
}
|
|
|
|
// occupiesSlot reports whether a task counts against the concurrency cap: it is
|
|
// live (not terminal), not queued, and not paused. A just-created / decomposing
|
|
// task counts too (it will run momentarily).
|
|
func (s *Server) occupiesSlot(t *Task) bool {
|
|
if t == nil || t.Queued || isTerminalStatus(t.Status) {
|
|
return false
|
|
}
|
|
return !t.Paused && !s.engine.IsPaused(t.ID)
|
|
}
|
|
|
|
// runningTaskCount counts tasks occupying a concurrency slot, excluding excludeID
|
|
// (pass the candidate's id when deciding admission so it doesn't count itself).
|
|
func (s *Server) runningTaskCount(excludeID string) int {
|
|
n := 0
|
|
for _, t := range s.m.List() {
|
|
if t.ID == excludeID {
|
|
continue
|
|
}
|
|
if s.occupiesSlot(t) {
|
|
n++
|
|
}
|
|
}
|
|
return n
|
|
}
|
|
|
|
// queueIfAtCapacity holds a task in the queued state when the concurrency cap is
|
|
// enabled and already full. Returns true if the task was queued (caller must not
|
|
// start it). No-op (false) when the feature is off or a slot is free.
|
|
func (s *Server) queueIfAtCapacity(t *Task) bool {
|
|
s.concMu.Lock()
|
|
defer s.concMu.Unlock()
|
|
enabled, limit := s.m.ConcurrencyLimit()
|
|
if !enabled || s.runningTaskCount(t.ID) < limit {
|
|
return false
|
|
}
|
|
if err := s.m.SetTaskQueued(t.ID, true); err != nil {
|
|
log.Printf("[concurrency] task %s 置排队失败: %v", t.ID, err)
|
|
return false // 落库失败宁可放它跑,也别静默丢任务
|
|
}
|
|
t.Queued = true
|
|
s.engine.emitActivity(t, db.Activity{Worker: "planner", Kind: "text",
|
|
Summary: fmt.Sprintf("已排队:达到并发上限 %d,等待空位后自动开始", limit)})
|
|
log.Printf("[concurrency] task %s 排队(并发上限 %d)", t.ID, limit)
|
|
return true
|
|
}
|
|
|
|
// reconcileConcurrency promotes queued tasks into free slots. Runs on every
|
|
// scheduler tick. When the feature is disabled it releases ALL queued tasks;
|
|
// when enabled it starts the oldest-queued tasks up to (limit - running).
|
|
func (s *Server) reconcileConcurrency() {
|
|
s.concMu.Lock()
|
|
defer s.concMu.Unlock()
|
|
var queued []*Task
|
|
for _, t := range s.m.List() {
|
|
if t.Queued && !isTerminalStatus(t.Status) {
|
|
queued = append(queued, t)
|
|
}
|
|
}
|
|
if len(queued) == 0 {
|
|
return
|
|
}
|
|
// 最早排队的先启动。
|
|
sort.Slice(queued, func(i, j int) bool { return queued[i].CreatedAt < queued[j].CreatedAt })
|
|
|
|
enabled, limit := s.m.ConcurrencyLimit()
|
|
slots := len(queued) // feature off → 全部放行
|
|
if enabled {
|
|
if !s.engine.Ready() {
|
|
return // 引擎未就绪(无 LLM):继续挂起,别空跑
|
|
}
|
|
slots = limit - s.runningTaskCount("")
|
|
}
|
|
for _, t := range queued {
|
|
if slots <= 0 {
|
|
break
|
|
}
|
|
if err := s.m.SetTaskQueued(t.ID, false); err != nil {
|
|
log.Printf("[concurrency] task %s 解除排队失败: %v", t.ID, err)
|
|
continue
|
|
}
|
|
t.Queued = false
|
|
log.Printf("[concurrency] task %s 出队启动", t.ID)
|
|
go s.startTaskEngine(t)
|
|
slots--
|
|
}
|
|
}
|
|
|
|
// reviveTask 让一个已停下的任务重新跑起来:把终态(done/failed/timeout)拉回 running、
|
|
// 解除暂停,并(重)启动引擎循环 + 唤醒。已在 running 且未暂停的任务:只剩 Run 里的一次
|
|
// Notify,近乎无副作用。用于「主 agent set_goals 新增目标」和「重跑 blocked 意图」两处。
|
|
//
|
|
// 为什么必须显式复活:planner/worker 循环的终态门(engine.go)会吞掉普通 notify——光改
|
|
// 图 + Notify 唤不醒已判完成的任务;重启后终态任务的 goroutine 也可能已不在,故还要 Run。
|
|
func (s *Server) reviveTask(t *Task) {
|
|
if t == nil {
|
|
return
|
|
}
|
|
// 排队中的任务不复活启动——它本就等待并发空位,交给 reconcileConcurrency。
|
|
if t.Queued {
|
|
return
|
|
}
|
|
// 终态 → 拉回 running(SetTaskStatus 会清 completed_at 并同步内存 handle 的 Status,
|
|
// 使 planner/worker 循环的终态门放行)。
|
|
if isTerminalStatus(t.Status) {
|
|
if err := s.m.SetTaskStatus(t.ID, "running"); err != nil {
|
|
log.Printf("[revive] task %s 置 running 失败: %v", t.ID, err)
|
|
}
|
|
}
|
|
// 确保引擎循环存活:已 started 只会 Notify;未 started(重启后未恢复的终态任务)则重起循环。
|
|
s.engine.Run(s.ctx, t)
|
|
// 暂停中 → 解除暂停(清引擎内存态 + Notify),并持久化 paused=false(存活过重启)。
|
|
if t.Paused || s.engine.IsPaused(t.ID) {
|
|
s.engine.Resume(t)
|
|
t.Paused = false
|
|
if err := s.m.SetTaskPaused(t.ID, false); err != nil {
|
|
log.Printf("[revive] task %s 解除暂停失败: %v", t.ID, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// createGoals materializes the goal node(s) under the task root (rel objective).
|
|
// Decomposition is done ENTIRELY by the LLM (the project requires an LLM). There is
|
|
// no rule-based fallback splitter — it only ever produced garbage (shredded URLs,
|
|
// meaningless 2-way splits). If the LLM yields nothing (an error), the raw task goal
|
|
// is used verbatim as a single goal so the task still has something to judge against.
|
|
// Returns the seeded specs so callers can emit activity records for them.
|
|
// emit, when non-nil, is forwarded to DecomposeGoals so LLM steps are visible in the UI.
|
|
func (s *Server) createGoals(ctx context.Context, t *Task, emit func(db.Activity)) []goalSpec {
|
|
// Use the SAME LLM the task runs on (its pinned profile, else the active profile),
|
|
// NOT agent.FromEnv() — the LLM is configured via the UI (DB profile), not env vars,
|
|
// so FromEnv returned empty and every task silently fell back to the crude rule
|
|
// splitter (which shredded URLs / made meaningless 2-way splits).
|
|
var specs []goalSpec
|
|
if prov, ok := s.goalsProvider(t); ok {
|
|
var as *db.AssetStore
|
|
if s.m != nil {
|
|
as = s.m.Assets()
|
|
}
|
|
taskID, _ := strconv.ParseInt(t.ID, 10, 64)
|
|
// DecomposeGoals persists each decomposed goal via the set_goals tool straight
|
|
// into t.Store; it returns the specs it wrote so we can emit activity + spot the
|
|
// empty (LLM error / no provider) case below. No write-back needed here anymore.
|
|
for _, g := range agent.DecomposeGoals(ctx, prov, s.m.dir, t.Goal, t.Description, as, t.Store, taskID, emit) {
|
|
if strings.TrimSpace(g.Text) != "" {
|
|
specs = append(specs, goalSpec{Text: g.Text, VulnClass: g.VulnClass})
|
|
}
|
|
}
|
|
}
|
|
if len(specs) == 0 {
|
|
// No decomposed goals (LLM error / no provider): use the raw task goal verbatim
|
|
// as a single goal so the task still has something to judge against. This is the
|
|
// only path that writes here — decomposed goals are already persisted by the tool.
|
|
if g := strings.TrimSpace(t.Goal); g != "" {
|
|
log.Printf("[goals] task %s: LLM 目标拆解无产出,回退为「原始目标作为单目标」", t.ID)
|
|
origin, _ := t.Store.OriginFactID()
|
|
id, _ := t.Store.AddNode(db.KindGoal, map[string]any{"text": g}, 0, "open", "system", nil)
|
|
if origin > 0 && id > 0 {
|
|
_ = t.Store.Link(origin, db.RelSpawns, id) // goal descends from the task root (origin fact)
|
|
}
|
|
specs = []goalSpec{{Text: g}}
|
|
}
|
|
}
|
|
return specs
|
|
}
|
|
|
|
// goalsProvider resolves the provider for goal decomposition by the standard
|
|
// precedence: the goals agent's own binding → the task's pinned profile → the
|
|
// global active provider (the very instance the engine runs on, so decomposition
|
|
// shares its rate limiter, gets recorded, and follows LLM failover).
|
|
func (s *Server) goalsProvider(t *Task) (llm.Provider, bool) {
|
|
var pin *int64
|
|
if t != nil {
|
|
pin = t.LLMProfileID
|
|
}
|
|
if eff := s.effectiveProfileForAgent("goals", pin); eff != nil {
|
|
if prov, cfg, ok := s.providerForProfile(*eff); ok {
|
|
return s.poolForBinding(*eff, prov, cfg), true
|
|
}
|
|
}
|
|
s.cfgMu.Lock()
|
|
prov, on := s.llmProv, s.llmOn
|
|
s.cfgMu.Unlock()
|
|
return prov, on && prov != nil
|
|
}
|