feat(tasks): add runtime controls and inherited context

This commit is contained in:
pphua committed 2026-08-24 19:29:00 +08:00
1 parent 18926c129f
commit 4392b4fee1
50 files changed
+8291 -663

No files matched your search

+13
View File
@@ -45,9 +45,11 @@ type chatAttachment struct {
// 文件先落这里,前端按返回的 abs 绝对路径写进任务描述)
func (s *Server) chatUpload(w http.ResponseWriter, r *http.Request) {
var sub string
taskScoped := false
switch r.URL.Query().Get("scope") {
case "task":
sub = "tasks"
taskScoped = true
case "session":
sub = "sessions"
case "staging":
@@ -61,6 +63,17 @@ func (s *Server) chatUpload(w http.ResponseWriter, r *http.Request) {
writeErr(w, 400, "非法 id")
return
}
if taskScoped {
if s.m.ResolveTask(id) == nil {
writeErr(w, 404, "task not found")
return
}
if !s.engine.beginTaskOperation(id) {
writeErr(w, http.StatusConflict, "任务正在删除,无法上传附件")
return
}
defer s.engine.decInflight(id)
}
dir := filepath.Join(s.m.dir, sub, id, "uploads")
if err := os.MkdirAll(dir, 0o755); err != nil {
writeErr(w, 500, "建目录失败: "+err.Error())
+523 -137
View File
@@ -71,8 +71,18 @@ func preview(s string, n int) string {
const (
modelErrorRetries = 2 // model_error 收场后额外重试的次数
modelErrorRetryBackoff = 3 * time.Second // 每次重试前的退避
workControlWaitTimeout = 30 * time.Second
)
var errWorkControlConflict = errors.New("work control conflict")
// retryableWorkerModelError excludes errors already handled by the task router.
// In particular, a quota error after partial streaming advances the task cursor
// for the next LLM call but must not replay this whole intent on the backup.
func retryableWorkerModelError(reason harness.TerminalReason, err error) bool {
return reason == harness.ReasonModelError && !isTaskLLMRuntimeError(err)
}
// Engine drives the event-driven exploration loop with real LLM agents
// (docs §4.3/§4.4): on asset/exploration-graph change (debounced) it wakes the
// planner, which reads the route, queries assets, judges goals and emits intents;
@@ -86,25 +96,40 @@ type Engine struct {
bc *Broadcaster // live activity pub/sub (SSE)
mu sync.RWMutex
planner *agent.Planner
worker *agent.Worker
started sync.Map // taskID -> bool, so Run is idempotent per task
lastAct sync.Map // taskID -> int64 unix, last planner/worker activity (heartbeat)
paused sync.Map // taskID -> bool, user-paused (planner + workers idle but loops alive)
dropCnt sync.Map // taskID -> *int64, running count of dropped (unpersistable) activity records
mu sync.RWMutex
planner *agent.Planner
worker *agent.Worker
started sync.Map // taskID -> bool, so Run is idempotent per task
lastAct sync.Map // taskID -> int64 unix, last planner/worker activity (heartbeat)
llmCalls sync.Map // taskID -> *int64, actual planner/worker/main-agent LLM calls
paused sync.Map // taskID -> bool, user-paused (planner + workers idle but loops alive)
deleting sync.Map // taskID -> bool, delete barrier (no new task-owned writes)
dropCnt sync.Map // taskID -> *int64, running count of dropped (unpersistable) activity records
// deleteMu makes installing the delete barrier atomic with registering a new
// task operation. Once BeginDelete returns, every admitted writer is reflected
// in inflight and every later writer is rejected.
deleteMu sync.RWMutex
// Every long-lived task goroutine (planner, workers and deadline coordinator)
// runs under one task-scoped context. Successful deletion cancels that context,
// waits for all goroutines, then releases every task-level Engine reference.
runtimeMu sync.Mutex
runtimes map[string]*taskRuntime
// per-task execution context: each planner.Plan / worker.Execute runs under it,
// so pausing can CANCEL an in-flight run (not just skip the next one). Recreated
// on resume since cancelling is one-shot.
// on resume since cancelling is one-shot. Every cancellation carries a named
// cause so the activity trace can identify the initiating control path.
execMu sync.Mutex
execCancel map[string]context.CancelFunc
execCancel map[string]context.CancelCauseFunc
execCtx map[string]context.Context
// per-work (per running intent) cancel, so the planner's kill_work can stop a
// single worker without pausing the whole task. Keyed by globally-unique intent id.
workMu sync.Mutex
workCancel map[int64]context.CancelFunc
// Per-work control lets the planner kill a worker and lets the UI pause/cancel
// one intent without pausing the whole task. The done channel closes only after
// runWorkerStep has stopped writing and committed its final state.
workMu sync.Mutex
work map[int64]*workExecution
// steerBox queues planner course-corrections for a running work (keyed by intent
// id). The worker's PreToolUse hook drains it before its next tool call and hands
@@ -121,9 +146,23 @@ type Engine struct {
inflight sync.Map // taskID -> *int64, 在跑的 planner.Plan + worker.Execute 计数(用于 drain)
coordStarted sync.Map // taskID -> bool, deadline 协调器是否已启动(Run/reload 去重)
// resolve, if set, returns a task's dedicated planner/worker when it pins a
// specific LLM profile (nil,nil = use the global active pair). Set once at startup.
resolve func(t *Task) (*agent.Planner, *agent.Worker)
// resolve, if set, returns a task's dedicated planner/worker. In legacy mode
// nil,nil falls back to the global pair; an authoritative resolver uses nil,nil
// to mean this task is deliberately unavailable (for example an exhausted chain).
resolve func(t *Task) (*agent.Planner, *agent.Worker)
resolveAuthoritative bool
}
type taskRuntime struct {
ctx context.Context
cancel context.CancelFunc
wg sync.WaitGroup
}
type workExecution struct {
cancel context.CancelCauseFunc
done chan error
action string // user action: pause | cancel
}
// nextPlannerRound returns the next planner round number for a task (1-based).
@@ -136,18 +175,128 @@ func (e *Engine) nextPlannerRound(taskID string) int {
// Pause stops a task: marks it paused AND cancels any in-flight planner/worker run
// for it (a long worker.Execute would otherwise keep going until it finishes).
func (e *Engine) Pause(taskID string) {
func (e *Engine) Pause(taskID string, cause error) {
e.paused.Store(taskID, true)
e.cancelExec(taskID)
e.cancelExec(taskID, cause)
}
// BeginDelete installs an execution barrier before task data/files are removed.
// The temporary pause is not a user pause. The server serializes this transition
// with lifecycle admission and tells AbortDelete whether the persisted task is
// paused/queued if cleanup fails.
func (e *Engine) BeginDelete(taskID string) bool {
e.deleteMu.Lock()
if _, loaded := e.deleting.LoadOrStore(taskID, true); loaded {
e.deleteMu.Unlock()
return false
}
e.paused.Store(taskID, true)
e.deleteMu.Unlock()
e.cancelExec(taskID, agent.AbortTaskDeleted)
return true
}
func (e *Engine) AbortDelete(taskID string, keepPaused bool) {
e.deleteMu.Lock()
if !e.IsDeleting(taskID) {
e.deleteMu.Unlock()
return
}
e.deleting.Delete(taskID)
if !keepPaused {
e.paused.Delete(taskID)
}
e.deleteMu.Unlock()
if !keepPaused && e.m != nil {
if t, ok := e.m.Task(taskID); ok {
t.Notify()
}
}
}
func (e *Engine) IsDeleting(taskID string) bool {
_, ok := e.deleting.Load(taskID)
return ok
}
// registerTaskRoutines reserves count goroutines in the task runtime. Callers
// hold deleteMu for reading so StopTask cannot race WaitGroup.Add with Wait.
func (e *Engine) registerTaskRoutines(parent context.Context, taskID string, count int) *taskRuntime {
e.runtimeMu.Lock()
defer e.runtimeMu.Unlock()
rt := e.runtimes[taskID]
if rt == nil {
ctx, cancel := context.WithCancel(parent)
rt = &taskRuntime{ctx: ctx, cancel: cancel}
e.runtimes[taskID] = rt
}
rt.wg.Add(count)
return rt
}
func runTaskRoutine(rt *taskRuntime, fn func(context.Context)) {
go func() {
defer rt.wg.Done()
fn(rt.ctx)
}()
}
// StopTask permanently stops every long-lived goroutine and removes all Engine
// state for a successfully deleted task. The delete barrier remains installed
// until cleanup finishes, so no new task operation can race the teardown.
func (e *Engine) StopTask(taskID string) {
e.deleteMu.Lock()
e.deleting.Store(taskID, true)
e.deleteMu.Unlock()
e.cancelExec(taskID, agent.AbortTaskDeleted)
e.runtimeMu.Lock()
rt := e.runtimes[taskID]
if rt != nil {
rt.cancel()
}
e.runtimeMu.Unlock()
if rt != nil {
rt.wg.Wait()
}
e.execMu.Lock()
if cancel := e.execCancel[taskID]; cancel != nil {
cancel(agent.AbortTaskDeleted)
}
delete(e.execCancel, taskID)
delete(e.execCtx, taskID)
e.execMu.Unlock()
e.runtimeMu.Lock()
if e.runtimes[taskID] == rt {
delete(e.runtimes, taskID)
}
e.runtimeMu.Unlock()
e.started.Delete(taskID)
e.lastAct.Delete(taskID)
e.llmCalls.Delete(taskID)
e.paused.Delete(taskID)
e.dropCnt.Delete(taskID)
e.plannerRound.Delete(taskID)
e.settling.Delete(taskID)
e.deadline.Delete(taskID)
e.stamped.Delete(taskID)
e.inflight.Delete(taskID)
e.coordStarted.Delete(taskID)
e.deleteMu.Lock()
e.deleting.Delete(taskID)
e.deleteMu.Unlock()
}
// cancelExec cancels a task's current per-task exec context (any in-flight
// planner.Plan / worker.Execute), if present. Shared by Pause and the settle
// sequence's hard-drain backstop.
func (e *Engine) cancelExec(taskID string) {
func (e *Engine) cancelExec(taskID string, cause error) {
e.execMu.Lock()
if cancel := e.execCancel[taskID]; cancel != nil {
cancel()
cancel(cause)
}
e.execMu.Unlock()
}
@@ -155,6 +304,17 @@ func (e *Engine) cancelExec(taskID string) {
// Resume un-pauses a task and nudges a fresh planning round. The next exec under
// it gets a fresh (uncancelled) context.
func (e *Engine) Resume(t *Task) {
// BeginDelete owns the pause barrier once deletion starts. A concurrent
// resume must never clear it and let a planner/worker re-enter while cleanup
// is waiting for task operations to drain.
if t == nil {
return
}
e.deleteMu.RLock()
defer e.deleteMu.RUnlock()
if e.IsDeleting(t.ID) {
return
}
e.paused.Delete(t.ID)
t.Notify()
}
@@ -166,14 +326,14 @@ func (e *Engine) execContextFor(parent context.Context, taskID string) context.C
defer e.execMu.Unlock()
if e.IsPaused(taskID) {
// never hand out a live context while paused (guards the claim→Execute race)
c, cancel := context.WithCancel(parent)
cancel()
c, cancel := context.WithCancelCause(parent)
cancel(agent.AbortPausedRaceGuard)
return c
}
if c := e.execCtx[taskID]; c != nil && c.Err() == nil {
return c
}
c, cancel := context.WithCancel(parent)
c, cancel := context.WithCancelCause(parent)
e.execCtx[taskID] = c
e.execCancel[taskID] = cancel
return c
@@ -200,32 +360,131 @@ func (e *Engine) LastActivity(taskID string) int64 {
return 0
}
// BeginLLMCall/EndLLMCall track actual provider calls separately from the
// scheduler's task-operation counter. A task can have live loops while all of
// them are waiting for a trigger; that state must remain idle in the UI.
func (e *Engine) BeginLLMCall(taskID string) {
v, _ := e.llmCalls.LoadOrStore(taskID, new(int64))
atomic.AddInt64(v.(*int64), 1)
}
func (e *Engine) EndLLMCall(taskID string) {
if v, ok := e.llmCalls.Load(taskID); ok {
p := v.(*int64)
if atomic.AddInt64(p, -1) <= 0 {
atomic.StoreInt64(p, 0)
}
}
}
func (e *Engine) ActiveLLMCalls(taskID string) int64 {
if v, ok := e.llmCalls.Load(taskID); ok {
return atomic.LoadInt64(v.(*int64))
}
return 0
}
func (e *Engine) touch(taskID string) { e.lastAct.Store(taskID, time.Now().Unix()) }
func NewEngine(m *Manager) *Engine {
return &Engine{m: m, debounce: 800 * time.Millisecond, bc: NewBroadcaster(),
execCancel: map[string]context.CancelFunc{}, execCtx: map[string]context.Context{},
workCancel: map[int64]context.CancelFunc{}, steerBox: map[int64][]string{}}
execCancel: map[string]context.CancelCauseFunc{}, execCtx: map[string]context.Context{},
work: map[int64]*workExecution{}, steerBox: map[int64][]string{},
runtimes: map[string]*taskRuntime{}}
}
// registerWork records the cancel for the work currently running intentID.
func (e *Engine) registerWork(intentID int64, cancel context.CancelFunc) {
func (e *Engine) registerWork(intentID int64, cancel context.CancelCauseFunc) {
e.workMu.Lock()
e.workCancel[intentID] = cancel
e.work[intentID] = &workExecution{cancel: cancel, done: make(chan error, 1)}
e.workMu.Unlock()
}
// unregisterWork removes and releases the work's cancel (called when Execute returns).
func (e *Engine) unregisterWork(intentID int64) {
// detachWork removes the live control handle once Execute has returned. complete
// must be called after the final intent state write so a waiting cancel handler can
// safely delete the worker's blackboard output without racing a late write.
func (e *Engine) detachWork(intentID int64) (action string, complete func(error)) {
e.workMu.Lock()
if c := e.workCancel[intentID]; c != nil {
delete(e.workCancel, intentID)
c() // release context resources (no-op if already cancelled)
run := e.work[intentID]
if run != nil {
delete(e.work, intentID)
action = run.action
run.cancel(agent.AbortWorkFinished) // release resources (no-op if already cancelled)
}
e.workMu.Unlock()
e.steerMu.Lock()
delete(e.steerBox, intentID) // drop any undelivered steering for a finished work
e.steerMu.Unlock()
if run == nil {
return action, func(error) {}
}
return action, func(err error) { run.done <- err }
}
// ControlWork requests a user-visible pause or cancellation and waits until the
// worker has fully stopped writing. Cancellation cleanup is performed by the API
// handler after this returns; pause state is committed by runWorkerStep itself.
func (e *Engine) ControlWork(ctx context.Context, intentID int64, action string) error {
if action != "pause" && action != "cancel" {
return fmt.Errorf("unsupported work action %q", action)
}
if ctx == nil {
ctx = context.Background()
}
e.workMu.Lock()
run := e.work[intentID]
if run == nil {
e.workMu.Unlock()
return fmt.Errorf("%w: 意图 %d 当前没有运行中的 work(可能已结束或未被领取)", errWorkControlConflict, intentID)
}
if run.action != "" {
e.workMu.Unlock()
return fmt.Errorf("%w: 意图 %d 正在执行 %s 操作", errWorkControlConflict, intentID, run.action)
}
run.action = action
done := run.done
cause := error(agent.AbortWorkPausedByUser)
if action == "cancel" {
cause = agent.AbortWorkCancelledByUser
}
run.cancel(cause)
e.workMu.Unlock()
timer := time.NewTimer(workControlWaitTimeout)
defer timer.Stop()
select {
case err := <-done:
return err
case <-ctx.Done():
e.releaseWorkControl(intentID, run, action)
return fmt.Errorf("等待意图 %d %s 收尾: %w", intentID, action, ctx.Err())
case <-timer.C:
e.releaseWorkControl(intentID, run, action)
return fmt.Errorf("等待意图 %d %s 收尾: %w", intentID, action, context.DeadlineExceeded)
}
}
// releaseWorkControl drops only this caller's reservation after its wait is
// cancelled. The work context stays cancelled; runWorkerStep recognizes the
// named cancellation cause and settles the intent into the recoverable paused
// state even if the HTTP caller has gone away.
func (e *Engine) releaseWorkControl(intentID int64, run *workExecution, action string) {
e.workMu.Lock()
if current := e.work[intentID]; current == run && current.action == action {
current.action = ""
}
e.workMu.Unlock()
}
func transitionIntentState(store *db.ExplorationStore, intentID int64, expected, state string) error {
changed, err := store.CompareAndSetIntentState(intentID, expected, state)
if err != nil {
return err
}
if !changed {
return fmt.Errorf("%w: 意图 %d 不再是 %s 状态", db.ErrIntentStateConflict, intentID, expected)
}
return nil
}
// SteerWork queues a mid-run course-correction for the work running intentID (the
@@ -236,7 +495,7 @@ func (e *Engine) SteerWork(intentID int64, msg string) error {
return fmt.Errorf("纠偏消息不能为空")
}
e.workMu.Lock()
running := e.workCancel[intentID] != nil
running := e.work[intentID] != nil
e.workMu.Unlock()
if !running {
return fmt.Errorf("意图 %d 当前没有运行中的 work(可能已结束或未被领取)", intentID)
@@ -301,12 +560,12 @@ func (h steerHooks) Stop(ctx context.Context, messages []llm.Message) (bool, []s
// The work's agent-core session honors ctx cancellation and aborts promptly.
func (e *Engine) KillWork(intentID int64) error {
e.workMu.Lock()
c := e.workCancel[intentID]
run := e.work[intentID]
e.workMu.Unlock()
if c == nil {
if run == nil {
return fmt.Errorf("意图 %d 当前没有运行中的 work(可能已结束或未被领取)", intentID)
}
c()
run.cancel(agent.AbortKilledByPlanner)
return nil
}
@@ -315,7 +574,7 @@ func (e *Engine) Broadcaster() *Broadcaster { return e.bc }
// emitActivity persists one captured step AND fans it out to live subscribers,
// from a single point so storage and the SSE stream never diverge.
func (e *Engine) emitActivity(t *Task, r db.Activity) {
func (e *Engine) emitActivity(t *Task, r db.Activity) db.Activity {
id, err := e.appendActivity(t, r)
if err != nil {
// NO LONGER SILENT: dropping a record breaks command↔result pairing in the
@@ -340,7 +599,7 @@ func (e *Engine) emitActivity(t *Task, r db.Activity) {
log.Printf("[activity] task %s 丢弃活动记录(该任务累计第 %d 条) worker=%s kind=%s tool=%s tuid=%s reason=%s summary=%q: %v%s",
t.ID, n, r.Worker, r.Kind, r.Tool, r.ToolUseID, dropReason(err), preview(r.Summary, 80), err, diag)
e.touch(t.ID)
return
return r
}
r.ID = id
if r.CreatedAt.IsZero() {
@@ -348,6 +607,7 @@ func (e *Engine) emitActivity(t *Task, r db.Activity) {
}
e.bc.Publish(t.ID, r)
e.touch(t.ID)
return r
}
// appendActivity persists one activity row, retrying briefly on write failure.
@@ -390,6 +650,15 @@ func (e *Engine) snapshot() (*agent.Planner, *agent.Worker) {
// Called once at startup before any task loop runs, so no lock is needed on reads.
func (e *Engine) SetAgentResolver(fn func(t *Task) (*agent.Planner, *agent.Worker)) {
e.resolve = fn
e.resolveAuthoritative = false
}
// SetAuthoritativeAgentResolver installs a resolver whose nil result must not
// fall through to the global provider. Task-level failover chains use this so a
// fully exhausted chain cannot silently bypass its configured boundary.
func (e *Engine) SetAuthoritativeAgentResolver(fn func(t *Task) (*agent.Planner, *agent.Worker)) {
e.resolve = fn
e.resolveAuthoritative = true
}
// snapshotFor returns the planner/worker a task should run on: the task's dedicated
@@ -397,7 +666,8 @@ func (e *Engine) SetAgentResolver(fn func(t *Task) (*agent.Planner, *agent.Worke
// pair. The resolver returning nil means "no override" (fall back to global).
func (e *Engine) snapshotFor(t *Task) (*agent.Planner, *agent.Worker) {
if e.resolve != nil {
if p, w := e.resolve(t); p != nil && w != nil {
p, w := e.resolve(t)
if (p != nil && w != nil) || e.resolveAuthoritative {
return p, w
}
}
@@ -410,6 +680,14 @@ func (e *Engine) Ready() bool {
return p != nil && w != nil
}
// ReadyFor reports whether a specific task can resolve a planner/worker pair.
// An explicit task profile chain can be runnable even when no global default
// provider is configured, so task status must not rely on Ready alone.
func (e *Engine) ReadyFor(t *Task) bool {
p, w := e.snapshotFor(t)
return p != nil && w != nil
}
// Mode reports "llm" when ready, else "idle".
func (e *Engine) Mode() string {
if e.Ready() {
@@ -422,15 +700,24 @@ func (e *Engine) Mode() string {
// but no-op until an LLM is configured (so a task created while idle picks up
// automatically once LLM is set from the UI).
func (e *Engine) Run(ctx context.Context, t *Task) {
workers := e.m.Workers()
e.deleteMu.RLock()
if e.IsDeleting(t.ID) {
e.deleteMu.RUnlock()
return
}
if _, loaded := e.started.LoadOrStore(t.ID, true); loaded {
e.deleteMu.RUnlock()
t.Notify() // already running — just nudge a planning round
return
}
rt := e.registerTaskRoutines(ctx, t.ID, 1+workers)
e.deleteMu.RUnlock()
e.touch(t.ID)
go e.plannerLoop(ctx, t)
workers := e.m.Workers()
runTaskRoutine(rt, func(loopCtx context.Context) { e.plannerLoop(loopCtx, t) })
for i := 0; i < workers; i++ {
go e.workerLoop(ctx, t, fmt.Sprintf("work#%d", i+1))
name := fmt.Sprintf("work#%d", i+1)
runTaskRoutine(rt, func(loopCtx context.Context) { e.workerLoop(loopCtx, t, name) })
}
e.startDeadlineCoordinator(ctx, t) // 任务级超时定时器(仅 timeout>0;去重)
// 仅在「完全没有活动意图(open+running)」时才 kick 首轮规划。带种子意图的任务:种子已
@@ -491,10 +778,13 @@ func (e *Engine) plannerLoop(ctx context.Context, t *Task) {
if e.IsPaused(t.ID) {
return // user-paused: don't plan
}
if e.IsDeleting(t.ID) {
return
}
// terminal task (goals all met → done, or failed): the run is over. A
// resume/nudge — e.g. auto-resume of the active task on restart — must NOT
// re-plan (it would burn an LLM round and re-confirm a settled result).
if isTerminalStatus(t.Status) {
if isTerminalStatus(t.lifecycleSnapshot().Status) {
return
}
// 任务级超时收尾中:丢弃普通唤醒——worker 收尾写回、Resume 的 Notify 都不再
@@ -502,10 +792,17 @@ func (e *Engine) plannerLoop(ctx context.Context, t *Task) {
if e.isSettling(t.ID) {
return
}
if !e.beginTaskOperation(t.ID) {
return
}
defer e.decInflight(t.ID)
e.stampFirstRun(t) // 首次真正规划 → 盖 first_run_at + 算 deadline(仅带 timeout 的任务)
e.touch(t.ID)
emit := func(r db.Activity) { e.emitActivity(t, r) }
ectx := e.clockCtx(e.execContextFor(ctx, t.ID), t, false) // cancellable by Pause; 带任务 deadline
if ectx.Err() != nil || e.IsDeleting(t.ID) {
return
}
log.Printf("[planner] task %s 规划中…(%s 触发)", t.ID, src)
// round marker: each Plan() is one planner round; emit a boundary so the
// UI can separate rounds in the transcript (kind='round').
@@ -514,10 +811,10 @@ func (e *Engine) plannerLoop(ctx context.Context, t *Task) {
// what fired this round (worker done / finding; may be several — debounce
// coalesces a burst; empty for time/heartbeat wakes).
triggers := t.drainTriggers()
e.incInflight(t.ID)
taskIDInt, _ := strconv.ParseInt(t.ID, 10, 64)
e.BeginLLMCall(t.ID)
met, reason, err := planner.Plan(ectx, taskIDInt, e.m.assets, t.Store, t.Goal, triggers, emit)
e.decInflight(t.ID)
e.EndLLMCall(t.ID)
switch {
case err != nil && ectx.Err() == nil:
log.Printf("[planner] task %s 规划出错: %v", t.ID, err)
@@ -530,7 +827,7 @@ func (e *Engine) plannerLoop(ctx context.Context, t *Task) {
// 任务已判完成 → 立刻取消在跑的 worker:它们手头的意图跑出来也没意义了。
// 下一轮 worker 循环撞终态门就不再领新意图;被取消的这批走下方"任务已完成"分支
// 归为 stopped(而非 blocked)。
e.cancelExec(t.ID)
e.cancelExec(t.ID, agent.AbortGoalMet)
default:
log.Printf("[planner] task %s 规划完成", t.ID)
}
@@ -572,6 +869,9 @@ func (e *Engine) workerLoop(ctx context.Context, t *Task, name string) {
}
continue // user-paused: don't claim/execute intents
}
if e.IsDeleting(t.ID) {
return
}
if e.isSettling(t.ID) {
if sleepCtx(ctx, 1000*time.Millisecond) {
return
@@ -584,105 +884,191 @@ func (e *Engine) workerLoop(ctx context.Context, t *Task, name string) {
}
continue // 任务已终态(done/failed/timeout):停止领取遗留意图,别在完成后空跑 frontier
}
intent := e.claimNext(t, name)
if intent == nil {
if sleepCtx(ctx, 800*time.Millisecond) {
return
}
continue
}
log.Printf("[worker %s] task %s 领取意图 #%d", name, t.ID, intent.ID)
e.stampFirstRun(t) // 首次真正执行 → 盖 first_run_at + 算 deadline(仅带 timeout 的任务)
e.touch(t.ID)
emit := func(r db.Activity) { e.emitActivity(t, r) }
ectx := e.clockCtx(e.execContextFor(ctx, t.ID), t, false) // cancellable by Pause; 带任务 deadline
// per-work child context so the planner's kill_work can stop just this work.
workCtx, workCancel := context.WithCancel(ectx)
e.registerWork(intent.ID, workCancel)
// wrap the guard hooks so steer_work can inject a mid-run course-correction
// for THIS intent (drained before the worker's next tool call).
iid := intent.ID
taskEmit := func(a db.Activity) {
nid := iid
a.NodeID, a.Worker = &nid, name
emit(a)
}
workCtx = intercept.WithTaskContext(workCtx, t.ID, fmt.Sprintf("%s · #%d", name, iid), taskEmit)
hooks := steerHooks{inner: t.Guard.Hooks(), drain: func() (string, bool) { return e.drainSteer(iid) }}
wTaskID, _ := strconv.ParseInt(t.ID, 10, 64)
e.incInflight(t.ID) // 计入在跑,供收尾时序 drain 等待
reason, wrote, err := worker.Execute(workCtx, name, wTaskID, e.m.assets, t.Store, intent, hooks, emit, e.m.enrich, t.NotifyFinding)
// model_error 收场 → 额外重跑几次(退避后再试)。仅在意图仍属本 work、任务
// 未暂停/未终止/未取消【且未进入收尾】时重试;否则让位给对应分支处理(收尾期不
// 再重试,避免退避挤占其他 worker 的优雅收尾窗口)。
for attempt := 1; attempt <= modelErrorRetries &&
reason == harness.ReasonModelError &&
workCtx.Err() == nil && ectx.Err() == nil && !e.IsPaused(t.ID) && !e.isSettling(t.ID); attempt++ {
log.Printf("[worker %s] task %s 意图 #%d model_error 收场,%v 后重试 (%d/%d)",
name, t.ID, intent.ID, modelErrorRetryBackoff, attempt, modelErrorRetries)
if sleepCtx(workCtx, modelErrorRetryBackoff) {
break // 退避期间被取消(终止/暂停)→ 交给下方分支处理
}
reason, wrote, err = worker.Execute(workCtx, name, wTaskID, e.m.assets, t.Store, intent, hooks, emit, e.m.enrich, t.NotifyFinding)
if !e.beginTaskOperation(t.ID) {
return
}
claimed := e.runWorkerStep(ctx, t, name, worker)
e.decInflight(t.ID)
// CAPTURE kill state BEFORE unregisterWork cancels workCtx. kill = this work's
// ctx was cancelled (planner kill_work) while the TASK ctx kept running; a
// pause cancels the task ctx (ectx) instead. Checking workCtx.Err() AFTER
// unregister would always be true (unregister cancels it) → every completed
// work would be wrongly marked stopped.
killed := workCtx.Err() != nil && ectx.Err() == nil
e.unregisterWork(intent.ID)
// if a pause cancelled this run mid-flight, return the intent to the frontier
// so it is re-claimed (and re-run from scratch) on resume — not marked done.
if ectx.Err() != nil && e.IsPaused(t.ID) {
_ = t.Store.SetIntentState(intent.ID, "open")
continue
if !claimed && sleepCtx(ctx, 800*time.Millisecond) {
return
}
// 任务超时收尾的硬兜底 cancel(非 pause、非 kill)取消了本 run → 归为 exhausted(已收尾),
// 不要误标 blocked。此时 worker 通常已在 settlement 阶段把结果写回。
if ectx.Err() != nil && e.isSettling(t.ID) {
_ = t.Store.SetIntentState(intent.ID, "exhausted")
log.Printf("[worker %s] task %s 意图 #%d 因任务超时收尾结束(exhausted),写回 %s", name, t.ID, intent.ID, wrote)
e.touch(t.ID)
continue
}
}
// runWorkerStep claims and fully settles at most one intent. Its caller holds one
// task-operation admission for the whole claim/execute/state-write sequence so a
// delete cannot observe quiescence between the LLM return and the final DB writes.
func (e *Engine) runWorkerStep(ctx context.Context, t *Task, name string, worker *agent.Worker) bool {
intent := e.claimNext(t, name)
if intent == nil {
return false
}
log.Printf("[worker %s] task %s 领取意图 #%d", name, t.ID, intent.ID)
e.stampFirstRun(t) // 首次真正执行 → 盖 first_run_at + 算 deadline(仅带 timeout 的任务)
e.touch(t.ID)
emit := func(r db.Activity) { e.emitActivity(t, r) }
ectx := e.clockCtx(e.execContextFor(ctx, t.ID), t, false) // cancellable by Pause; 带任务 deadline
if ectx.Err() != nil || e.IsDeleting(t.ID) {
if err := transitionIntentState(t.Store, intent.ID, "running", "open"); err != nil {
log.Printf("[worker %s] task %s 意图 #%d 领取后回退失败: %v", name, t.ID, intent.ID, err)
}
// 任务已判完成(done via 常规路径)→ 上面 cancelExec 取消了本 run。意图结果已无意义,
// 标 stopped(不是 blocked),别污染已完成任务的意图状态。
if ectx.Err() != nil && isTerminalStatus(e.m.TaskStatus(t.ID)) {
_ = t.Store.SetIntentState(intent.ID, "stopped")
log.Printf("[worker %s] task %s 意图 #%d 因任务已完成而取消(stopped)", name, t.ID, intent.ID)
e.touch(t.ID)
continue
return true
}
// per-work child context so the planner's kill_work can stop just this work.
workCtx, workCancel := context.WithCancelCause(ectx)
e.registerWork(intent.ID, workCancel)
// wrap the guard hooks so steer_work can inject a mid-run course-correction
// for THIS intent (drained before the worker's next tool call).
iid := intent.ID
taskEmit := func(a db.Activity) {
nid := iid
a.NodeID, a.Worker = &nid, name
emit(a)
}
workCtx = intercept.WithTaskContext(workCtx, t.ID, fmt.Sprintf("%s · #%d", name, iid), taskEmit)
hooks := steerHooks{inner: t.Guard.Hooks(), drain: func() (string, bool) { return e.drainSteer(iid) }}
wTaskID, _ := strconv.ParseInt(t.ID, 10, 64)
e.BeginLLMCall(t.ID)
reason, wrote, err := worker.Execute(workCtx, name, wTaskID, e.m.assets, t.Store, intent, hooks, emit, e.m.enrich, t.NotifyFinding)
e.EndLLMCall(t.ID)
// model_error 收场 → 额外重跑几次(退避后再试)。仅在意图仍属本 work、任务
// 未暂停/未终止/未取消【且未进入收尾】时重试;否则让位给对应分支处理(收尾期不
// 再重试,避免退避挤占其他 worker 的优雅收尾窗口)。
for attempt := 1; attempt <= modelErrorRetries &&
retryableWorkerModelError(reason, err) &&
workCtx.Err() == nil && ectx.Err() == nil && !e.IsPaused(t.ID) && !e.isSettling(t.ID); attempt++ {
log.Printf("[worker %s] task %s 意图 #%d model_error 收场,%v 后重试 (%d/%d)",
name, t.ID, intent.ID, modelErrorRetryBackoff, attempt, modelErrorRetries)
if sleepCtx(workCtx, modelErrorRetryBackoff) {
break // 退避期间被取消(终止/暂停)→ 交给下方分支处理
}
// killed by the planner: mark stopped (don't write back results, don't auto-reclaim).
if killed {
_ = t.Store.SetIntentState(intent.ID, "stopped")
log.Printf("[worker %s] task %s 意图 #%d 被终止(stopped)", name, t.ID, intent.ID)
e.touch(t.ID)
t.Notify()
continue
}
if err != nil {
log.Printf("[worker %s] intent %d: %v", name, intent.ID, err)
}
// terminal分流:撞步数上限 ≠ 完成。max_turns→exhausted(规划者据此知道这个方向
// 试过但没真正做完、需换角度,而非当成已覆盖永久跳过);出错→blocked;正常→done。
state := "done"
e.BeginLLMCall(t.ID)
reason, wrote, err = worker.Execute(workCtx, name, wTaskID, e.m.assets, t.Store, intent, hooks, emit, e.m.enrich, t.NotifyFinding)
e.EndLLMCall(t.ID)
}
// Capture kill state before detachWork cancels workCtx. kill = this work's
// ctx was cancelled (planner kill_work) while the TASK ctx kept running; a
// pause cancels the task ctx (ectx) instead. Checking workCtx.Err() AFTER
// unregister would always be true (unregister cancels it) → every completed
// work would be wrongly marked stopped.
workCause := context.Cause(workCtx)
killed := workCtx.Err() != nil && ectx.Err() == nil
action, completeWork := e.detachWork(intent.ID)
// A caller may stop waiting and release its in-memory reservation before the
// agent honors cancellation. The named context cause remains authoritative and
// still settles the stopped run into a recoverable state.
if action == "" {
switch {
case err != nil:
state = "blocked"
case reason == harness.ReasonMaxTurns:
state = "exhausted"
log.Printf("[worker %s] intent %d 撞步数上限(exhausted),本次写回 %s", name, intent.ID, wrote)
case reason == harness.ReasonTimeout:
state = "exhausted"
log.Printf("[worker %s] intent %d 运行超时(exhausted),收尾后写回 %s", name, intent.ID, wrote)
case errors.Is(workCause, agent.AbortWorkPausedByUser):
action = "pause"
case errors.Is(workCause, agent.AbortWorkCancelledByUser):
action = "cancel"
}
_ = t.Store.SetIntentState(intent.ID, state)
log.Printf("[worker %s] task %s 意图 #%d 结束: %s (写回 %s)", name, t.ID, intent.ID, state, wrote)
}
var controlErr error
defer func() { completeWork(controlErr) }()
if action == "pause" {
controlErr = transitionIntentState(t.Store, intent.ID, "running", "paused")
if controlErr != nil {
log.Printf("[worker %s] task %s 意图 #%d 暂停状态落库失败: %v", name, t.ID, intent.ID, controlErr)
return true
}
log.Printf("[worker %s] task %s 意图 #%d 已暂停", name, t.ID, intent.ID)
e.touch(t.ID)
t.NotifyDone(intent.ID) // results changed the graph -> wake the planner (with the just-finished intent id)
return true
}
if action == "cancel" {
// Park the stopped run in paused before handing cleanup to the API. If the
// request disconnects after cancellation, the intent remains recoverable and
// a later cancel can finish cleanup instead of leaving a phantom running row.
controlErr = transitionIntentState(t.Store, intent.ID, "running", "paused")
if controlErr != nil {
log.Printf("[worker %s] task %s 意图 #%d 取消栅栏落库失败: %v", name, t.ID, intent.ID, controlErr)
return true
}
log.Printf("[worker %s] task %s 意图 #%d 已停止,等待取消清理", name, t.ID, intent.ID)
e.touch(t.ID)
return true
}
// if a pause cancelled this run mid-flight, return the intent to the frontier
// so it is re-claimed on resume — the worker will resume the prior LLM
// conversation from its transcript instead of restarting from scratch.
if ectx.Err() != nil && taskExecutionPaused(context.Cause(ectx)) {
if err := transitionIntentState(t.Store, intent.ID, "running", "open"); err != nil {
log.Printf("[worker %s] task %s 意图 #%d 任务暂停回退失败: %v", name, t.ID, intent.ID, err)
}
return true
}
// 任务超时收尾的硬兜底 cancel(非 pause、非 kill)取消了本 run → 归为 exhausted(已收尾),
// 不要误标 blocked。此时 worker 通常已在 settlement 阶段把结果写回。
if ectx.Err() != nil && e.isSettling(t.ID) {
if err := transitionIntentState(t.Store, intent.ID, "running", "exhausted"); err != nil {
log.Printf("[worker %s] task %s 意图 #%d 超时收尾状态落库失败: %v", name, t.ID, intent.ID, err)
}
log.Printf("[worker %s] task %s 意图 #%d 因任务超时收尾结束(exhausted),写回 %s", name, t.ID, intent.ID, wrote)
e.touch(t.ID)
return true
}
// 任务已判完成(done via 常规路径)→ 上面 cancelExec 取消了本 run。意图结果已无意义,
// 标 stopped(不是 blocked),别污染已完成任务的意图状态。
if ectx.Err() != nil && isTerminalStatus(e.m.TaskStatus(t.ID)) {
if err := transitionIntentState(t.Store, intent.ID, "running", "stopped"); err != nil {
log.Printf("[worker %s] task %s 意图 #%d 终态停止落库失败: %v", name, t.ID, intent.ID, err)
}
log.Printf("[worker %s] task %s 意图 #%d 因任务已完成而取消(stopped)", name, t.ID, intent.ID)
e.touch(t.ID)
return true
}
// killed by the planner: mark stopped (don't write back results, don't auto-reclaim).
if killed {
if err := transitionIntentState(t.Store, intent.ID, "running", "stopped"); err != nil {
log.Printf("[worker %s] task %s 意图 #%d planner 停止落库失败: %v", name, t.ID, intent.ID, err)
}
log.Printf("[worker %s] task %s 意图 #%d 被终止(stopped)", name, t.ID, intent.ID)
e.touch(t.ID)
t.Notify()
return true
}
if err != nil {
log.Printf("[worker %s] intent %d: %v", name, intent.ID, err)
}
// terminal分流:撞步数上限 ≠ 完成。max_turns→exhausted(规划者据此知道这个方向
// 试过但没真正做完、需换角度,而非当成已覆盖永久跳过);出错→blocked;正常→done。
state := "done"
switch {
case err != nil:
state = "blocked"
case reason == harness.ReasonMaxTurns:
state = "exhausted"
log.Printf("[worker %s] intent %d 撞步数上限(exhausted),本次写回 %s", name, intent.ID, wrote)
case reason == harness.ReasonTimeout:
state = "exhausted"
log.Printf("[worker %s] intent %d 运行超时(exhausted),收尾后写回 %s", name, intent.ID, wrote)
}
if state == "blocked" && isTaskLLMChainExhausted(err) {
_ = t.Store.SetIntentBlockedReason(intent.ID, db.IntentBlockedLLMQuota)
} else {
if stateErr := transitionIntentState(t.Store, intent.ID, "running", state); stateErr != nil {
log.Printf("[worker %s] task %s 意图 #%d 终态 %s 落库失败: %v", name, t.ID, intent.ID, state, stateErr)
}
}
log.Printf("[worker %s] task %s 意图 #%d 结束: %s (写回 %s)", name, t.ID, intent.ID, state, wrote)
e.touch(t.ID)
t.NotifyDone(intent.ID) // results changed the graph -> wake the planner (with the just-finished intent id)
return true
}
func taskExecutionPaused(cause error) bool {
var abort *agent.AbortCause
if !errors.As(cause, &abort) {
return false
}
switch abort.Code {
case "paused_by_user", "paused_by_orchestrator", "paused_on_reload", "paused_race_guard",
"queued_for_admission", "llm_unavailable_queued", "task_deleted":
return true
default:
return false
}
}
+217
View File
@@ -0,0 +1,217 @@
package server
import (
"context"
"errors"
"testing"
"time"
"github.com/Autumn-27/artex/agent"
)
func TestPauseCancelsWithNamedCause(t *testing.T) {
e := NewEngine(nil)
execCtx := e.execContextFor(context.Background(), "t1")
workCtx, workCancel := context.WithCancelCause(execCtx)
defer workCancel(nil)
e.Pause("t1", agent.AbortPausedByUser)
if code, _, _, ok := agent.AbortReason(workCtx); !ok || code != "paused_by_user" {
t.Fatalf("code=%q ok=%v, want paused_by_user", code, ok)
}
}
func TestKillWorkCancelsOnlyThatWorkWithNamedCause(t *testing.T) {
e := NewEngine(nil)
execCtx := e.execContextFor(context.Background(), "t1")
workCtx, workCancel := context.WithCancelCause(execCtx)
e.registerWork(42, workCancel)
if err := e.KillWork(42); err != nil {
t.Fatal(err)
}
if code, _, _, ok := agent.AbortReason(workCtx); !ok || code != "killed_by_planner" {
t.Fatalf("code=%q ok=%v, want killed_by_planner", code, ok)
}
if execCtx.Err() != nil {
t.Fatal("kill_work cancelled the entire task context")
}
_, complete := e.detachWork(42)
complete(nil)
}
func TestControlWorkCarriesUserActionCause(t *testing.T) {
for _, tc := range []struct {
action string
code string
}{
{action: "pause", code: "work_paused_by_user"},
{action: "cancel", code: "work_cancelled_by_user"},
} {
t.Run(tc.action, func(t *testing.T) {
e := NewEngine(nil)
ctx, cancel := context.WithCancelCause(context.Background())
e.registerWork(42, cancel)
done := make(chan error, 1)
go func() { done <- e.ControlWork(context.Background(), 42, tc.action) }()
<-ctx.Done()
if code, _, _, _ := agent.AbortReason(ctx); code != tc.code {
t.Fatalf("code=%q, want %q", code, tc.code)
}
action, complete := e.detachWork(42)
if action != tc.action {
t.Fatalf("action=%q, want %q", action, tc.action)
}
complete(nil)
if err := <-done; err != nil {
t.Fatal(err)
}
})
}
}
func TestControlWorkReturnsSettlementError(t *testing.T) {
e := NewEngine(nil)
ctx, cancel := context.WithCancelCause(context.Background())
e.registerWork(42, cancel)
done := make(chan error, 1)
go func() { done <- e.ControlWork(context.Background(), 42, "pause") }()
<-ctx.Done()
_, complete := e.detachWork(42)
want := errors.New("pause persistence failed")
complete(want)
if err := <-done; !errors.Is(err, want) {
t.Fatalf("ControlWork error=%v, want %v", err, want)
}
}
func TestControlWorkWaitHonorsContextAndReleasesReservation(t *testing.T) {
e := NewEngine(nil)
_, cancel := context.WithCancelCause(context.Background())
e.registerWork(42, cancel)
waitCtx, stop := context.WithTimeout(context.Background(), 20*time.Millisecond)
defer stop()
if err := e.ControlWork(waitCtx, 42, "pause"); !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("ControlWork error=%v, want deadline exceeded", err)
}
e.workMu.Lock()
run := e.work[42]
if run == nil || run.action != "" {
t.Fatalf("timed-out control left reservation: %+v", run)
}
e.workMu.Unlock()
_, complete := e.detachWork(42)
complete(nil)
}
func TestControlWorkRejectsConcurrentController(t *testing.T) {
e := NewEngine(nil)
ctx, cancel := context.WithCancelCause(context.Background())
e.registerWork(42, cancel)
first := make(chan error, 1)
go func() { first <- e.ControlWork(context.Background(), 42, "pause") }()
<-ctx.Done()
if err := e.ControlWork(context.Background(), 42, "cancel"); !errors.Is(err, errWorkControlConflict) {
t.Fatalf("second control error=%v, want conflict", err)
}
action, complete := e.detachWork(42)
if action != "pause" {
t.Fatalf("winning action=%q, want pause", action)
}
complete(nil)
if err := <-first; err != nil {
t.Fatal(err)
}
}
func TestSettleDrainAndGoalMetCausesAreDistinct(t *testing.T) {
for _, tc := range []struct {
cause *agent.AbortCause
code string
}{
{agent.AbortSettleDrainTimeout, "settle_drain_timeout"},
{agent.AbortGoalMet, "goal_met"},
} {
e := NewEngine(nil)
ctx := e.execContextFor(context.Background(), "t1")
e.cancelExec("t1", tc.cause)
if code, _, _, _ := agent.AbortReason(ctx); code != tc.code {
t.Fatalf("code=%q, want %q", code, tc.code)
}
}
}
func TestExecContextWhilePausedCarriesRaceGuardCause(t *testing.T) {
e := NewEngine(nil)
e.Pause("t1", agent.AbortPausedByUser)
ctx := e.execContextFor(context.Background(), "t1")
if ctx.Err() == nil {
t.Fatal("paused task received a live execution context")
}
if code, _, _, _ := agent.AbortReason(ctx); code != "paused_race_guard" {
t.Fatalf("code=%q, want paused_race_guard", code)
}
}
func TestResumeYieldsFreshContext(t *testing.T) {
e := NewEngine(nil)
old := e.execContextFor(context.Background(), "t1")
e.Pause("t1", agent.AbortPausedByUser)
if old.Err() == nil {
t.Fatal("Pause did not cancel old context")
}
e.paused.Delete("t1")
if fresh := e.execContextFor(context.Background(), "t1"); fresh.Err() != nil {
t.Fatal("resume did not create a fresh execution context")
}
}
func TestTaskPauseCauseSurvivesImmediateResume(t *testing.T) {
e := NewEngine(nil)
task := &Task{ID: "t1", notify: make(chan struct{}, 1)}
old := e.execContextFor(context.Background(), task.ID)
e.Pause(task.ID, agent.AbortPausedByUser)
e.Resume(task)
if e.IsPaused(task.ID) {
t.Fatal("resume did not clear the task pause flag")
}
if old.Err() == nil || !taskExecutionPaused(context.Cause(old)) {
t.Fatalf("old execution cause=%v was not retained as a task pause", context.Cause(old))
}
if taskExecutionPaused(agent.AbortGoalMet) {
t.Fatal("goal completion was misclassified as a task pause")
}
}
func TestDeleteCancellationReturnsRunningIntentToFrontier(t *testing.T) {
if !taskExecutionPaused(agent.AbortTaskDeleted) {
t.Fatal("delete cancellation must settle a running intent as open so AbortDelete can resume it")
}
if taskExecutionPaused(agent.AbortGoalMet) {
t.Fatal("terminal task cancellation must not reopen completed work")
}
}
func TestResumeDoesNotClearDeleteBarrier(t *testing.T) {
e := NewEngine(nil)
task := &Task{ID: "t1", notify: make(chan struct{}, 1)}
e.paused.Store(task.ID, true)
e.deleting.Store(task.ID, true)
e.Resume(task)
if !e.IsPaused(task.ID) {
t.Fatal("resume cleared the pause barrier owned by task deletion")
}
}
func TestBootstrapAdmissionClearsQueuedPauseBarrier(t *testing.T) {
e := NewEngine(nil)
ctx, cancel := context.WithCancel(context.Background())
cancel()
task := &Task{ID: "t1", notify: make(chan struct{}, 1)}
s := &Server{engine: e, ctx: ctx}
e.Pause(task.ID, agent.AbortPausedByUser)
s.startAdmittedTask(task, "bootstrap")
if e.IsPaused(task.ID) {
t.Fatal("bootstrap admission left the queued task behind the pause barrier")
}
}
+59 -11
View File
@@ -43,7 +43,20 @@ func (e *Engine) inflightCounter(taskID string) *int64 {
v, _ := e.inflight.LoadOrStore(taskID, new(int64))
return v.(*int64)
}
func (e *Engine) incInflight(taskID string) { atomic.AddInt64(e.inflightCounter(taskID), 1) }
// beginTaskOperation atomically registers a task-owned operation unless deletion
// has already installed its barrier. The delete handler can therefore wait for
// inflight==0 without a check-then-start race recreating files after cleanup.
func (e *Engine) beginTaskOperation(taskID string) bool {
e.deleteMu.RLock()
defer e.deleteMu.RUnlock()
if e.IsDeleting(taskID) {
return false
}
atomic.AddInt64(e.inflightCounter(taskID), 1)
return true
}
func (e *Engine) decInflight(taskID string) { atomic.AddInt64(e.inflightCounter(taskID), -1) }
func (e *Engine) inflightCount(taskID string) int64 {
return atomic.LoadInt64(e.inflightCounter(taskID))
@@ -58,13 +71,26 @@ func (e *Engine) taskDeadline(t *Task) int64 {
if v, ok := e.deadline.Load(t.ID); ok {
return v.(int64)
}
if t.DeadlineAt > 0 {
e.deadline.Store(t.ID, t.DeadlineAt)
return t.DeadlineAt
deadlineAt := t.lifecycleSnapshot().DeadlineAt
if deadlineAt > 0 {
e.deadline.Store(t.ID, deadlineAt)
return deadlineAt
}
return 0
}
// resetTimeoutRevival clears only the per-run timeout state after PostgreSQL has
// atomically committed timeout -> running and reset first_run_at/deadline_at. The
// configured TimeoutSeconds remains on Task, so the next real Planner/Worker run
// stamps a fresh full budget. coordStarted is reset because the coordinator that
// produced the timeout has already completed (or is in its final return path).
func (e *Engine) resetTimeoutRevival(taskID string) {
e.settling.Delete(taskID)
e.deadline.Delete(taskID)
e.stamped.Delete(taskID)
e.coordStarted.Delete(taskID)
}
// stampFirstRun records first_run_at + deadline_at on the FIRST real run (LLM ready)
// of a timeout task, once per process. No-op when the task has no timeout.
func (e *Engine) stampFirstRun(t *Task) {
@@ -106,10 +132,18 @@ func (e *Engine) startDeadlineCoordinator(ctx context.Context, t *Task) {
if t == nil || t.TimeoutSeconds <= 0 {
return
}
if _, loaded := e.coordStarted.LoadOrStore(t.ID, true); loaded {
e.deleteMu.RLock()
if e.IsDeleting(t.ID) {
e.deleteMu.RUnlock()
return
}
go e.deadlineCoordinator(ctx, t)
if _, loaded := e.coordStarted.LoadOrStore(t.ID, true); loaded {
e.deleteMu.RUnlock()
return
}
rt := e.registerTaskRoutines(ctx, t.ID, 1)
e.deleteMu.RUnlock()
runTaskRoutine(rt, func(loopCtx context.Context) { e.deadlineCoordinator(loopCtx, t) })
}
// deadlineCoordinator waits until the task's absolute deadline, then runs the settle
@@ -159,7 +193,7 @@ func (e *Engine) settleTask(ctx context.Context, t *Task) {
for e.inflightCount(t.ID) > 0 {
if time.Now().After(hardStop) {
log.Printf("[deadline] task %s drain 超时(%s),硬取消在跑 run", t.ID, settleDrainGrace)
e.cancelExec(t.ID)
e.cancelExec(t.ID, agent.AbortSettleDrainTimeout)
_ = sleepCtx(ctx, 3*time.Second) // 给 worker 分支一点时间落库/归类
break
}
@@ -170,6 +204,10 @@ func (e *Engine) settleTask(ctx context.Context, t *Task) {
// ⑤ 终局一轮 planner(任务超时词,最后目标判定,不产新意图)。
met := e.runFinalPlannerRound(ctx, t)
if !e.beginTaskOperation(t.ID) {
return
}
defer e.decInflight(t.ID)
// ⑥ 定终态(带守卫):met → done(completed);否则 timeout。若常规路径已先落 done,
// 守卫(SetTaskStatusGuarded)会拒绝覆盖,保留 completed 语义。
@@ -192,6 +230,9 @@ func (e *Engine) settleTask(ctx context.Context, t *Task) {
// task-timeout planner words (final goal judgment; no new intents). Waits for the
// LLM to be ready (bounded by ctx) so a completable task isn't mis-judged timeout.
func (e *Engine) runFinalPlannerRound(ctx context.Context, t *Task) (met bool) {
if e.IsDeleting(t.ID) {
return false
}
planner, _ := e.snapshotFor(t)
for planner == nil {
if sleepCtx(ctx, deadlinePollInterval) {
@@ -200,17 +241,24 @@ func (e *Engine) runFinalPlannerRound(ctx context.Context, t *Task) (met bool) {
if isTerminalStatus(e.m.TaskStatus(t.ID)) {
return false
}
if e.IsDeleting(t.ID) {
return false
}
planner, _ = e.snapshotFor(t)
}
// 独立 ctx(不挂 execCancel,避免 pause/硬 cancel 打断这最后一轮),带 Final 注入任务超时词。
fctx := e.clockCtx(ctx, t, true)
if !e.beginTaskOperation(t.ID) {
return false
}
defer e.decInflight(t.ID)
emit := func(r db.Activity) { e.emitActivity(t, r) }
e.emitActivity(t, db.Activity{Worker: "planner", Kind: "round",
Summary: fmt.Sprintf("任务超时收尾·终局判定(第 %d 轮)", e.nextPlannerRound(t.ID))})
// 独立 ctx(不挂 execCancel,避免 pause/硬 cancel 打断这最后一轮),带 Final 注入任务超时词。
fctx := e.clockCtx(ctx, t, true)
e.incInflight(t.ID)
tTaskID, _ := strconv.ParseInt(t.ID, 10, 64)
e.BeginLLMCall(t.ID)
met, reason, err := planner.Plan(fctx, tTaskID, e.m.assets, t.Store, t.Goal, t.drainTriggers(), emit)
e.decInflight(t.ID)
e.EndLLMCall(t.ID)
if err != nil {
log.Printf("[deadline] task %s 终局规划出错: %v", t.ID, err)
} else if met {
+330 -59
View File
@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"log"
"sort"
"strconv"
"strings"
@@ -25,25 +26,324 @@ type goalSpec struct {
//
// 异步(goroutine)所以调用方立即返回,两条路径行为一致:秒建任务、后台拆目标。
func (s *Server) launchTask(t *Task, seedText string, seedFirstIntent bool) {
if !s.engine.beginTaskOperation(t.ID) {
return
}
s.seed(t, seedText)
if seedFirstIntent {
s.seedFirstIntent(t)
}
go func() {
s.engine.emitActivity(t, db.Activity{Worker: "planner", Kind: "round",
Summary: "第 0 轮目标拆解"})
goals := s.createGoals(s.ctx, t, func(r db.Activity) {
s.engine.emitActivity(t, r)
})
for _, g := range goals {
summary := g.Text
if g.VulnClass != "" {
summary = fmt.Sprintf("[%s] %s", g.VulnClass, g.Text)
}
s.engine.emitActivity(t, db.Activity{Worker: "planner", Kind: "text", Summary: summary})
s.engine.decInflight(t.ID)
if _, err := s.admitTask(t, "bootstrap"); err != nil {
log.Printf("[concurrency] task %s 启动失败: %v", t.ID, err)
}
}
func (s *Server) startTaskEngine(t *Task) {
ctx := s.engine.execContextFor(s.ctx, t.ID)
if ctx.Err() != nil || s.engine.IsDeleting(t.ID) {
return
}
s.engine.emitActivity(t, db.Activity{Worker: "planner", Kind: "round",
Summary: "第 0 轮目标拆解"})
goals := s.createGoals(ctx, t, func(r db.Activity) {
s.engine.emitActivity(t, r)
})
if ctx.Err() != nil || s.engine.IsDeleting(t.ID) {
return
}
for _, g := range goals {
summary := g.Text
if g.VulnClass != "" {
summary = fmt.Sprintf("[%s] %s", g.VulnClass, g.Text)
}
s.engine.Run(s.ctx, t)
}()
s.engine.emitActivity(t, db.Activity{Worker: "planner", Kind: "text", Summary: summary})
}
if ctx.Err() != nil || s.engine.IsDeleting(t.ID) {
return
}
s.engine.Run(s.ctx, t)
}
func (s *Server) occupiesConcurrencySlot(t *Task) bool {
if t == nil {
return false
}
lifecycle := t.lifecycleSnapshot()
if lifecycle.Queued || lifecycle.Paused || isTerminalStatus(lifecycle.Status) {
return false
}
// Deletion temporarily pauses the Engine but has not committed yet. Preserve
// the task's slot until PostgreSQL deletion succeeds; otherwise FIFO promotion
// during the drain window could over-admit if deletion later aborts and the
// persisted running task is restored.
if s.engine.IsDeleting(t.ID) {
return true
}
return !s.engine.IsPaused(t.ID) && (s.engine.ReadyFor(t) || s.engine.ActiveLLMCalls(t.ID) > 0)
}
func (s *Server) runningTaskCount(excludeID string) int {
count := 0
for _, task := range s.m.List() {
if task.ID != excludeID && s.occupiesConcurrencySlot(task) {
count++
}
}
return count
}
// admitTask is the single admission path for new, resumed, rerun and follow-up
// work. It atomically either starts the task or appends it to the persistent FIFO
// queue. mode is bootstrap for a freshly-created task and resume otherwise.
func (s *Server) admitTask(t *Task, mode string) (queued bool, err error) {
return s.admitTaskWhen(t, mode, false)
}
// admitPausedTask is the task-control resume path. The paused precondition is
// checked under the same scheduler lock as admission so a concurrent pause,
// dequeue or FIFO promotion cannot leave database and Engine state divergent.
func (s *Server) admitPausedTask(t *Task) (queued bool, err error) {
return s.admitTaskWhen(t, "resume", true)
}
func (s *Server) admitTaskWhen(t *Task, mode string, requirePaused bool) (queued bool, err error) {
if t == nil {
return false, fmt.Errorf("task not found")
}
if mode != "bootstrap" {
mode = "resume"
}
s.concMu.Lock()
defer s.concMu.Unlock()
// Delete installs its barrier under concMu as well. Re-resolve after acquiring
// the lock so a request that captured a task pointer before successful deletion
// cannot revive that stale handle after StopTask clears its Engine maps.
current, exists := s.m.Task(t.ID)
if !exists || current != t || s.engine.IsDeleting(t.ID) {
return false, fmt.Errorf("task is being deleted")
}
if !s.engine.beginTaskOperation(t.ID) {
return false, fmt.Errorf("task is being deleted")
}
defer s.engine.decInflight(t.ID)
lifecycle := t.lifecycleSnapshot()
if requirePaused {
if isTerminalStatus(lifecycle.Status) {
return false, fmt.Errorf("终态任务不能执行继续")
}
if !lifecycle.Paused {
return false, fmt.Errorf("仅已暂停的任务可以继续")
}
mode = s.resumeAdmissionMode(t)
}
wasTerminal := isTerminalStatus(lifecycle.Status)
wasPaused := lifecycle.Paused || s.engine.IsPaused(t.ID)
wasQueued := lifecycle.Queued
engineWasPaused := s.engine.IsPaused(t.ID)
enabled, limit := s.m.ConcurrencyLimit()
ready := s.engine.ReadyFor(t)
// Work added to a task that is already admitted only needs a wake-up. This
// matters when the configured limit was lowered below the current running
// count: an existing task must not suddenly mark itself queued while its
// planner/workers are still live. Still compare-and-commit the persisted
// status: a concurrent terminal transition must win instead of being silently
// reported as a successful follow-up admission.
if !wasTerminal && !wasPaused && !wasQueued && s.engine.Started(t.ID) && (!enabled || ready) {
if err := s.m.ApplyTaskAdmission(t.ID, lifecycle.Status, lifecycle.Status, false, "resume", false); err != nil {
return false, err
}
s.startAdmittedTask(t, "resume")
return false, nil
}
// Preserve the original first-run mode across repeated admissions. Legacy
// queued rows have an empty queue_mode, so infer bootstrap from their graph.
if wasQueued {
switch lifecycle.QueueMode {
case "bootstrap":
mode = "bootstrap"
case "":
mode = s.resumeAdmissionMode(t)
}
}
readyBacklog := enabled && s.hasReadyQueuedTask(t.ID)
atCapacity := enabled && s.runningTaskCount(t.ID) >= limit
shouldQueue := enabled && (!ready || readyBacklog || atCapacity)
// Install the execution barrier before reviving a terminal/paused task. Without
// this ordering, its already-running worker loops can claim the newly-opened
// intent in the gap between status=running and queued=true.
if shouldQueue || wasTerminal || wasPaused || wasQueued {
s.engine.Pause(t.ID, agent.Causef("queued_for_admission", "任务等待运行准入",
"任务正在等待并发队列或准入状态提交,本次执行已停止;只有获得运行槽后才会重新领取意图"))
}
status := lifecycle.Status
if wasTerminal {
status = "running"
}
if err := s.m.ApplyTaskAdmission(t.ID, lifecycle.Status, status, shouldQueue, mode, wasQueued); err != nil {
if !engineWasPaused && !wasPaused && !wasQueued {
s.engine.Resume(t)
}
return false, err
}
if lifecycle.Status == "timeout" && status == "running" {
// ApplyTaskAdmission reset this timed-out run's persisted clock. Clear the
// matching Engine gates before either parking it in FIFO or starting work;
// otherwise the old settling flag would make every worker skip forever.
s.engine.resetTimeoutRevival(t.ID)
}
if shouldQueue {
if !wasQueued {
summary := fmt.Sprintf("已排队:达到并发上限 %d,等待空位后自动开始", limit)
switch {
case !ready:
summary = "已排队:当前没有可运行的 LLM 配置,配置恢复后自动开始"
case readyBacklog:
summary = "已排队:已有更早的任务等待运行,将按 FIFO 顺序自动开始"
}
s.engine.emitActivity(t, db.Activity{Worker: "system", Kind: "text", Summary: summary})
}
return true, nil
}
s.startAdmittedTask(t, mode)
return false, nil
}
func (s *Server) hasReadyQueuedTask(excludeID string) bool {
for _, task := range s.m.List() {
lifecycle := task.lifecycleSnapshot()
if task.ID != excludeID && lifecycle.Queued && !lifecycle.Paused && !isTerminalStatus(lifecycle.Status) && s.engine.ReadyFor(task) {
return true
}
}
return false
}
func (s *Server) startAdmittedTask(t *Task, mode string) {
if mode == "bootstrap" {
if !s.engine.beginTaskOperation(t.ID) {
return
}
// A first-run task may have kept the Engine pause barrier while waiting
// in the concurrency queue. Clear it only after operation admission, or
// startTaskEngine's first execContextFor call would return a cancelled
// context and silently strand the dequeued task.
if s.engine.IsPaused(t.ID) {
s.engine.Resume(t)
}
go func() {
defer s.engine.decInflight(t.ID)
s.startTaskEngine(t)
}()
return
}
s.engine.Run(s.ctx, t)
s.engine.Resume(t)
// Run returns early for an already-started task. Explicitly ensure a timeout
// coordinator exists after resume; resetTimeoutRevival cleared the completed
// coordinator's marker and the next real call will stamp a fresh deadline.
s.engine.startDeadlineCoordinator(s.ctx, t)
}
func (s *Server) reconcileConcurrency() {
s.concMu.Lock()
defer s.concMu.Unlock()
enabled, limit := s.m.ConcurrencyLimit()
// A task whose provider chain becomes unavailable cannot do useful work and
// must not reserve a limited running slot forever. Park it persistently so a
// later profile edit/recovery can re-enter through the same FIFO path.
if enabled {
for _, task := range s.m.List() {
lifecycle := task.lifecycleSnapshot()
if lifecycle.Queued || lifecycle.Paused || s.engine.IsPaused(task.ID) || s.engine.IsDeleting(task.ID) ||
isTerminalStatus(lifecycle.Status) || s.engine.ReadyFor(task) || s.engine.ActiveLLMCalls(task.ID) > 0 {
continue
}
mode := s.resumeAdmissionMode(task)
s.engine.Pause(task.ID, agent.Causef("llm_unavailable_queued", "LLM 不可用,任务进入等待队列",
"任务当前无法解析可运行的 Planner/Worker LLM,已释放并发槽;配置恢复后按队列顺序继续"))
if err := s.m.EnqueueTask(task.ID, mode); err != nil {
s.engine.Resume(task)
log.Printf("[concurrency] task %s 因 LLM 不可用入队失败: %v", task.ID, err)
continue
}
s.engine.emitActivity(task, db.Activity{Worker: "system", Kind: "text",
Summary: "已排队:当前没有可运行的 LLM 配置,配置恢复后自动开始"})
}
}
type queuedTask struct {
task *Task
lifecycle taskLifecycleState
}
queued := []queuedTask{}
for _, task := range s.m.List() {
lifecycle := task.lifecycleSnapshot()
if lifecycle.Queued && !isTerminalStatus(lifecycle.Status) {
queued = append(queued, queuedTask{task: task, lifecycle: lifecycle})
}
}
sort.SliceStable(queued, func(i, j int) bool {
left, right := queued[i].lifecycle.QueuedAt, queued[j].lifecycle.QueuedAt
if left == 0 {
left = queued[i].task.CreatedAt * int64(1e9)
}
if right == 0 {
right = queued[j].task.CreatedAt * int64(1e9)
}
if left != right {
return left < right
}
// Old rows may not have queued_at and task creation timestamps are only
// kept to second precision in memory. Task ids are monotonic, providing a
// deterministic oldest-first fallback for those ties.
leftID, leftErr := strconv.ParseInt(queued[i].task.ID, 10, 64)
rightID, rightErr := strconv.ParseInt(queued[j].task.ID, 10, 64)
if leftErr == nil && rightErr == nil {
return leftID < rightID
}
return queued[i].task.ID < queued[j].task.ID
})
slots := len(queued)
if enabled {
slots = limit - s.runningTaskCount("")
}
for _, entry := range queued {
if slots <= 0 {
break
}
task := entry.task
lifecycle := task.lifecycleSnapshot()
// Keep FIFO order, but do not consume a concurrency slot for a task
// whose explicit chain/global provider is unavailable. It will be retried
// after the user configures or resets its LLM chain.
if lifecycle.Paused || (enabled && !s.engine.ReadyFor(task)) {
continue
}
mode := lifecycle.QueueMode
if mode != "bootstrap" && mode != "resume" {
mode = s.resumeAdmissionMode(task)
}
if mode != "bootstrap" {
mode = "resume"
}
if !s.engine.beginTaskOperation(task.ID) {
continue
}
if err := s.m.ApplyTaskAdmission(task.ID, lifecycle.Status, lifecycle.Status, false, mode, false); err != nil {
s.engine.decInflight(task.ID)
continue
}
s.startAdmittedTask(task, mode)
s.engine.decInflight(task.ID)
slots--
}
}
// reviveTask 让一个已停下的任务重新跑起来:把终态(done/failed/timeout)拉回 running、
@@ -56,22 +356,8 @@ func (s *Server) reviveTask(t *Task) {
if t == nil {
return
}
// 终态 → 拉回 running(SetTaskStatus 会清 completed_at 并同步内存 handle 的 Status,
// 使 planner/worker 循环的终态门放行)。
if isTerminalStatus(t.Status) {
if err := s.m.SetTaskStatus(t.ID, "running"); err != nil {
log.Printf("[revive] task %s 置 running 失败: %v", t.ID, err)
}
}
// 确保引擎循环存活:已 started 只会 Notify;未 started(重启后未恢复的终态任务)则重起循环。
s.engine.Run(s.ctx, t)
// 暂停中 → 解除暂停(清引擎内存态 + Notify),并持久化 paused=false(存活过重启)。
if t.Paused || s.engine.IsPaused(t.ID) {
s.engine.Resume(t)
t.Paused = false
if err := s.m.SetTaskPaused(t.ID, false); err != nil {
log.Printf("[revive] task %s 解除暂停失败: %v", t.ID, err)
}
if _, err := s.admitTask(t, "resume"); err != nil {
log.Printf("[revive] task %s 恢复失败: %v", t.ID, err)
}
}
@@ -83,24 +369,25 @@ func (s *Server) reviveTask(t *Task) {
// Returns the seeded specs so callers can emit activity records for them.
// emit, when non-nil, is forwarded to DecomposeGoals so LLM steps are visible in the UI.
func (s *Server) createGoals(ctx context.Context, t *Task, emit func(db.Activity)) []goalSpec {
if t == nil {
return nil
}
// Use the SAME LLM the task runs on (its pinned profile, else the active profile),
// NOT agent.FromEnv() — the LLM is configured via the UI (DB profile), not env vars,
// so FromEnv returned empty and every task silently fell back to the crude rule
// splitter (which shredded URLs / made meaningless 2-way splits).
var specs []goalSpec
if cfg, ok := s.goalsLLMConfig(t); ok {
var as *db.AssetStore
if s.m != nil {
as = s.m.Assets()
}
taskID, _ := strconv.ParseInt(t.ID, 10, 64)
// DecomposeGoals persists each decomposed goal via the set_goals tool straight
// into t.Store; it returns the specs it wrote so we can emit activity + spot the
// empty (LLM error / no provider) case below. No write-back needed here anymore.
for _, g := range agent.DecomposeGoals(ctx, cfg, s.m.dir, t.Goal, t.Description, as, t.Store, taskID, emit) {
if strings.TrimSpace(g.Text) != "" {
specs = append(specs, goalSpec{Text: g.Text, VulnClass: g.VulnClass})
}
var as *db.AssetStore
if s.m != nil {
as = s.m.Assets()
}
taskID, _ := strconv.ParseInt(t.ID, 10, 64)
s.engine.BeginLLMCall(t.ID)
decomposed := agent.DecomposeGoalsWithProvider(ctx, s.agentsForTask(t).runtime, s.m.dir, t.Goal, t.Description, as, t.Store, taskID, emit)
s.engine.EndLLMCall(t.ID)
for _, g := range decomposed {
if strings.TrimSpace(g.Text) != "" {
specs = append(specs, goalSpec{Text: g.Text, VulnClass: g.VulnClass})
}
}
if len(specs) == 0 {
@@ -119,19 +406,3 @@ func (s *Server) createGoals(ctx context.Context, t *Task, emit func(db.Activity
}
return specs
}
// goalsLLMConfig resolves the LLM config for goal decomposition by the standard
// precedence: the goals agent's own binding → the task's pinned profile → the global
// active profile (same source the engine runs on).
func (s *Server) goalsLLMConfig(t *Task) (agent.Config, bool) {
var pin *int64
if t != nil {
pin = t.LLMProfileID
}
if eff := s.effectiveProfileForAgent("goals", pin); eff != nil {
if cfg, ok := s.loadProfileConfig(*eff); ok {
return cfg, true
}
}
return s.loadLLMConfig()
}
+12
View File
@@ -0,0 +1,12 @@
package server
import (
"context"
"testing"
)
func TestCreateGoalsNilTask(t *testing.T) {
if got := (&Server{}).createGoals(context.Background(), nil, nil); len(got) != 0 {
t.Fatalf("nil task produced goals: %+v", got)
}
}
+100
View File
@@ -0,0 +1,100 @@
package server
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"testing"
"github.com/Autumn-27/artex/db"
)
func TestInheritedActivityDetailAndRelationDeletion(t *testing.T) {
m, err := NewManager(t.TempDir(), "")
if err != nil {
t.Skipf("postgres unavailable (%v) - skipping", err)
}
defer m.Close()
source, err := m.CreateTaskWithOptions("detail source", "source goal", db.TaskCreateOptions{})
if err != nil {
t.Fatal(err)
}
sourceID := mustTaskID(t, source.ID)
current, err := m.CreateTaskWithOptions("detail current", "current goal", db.TaskCreateOptions{SourceTaskIDs: []int64{sourceID}})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
_, _ = m.DeleteTask(current.ID, DeleteTaskOptions{})
_, _ = m.DeleteTask(source.ID, DeleteTaskOptions{})
})
plannerStep, err := source.Store.AppendActivity(db.Activity{
Worker: "planner", Kind: "text", Summary: "private planner", Detail: "private planner detail",
})
if err != nil {
t.Fatal(err)
}
intentID, err := source.Store.AddIntent(map[string]any{"summary": "completed source work"}, 1, nil, "planner")
if err != nil {
t.Fatal(err)
}
if err := source.Store.SetIntentState(intentID, "done"); err != nil {
t.Fatal(err)
}
workerStep, err := source.Store.AppendActivity(db.Activity{
NodeID: &intentID, Worker: "worker", Kind: "result", Summary: "shared result", Detail: "shared worker detail",
})
if err != nil {
t.Fatal(err)
}
s := New(context.Background(), m, t.TempDir(), t.TempDir(), t.TempDir())
token, err := signJWT(s.jwtKey)
if err != nil {
t.Fatal(err)
}
fetchDetail := func(stepID int64) string {
req := httptest.NewRequest(http.MethodGet,
fmt.Sprintf("/api/exploration/activity/%d?task=%s", stepID, current.ID), nil)
req.Header.Set("Authorization", "Bearer "+token)
rec := httptest.NewRecorder()
s.Handler().ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("activity detail %d: status=%d body=%s", stepID, rec.Code, rec.Body.String())
}
var body struct {
Detail string `json:"detail"`
}
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
t.Fatal(err)
}
return body.Detail
}
if got := fetchDetail(plannerStep); got != "" {
t.Fatalf("source planner transcript leaked through API: %q", got)
}
if got := fetchDetail(workerStep); got != "shared worker detail" {
t.Fatalf("terminal inherited worker detail missing: %q", got)
}
if _, err := m.DeleteTask(source.ID, DeleteTaskOptions{}); err != nil {
t.Fatal(err)
}
currentAfter, ok := m.Task(current.ID)
if !ok || len(currentAfter.SourceTaskIDs) != 0 {
t.Fatalf("deleted source remained in live task DTO state: %+v", currentAfter)
}
}
func mustTaskID(t *testing.T, raw string) int64 {
t.Helper()
var id int64
if _, err := fmt.Sscan(raw, &id); err != nil || id <= 0 {
t.Fatalf("bad task id %q: %v", raw, err)
}
return id
}
+117
View File
@@ -0,0 +1,117 @@
package server
import (
"encoding/json"
"testing"
"time"
"github.com/Autumn-27/artex/db"
)
func TestInheritedDTOProvenance(t *testing.T) {
createdAt := time.Date(2026, time.August, 19, 12, 0, 0, 0, time.UTC)
node := &db.Node{
ID: 41,
Kind: db.KindFinding,
Payload: json.RawMessage(`{"vulnclass":"xss","severity":"high","summary":"source finding"}`),
State: "confirmed",
CreatedAt: createdAt,
SourceTaskID: 7,
Inherited: true,
}
nodeDTO := taskNodeDTO(node)
if !nodeDTO.Inherited || nodeDTO.SourceTaskID != "7" {
t.Fatalf("node provenance lost: %+v", nodeDTO)
}
finding := findingDTOsForOwner("7", "source", []*db.Node{node}, nil, nil)
if len(finding) != 1 || !finding[0].Inherited || finding[0].SourceTaskID != "7" || finding[0].TaskID != "7" {
t.Fatalf("finding provenance lost: %+v", finding)
}
intentID := int64(44)
activity := activityDTO(db.Activity{
ID: 45,
NodeID: &intentID,
CreatedAt: createdAt,
SourceTaskID: 7,
Inherited: true,
})
if !activity.Inherited || activity.SourceTaskID != "7" || activity.IntentID != "44" {
t.Fatalf("activity provenance lost: %+v", activity)
}
assetRef := coverageAssetRefDTO(db.AssetRef{
ID: 46, Kind: db.KindFact, State: "confirmed", Summary: "source fact", SourceTaskID: 7, Inherited: true,
})
if !assetRef.Inherited || assetRef.SourceTaskID != "7" {
t.Fatalf("asset ref provenance lost: %+v", assetRef)
}
}
func TestInheritedIntentResultOnlyAllowsHistory(t *testing.T) {
for _, state := range []string{"done", "blocked", "exhausted", "stopped"} {
if !inheritedIntentResult(state) {
t.Errorf("terminal state %q should be inherited history", state)
}
}
for _, state := range []string{"open", "running", ""} {
if inheritedIntentResult(state) {
t.Errorf("live state %q must not enter inherited history", state)
}
}
}
func TestInheritedGraphSnapshotFiltersLiveIntentsAndEdges(t *testing.T) {
nodes := []*db.Node{
{ID: 1, Kind: db.KindFact, State: "confirmed"},
{ID: 2, Kind: db.KindIntent, State: "running"},
{ID: 3, Kind: db.KindIntent, State: "done"},
{ID: 4, Kind: db.KindFinding, State: "confirmed"},
}
edges := []db.Edge{
{From: 1, Rel: db.RelDerivedFrom, To: 2},
{From: 1, Rel: db.RelDerivedFrom, To: 3},
{From: 3, Rel: db.RelYields, To: 4},
}
gotNodes, gotEdges := inheritedGraphSnapshot(nodes, edges, 77)
if len(gotNodes) != 3 {
t.Fatalf("expected fact, terminal intent and finding; got %+v", gotNodes)
}
for _, node := range gotNodes {
if node.ID == 2 {
t.Fatal("running inherited intent leaked into graph snapshot")
}
if !node.Inherited || node.SourceTaskID != 77 {
t.Fatalf("source provenance missing: %+v", node)
}
}
if len(gotEdges) != 2 {
t.Fatalf("edge referencing hidden live intent was not removed: %+v", gotEdges)
}
for _, edge := range gotEdges {
if edge.From == 2 || edge.To == 2 {
t.Fatalf("live intent id leaked through edge: %+v", edge)
}
}
}
func TestFindingProvenanceInTask(t *testing.T) {
task := &Task{ID: "10", SourceTaskIDs: []int64{7, 8}}
local := int64(10)
if source, inherited, allowed := findingProvenanceInTask(task, &local); !allowed || inherited || source != "" {
t.Fatalf("local finding: source=%q inherited=%v allowed=%v", source, inherited, allowed)
}
sourceID := int64(7)
if source, inherited, allowed := findingProvenanceInTask(task, &sourceID); !allowed || !inherited || source != "7" {
t.Fatalf("direct source finding: source=%q inherited=%v allowed=%v", source, inherited, allowed)
}
unrelated := int64(6)
if _, _, allowed := findingProvenanceInTask(task, &unrelated); allowed {
t.Fatal("unrelated finding must not be readable through task context")
}
if _, _, allowed := findingProvenanceInTask(task, nil); allowed {
t.Fatal("detached finding must not be readable through task context")
}
}
+850 -63
View File
File diff suppressed because it is too large. Load diff
+280
View File
@@ -0,0 +1,280 @@
package server
import (
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
pgdb "github.com/Autumn-27/artex/db"
"github.com/Autumn-27/artex/traffic"
)
func TestSeedAssociatesTargetAssetWithTask(t *testing.T) {
m, err := NewManager(t.TempDir(), "")
if err != nil {
t.Skipf("postgres unavailable (%v) - skipping", err)
}
defer m.Close()
task, err := m.CreateTask("seed ownership", "seed ownership", nil, 0, 0)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
_, _ = m.DeleteTask(task.ID, DeleteTaskOptions{DeleteAssets: true})
})
host := fmt.Sprintf("seed-%s.example.test", task.ID)
s := &Server{m: m}
s.seed(task, "https://"+host)
taskID := mustTaskID(t, task.ID)
assets, err := m.Assets().QueryByTask(taskID, "", 20, 0)
if err != nil {
t.Fatal(err)
}
found := false
for _, asset := range assets {
if asset.Domain == host || asset.URL == "https://"+host {
found = true
break
}
}
if !found {
t.Fatalf("seed target was not associated with task %s: %+v", task.ID, assets)
}
}
func TestDeleteTaskFilesRemovesOnlyOwnedWorkspaceAndTranscripts(t *testing.T) {
dataDir := t.TempDir()
removed := []struct {
path string
isDir bool
}{
{path: filepath.Join("tasks", "77"), isDir: true},
{path: filepath.Join("transcripts", "exp12-main.jsonl")},
{path: filepath.Join("transcripts", "exp12-planner.jsonl")},
{path: filepath.Join("transcripts", "exp12-worker-i9.jsonl")},
{path: filepath.Join("transcripts", "exp12-main"), isDir: true},
{path: filepath.Join("transcripts", "exp12-worker-i9"), isDir: true},
}
kept := []struct {
path string
isDir bool
}{
{path: filepath.Join("tasks", "78"), isDir: true},
{path: filepath.Join("transcripts", "exp123-main.jsonl")},
{path: filepath.Join("transcripts", "exp123-main"), isDir: true},
{path: filepath.Join("transcripts", "exp1-main.jsonl")},
{path: filepath.Join("transcripts", "exp12main.jsonl")},
{path: filepath.Join("transcripts", "exp12-main.jsonl.bak")},
{path: filepath.Join("transcripts", "exp12-main.txt")},
{path: filepath.Join("transcripts", "conv-12.jsonl")},
{path: filepath.Join("transcripts", "unrelated-session"), isDir: true},
}
for _, fixture := range append(removed, kept...) {
createDeleteFixture(t, dataDir, fixture.path, fixture.isDir)
}
deleted, err := deleteTaskFiles(dataDir, "77", 12)
if err != nil {
t.Fatalf("delete task files: %v", err)
}
if !deleted {
t.Fatal("expected deletion to be reported")
}
for _, fixture := range removed {
assertPathMissing(t, filepath.Join(dataDir, fixture.path))
}
for _, fixture := range kept {
assertPathExists(t, filepath.Join(dataDir, fixture.path))
}
}
func TestDeleteTaskFilesReportsTranscriptOnlyDeletion(t *testing.T) {
dataDir := t.TempDir()
path := filepath.Join("transcripts", "exp44-main.jsonl")
createDeleteFixture(t, dataDir, path, false)
deleted, err := deleteTaskFiles(dataDir, "missing-task", 44)
if err != nil {
t.Fatalf("delete task files: %v", err)
}
if !deleted {
t.Fatal("expected transcript deletion to be reported")
}
assertPathMissing(t, filepath.Join(dataDir, path))
}
func TestDeleteTaskFilesMissingTargetsIsIdempotent(t *testing.T) {
deleted, err := deleteTaskFiles(t.TempDir(), "77", 12)
if err != nil {
t.Fatalf("delete missing task files: %v", err)
}
if deleted {
t.Fatal("missing targets must not be reported as deleted")
}
}
func TestStageTaskFilesRollbackRestoresWorkspaceAndTranscripts(t *testing.T) {
dataDir := t.TempDir()
paths := []string{
filepath.Join("tasks", "77", "notes.txt"),
filepath.Join("transcripts", "exp12-main.jsonl"),
filepath.Join("transcripts", "exp12-worker-i9", "sidechain.jsonl"),
}
for _, path := range paths {
createDeleteFixture(t, dataDir, path, false)
}
stage, err := stageTaskFiles(dataDir, "77", 12)
if err != nil {
t.Fatal(err)
}
if !stage.deleted {
t.Fatal("expected files to be staged")
}
for _, path := range paths {
assertPathMissing(t, filepath.Join(dataDir, path))
}
if err := stage.rollback(); err != nil {
t.Fatal(err)
}
for _, path := range paths {
assertPathExists(t, filepath.Join(dataDir, path))
}
}
func TestStageTaskFilesRollbackReportsRestoreFailure(t *testing.T) {
dataDir := t.TempDir()
workspace := filepath.Join(dataDir, "tasks", "77")
createDeleteFixture(t, dataDir, filepath.Join("tasks", "77"), true)
stage, err := stageTaskFiles(dataDir, "77", 12)
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(workspace, []byte("conflict"), 0o644); err != nil {
t.Fatal(err)
}
cause := errors.New("forced PostgreSQL delete failure")
err = rollbackTaskDelete(cause, nil, stage)
if err == nil || !errors.Is(err, cause) || !strings.Contains(err.Error(), "destination already exists") {
t.Fatalf("rollback err=%v, want original and restore errors", err)
}
if _, statErr := os.Stat(stage.stageDir); statErr != nil {
t.Fatalf("staging was removed after failed restore: %v", statErr)
}
}
func TestManagerDeleteTaskRestoresFilesAndTrafficWhenPostgresDeleteFails(t *testing.T) {
dataDir := t.TempDir()
m, err := NewManager(dataDir, "")
if err != nil {
t.Skipf("postgres unavailable (%v) - skipping", err)
}
t.Cleanup(func() { _ = m.Close() })
task, err := m.CreateTask("delete rollback", "delete rollback", nil, 0, 0)
if err != nil {
t.Fatal(err)
}
taskID := mustTaskID(t, task.ID)
triggerName := fmt.Sprintf("test_fail_task_delete_%d", taskID)
functionName := triggerName + "_fn"
cleanupDB := func() {
_, _ = m.pg.Exec(fmt.Sprintf(`DROP TRIGGER IF EXISTS %s ON tasks`, triggerName))
_, _ = m.pg.Exec(fmt.Sprintf(`DROP FUNCTION IF EXISTS %s()`, functionName))
_, _ = m.DeleteTask(task.ID, DeleteTaskOptions{DeleteAssets: true, DeleteTraffic: true, DeleteFiles: true})
}
t.Cleanup(cleanupDB)
host := fmt.Sprintf("delete-rollback-%d.example.test", taskID)
if _, err := m.Assets().UpsertHTTPService(pgdb.UpsertHTTPServiceReq{URL: "https://" + host, TaskID: taskID}); err != nil {
t.Fatal(err)
}
createDeleteFixture(t, dataDir, filepath.Join("tasks", task.ID), true)
transcriptPath := filepath.Join(dataDir, "transcripts", fmt.Sprintf("exp%d-main.jsonl", task.ExpID))
createDeleteFixture(t, dataDir, filepath.Join("transcripts", filepath.Base(transcriptPath)), false)
tr, err := traffic.Open(filepath.Join(dataDir, "traffic"), "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
m.traffic = tr
trafficTree := filepath.Join(dataDir, "traffic", host, "GET", "1-0001")
if err := os.MkdirAll(trafficTree, 0o755); err != nil {
t.Fatal(err)
}
trafficMarker := filepath.Join(trafficTree, "request.http")
if err := os.WriteFile(trafficMarker, []byte("original traffic"), 0o644); err != nil {
t.Fatal(err)
}
if _, err := tr.DB().Exec(`INSERT INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
VALUES(?,?,?,?,?,?,?,?,?,?,?)`, "1-0001", 1, host, "GET", "/", "https://"+host+"/", 200, "text/plain", 0, 0, host+"/GET/1-0001"); err != nil {
t.Fatal(err)
}
if _, err := m.pg.Exec(fmt.Sprintf(`CREATE FUNCTION %s() RETURNS trigger LANGUAGE plpgsql AS $body$
BEGIN RAISE EXCEPTION 'forced task delete failure'; END $body$`, functionName)); err != nil {
t.Fatal(err)
}
if _, err := m.pg.Exec(fmt.Sprintf(`CREATE TRIGGER %s BEFORE DELETE ON tasks
FOR EACH ROW WHEN (OLD.id = %d) EXECUTE FUNCTION %s()`, triggerName, taskID, functionName)); err != nil {
t.Fatal(err)
}
_, err = m.DeleteTask(task.ID, DeleteTaskOptions{DeleteTraffic: true, DeleteFiles: true})
if err == nil || !strings.Contains(err.Error(), "forced task delete failure") {
t.Fatalf("delete err=%v, want injected PostgreSQL failure", err)
}
if got, err := m.pg.GetTask(taskID); err != nil || got == nil {
t.Fatalf("task row was lost after failed delete: task=%+v err=%v", got, err)
}
assertPathExists(t, filepath.Join(dataDir, "tasks", task.ID))
assertPathExists(t, transcriptPath)
if got, err := os.ReadFile(trafficMarker); err != nil || string(got) != "original traffic" {
t.Fatalf("traffic tree was not restored: content=%q err=%v", got, err)
}
var trafficRows int
if err := tr.DB().QueryRow(`SELECT COUNT(*) FROM exchanges WHERE host=?`, host).Scan(&trafficRows); err != nil || trafficRows != 1 {
t.Fatalf("traffic index was not restored: rows=%d err=%v", trafficRows, err)
}
}
func createDeleteFixture(t *testing.T, root, relative string, isDir bool) {
t.Helper()
path := filepath.Join(root, relative)
if isDir {
if err := os.MkdirAll(filepath.Join(path, "subagents"), 0o755); err != nil {
t.Fatalf("create fixture directory %s: %v", path, err)
}
if err := os.WriteFile(filepath.Join(path, "subagents", "agent-test.jsonl"), []byte("test"), 0o644); err != nil {
t.Fatalf("create fixture sidechain %s: %v", path, err)
}
return
}
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
t.Fatalf("create fixture parent %s: %v", path, err)
}
if err := os.WriteFile(path, []byte("test"), 0o644); err != nil {
t.Fatalf("create fixture file %s: %v", path, err)
}
}
func assertPathMissing(t *testing.T, path string) {
t.Helper()
if _, err := os.Lstat(path); !os.IsNotExist(err) {
t.Fatalf("expected %s to be removed, stat err=%v", path, err)
}
}
func assertPathExists(t *testing.T, path string) {
t.Helper()
if _, err := os.Lstat(path); err != nil {
t.Fatalf("expected %s to remain: %v", path, err)
}
}
+109
View File
@@ -0,0 +1,109 @@
package server
import (
"fmt"
"sync"
"testing"
)
func TestTaskLifecycleSnapshotCopiesContextSlices(t *testing.T) {
task := &Task{
Status: "running",
SourceTaskIDs: []int64{11, 12},
CompanyIDs: []int64{21, 22},
}
snapshot := task.lifecycleSnapshot()
snapshot.SourceTaskIDs[0] = 99
snapshot.CompanyIDs[0] = 98
fresh := task.lifecycleSnapshot()
if got := fresh.SourceTaskIDs[0]; got != 11 {
t.Fatalf("snapshot source IDs alias task storage: got %d", got)
}
if got := fresh.CompanyIDs[0]; got != 21 {
t.Fatalf("snapshot company IDs alias task storage: got %d", got)
}
sources := []int64{31, 32}
companies := []int64{41, 42}
task.updateLifecycle(func(state *taskLifecycleState) {
state.SourceTaskIDs = sources
state.CompanyIDs = companies
})
sources[0] = 97
companies[0] = 96
fresh = task.lifecycleSnapshot()
if got := fresh.SourceTaskIDs[0]; got != 31 {
t.Fatalf("lifecycle update retained caller source slice: got %d", got)
}
if got := fresh.CompanyIDs[0]; got != 41 {
t.Fatalf("lifecycle update retained caller company slice: got %d", got)
}
}
func TestTaskLifecycleSnapshotConcurrentConsistency(t *testing.T) {
task := &Task{}
writeState := func(generation int64, queued bool) {
task.updateLifecycle(func(state *taskLifecycleState) {
state.Status = map[bool]string{true: "queued", false: "paused"}[queued]
state.Queued = queued
state.Paused = !queued
state.QueuedAt = generation
state.QueueMode = map[bool]string{true: "resume", false: "bootstrap"}[queued]
state.CompletedAt = generation
state.FirstRunAt = generation
state.DeadlineAt = generation
state.SourceTaskIDs = []int64{generation, generation + 1}
state.CompanyIDs = []int64{generation, generation + 1}
})
}
writeState(1, true)
var wg sync.WaitGroup
for writer := 0; writer < 4; writer++ {
writer := writer
wg.Add(1)
go func() {
defer wg.Done()
for i := 1; i <= 1000; i++ {
generation := int64(writer*1000 + i)
writeState(generation, generation%2 == 0)
}
}()
}
errCh := make(chan error, 4)
for reader := 0; reader < 4; reader++ {
wg.Add(1)
go func() {
defer wg.Done()
for i := 0; i < 1000; i++ {
state := task.lifecycleSnapshot()
if state.Queued == state.Paused {
errCh <- fmt.Errorf("queued=%v paused=%v", state.Queued, state.Paused)
return
}
wantStatus := map[bool]string{true: "queued", false: "paused"}[state.Queued]
wantMode := map[bool]string{true: "resume", false: "bootstrap"}[state.Queued]
if state.Status != wantStatus || state.QueueMode != wantMode {
errCh <- fmt.Errorf("inconsistent status/mode: %+v", state)
return
}
generation := state.QueuedAt
if state.CompletedAt != generation || state.FirstRunAt != generation || state.DeadlineAt != generation ||
len(state.SourceTaskIDs) != 2 || len(state.CompanyIDs) != 2 ||
state.SourceTaskIDs[0] != generation || state.CompanyIDs[0] != generation {
errCh <- fmt.Errorf("mixed lifecycle snapshot: %+v", state)
return
}
}
}()
}
wg.Wait()
close(errCh)
for err := range errCh {
t.Fatal(err)
}
}
+14 -9
View File
@@ -199,12 +199,13 @@ func (s *Server) toolListTasks() actool.CoreTool {
if t.ParentRef != "" {
row["parent_ref"] = t.ParentRef
}
if t.LLMProfileID == nil {
llmState := t.llmStateSnapshot()
if llmState.ProfileID == nil {
row["llm_profile"] = "(激活配置)"
} else if n, ok := profName[*t.LLMProfileID]; ok {
} else if n, ok := profName[*llmState.ProfileID]; ok {
row["llm_profile"] = n
} else {
row["llm_profile"] = fmt.Sprintf("#%d(已删除)", *t.LLMProfileID)
row["llm_profile"] = fmt.Sprintf("#%d(已删除)", *llmState.ProfileID)
}
out = append(out, row)
}
@@ -301,10 +302,13 @@ func (s *Server) toolPauseTask() actool.CoreTool {
TaskID string `json:"task_id"`
}
_ = json.Unmarshal(in, &a)
if _, ok := s.m.Task(a.TaskID); !ok {
t, ok := s.m.Task(a.TaskID)
if !ok {
return actool.Errorf("task 不存在: " + a.TaskID), nil
}
s.engine.Pause(a.TaskID)
if _, err := s.applyTaskControlWithCause(t, "pause", agent.AbortPausedByOrchestrator); err != nil {
return actool.Errorf(err.Error()), nil
}
return actool.Text("task paused: " + a.TaskID), nil
})
}
@@ -414,12 +418,13 @@ func (s *Server) toolUpdateFindingReport() actool.CoreTool {
// deriveTaskStatus mirrors listTasks' status derivation for the list_tasks tool.
func (s *Server) deriveTaskStatus(t *Task) string {
lifecycle := t.lifecycleSnapshot()
switch {
case isTerminalStatus(t.Status):
return t.Status
case t.Paused || s.engine.IsPaused(t.ID):
case isTerminalStatus(lifecycle.Status):
return lifecycle.Status
case lifecycle.Paused || s.engine.IsPaused(t.ID):
return "paused"
case s.engine.Ready() && s.engine.Started(t.ID):
case s.engine.ReadyFor(t) && s.engine.Started(t.ID):
return "running"
}
return "created"
+24 -15
View File
@@ -50,6 +50,7 @@ func (sc *Scheduler) Run(ctx context.Context) {
case <-ctx.Done():
return
case <-t.C:
sc.s.reconcileConcurrency() // 并发上限:有空位就把排队任务补位启动
sc.step()
}
}
@@ -155,19 +156,22 @@ func (sc *Scheduler) fireFindings(triggers []*db.AgentTrigger) {
want = append(want, tr)
}
}
if len(want) == 0 {
return
}
last, _ := strconv.ParseInt(sc.mustState(schedKeyLastFinding), 10, 64)
events, err := sc.pg.NewFindingsSince(last)
if err != nil || len(events) == 0 {
return
}
// Advance the watermark whether or not any on_finding trigger is active: a
// finding fires only for triggers live at the moment it appears. Otherwise a
// trigger enabled later would replay the entire historical backlog at once.
maxID := last
for _, e := range events {
if e.NodeID > maxID {
maxID = e.NodeID
}
if len(want) == 0 {
continue
}
msgCtx := fmt.Sprintf("\n\n【本次由任务发现 finding 触发】\n任务: #%d %s(目标:%s)\n发现: [%s/%s] %s",
e.TaskID, e.TaskDesc, e.TaskGoal, e.VulnClass, e.Severity, e.Summary)
for _, tr := range want {
@@ -185,13 +189,12 @@ func (sc *Scheduler) fireGoals(triggers []*db.AgentTrigger) {
want = append(want, tr)
}
}
if len(want) == 0 {
return
}
events, err := sc.pg.MetGoals()
if err != nil || len(events) == 0 {
return
}
// Mark goals as consumed whether or not a trigger is active, so enabling an
// on_goal_met trigger later doesn't replay every already-met goal.
fired := sc.firedGoalSet()
changed := false
for _, e := range events {
@@ -200,6 +203,9 @@ func (sc *Scheduler) fireGoals(triggers []*db.AgentTrigger) {
}
fired[e.NodeID] = true
changed = true
if len(want) == 0 {
continue
}
msgCtx := fmt.Sprintf("\n\n【本次由任务完成目标触发】\n任务: #%d %s(目标:%s)\n达成目标: %s",
e.TaskID, e.TaskDesc, e.TaskGoal, e.Summary)
for _, tr := range want {
@@ -220,19 +226,20 @@ func (sc *Scheduler) fireTaskTimeouts(triggers []*db.AgentTrigger) {
want = append(want, tr)
}
}
if len(want) == 0 {
return
}
last, _ := strconv.ParseInt(sc.mustState(schedKeyLastTimeout), 10, 64)
events, err := sc.pg.TimedOutTasksSince(last)
if err != nil || len(events) == 0 {
return
}
// Advance the watermark even with no active trigger — see fireFindings.
maxID := last
for _, e := range events {
if e.NodeID > maxID {
maxID = e.NodeID
}
if len(want) == 0 {
continue
}
msgCtx := fmt.Sprintf("\n\n【本次由任务超时触发】\n任务: #%d %s(目标:%s)",
e.TaskID, e.TaskDesc, e.TaskGoal)
for _, tr := range want {
@@ -251,19 +258,20 @@ func (sc *Scheduler) fireTaskCreates(triggers []*db.AgentTrigger) {
want = append(want, tr)
}
}
if len(want) == 0 {
return
}
last, _ := strconv.ParseInt(sc.mustState(schedKeyLastTaskCreate), 10, 64)
events, err := sc.pg.NewTasksSince(last)
if err != nil || len(events) == 0 {
return
}
// Advance the watermark even with no active trigger — see fireFindings.
maxID := last
for _, e := range events {
if e.NodeID > maxID {
maxID = e.NodeID
}
if len(want) == 0 {
continue
}
msgCtx := fmt.Sprintf("\n\n【本次由任务创建触发】\n任务ID: #%d\n任务描述: %s\n任务目标: %s",
e.TaskID, e.TaskDesc, e.TaskGoal)
for _, tr := range want {
@@ -283,19 +291,20 @@ func (sc *Scheduler) fireToolCalls(triggers []*db.AgentTrigger) {
want = append(want, tr)
}
}
if len(want) == 0 {
return
}
last, _ := strconv.ParseInt(sc.mustState(schedKeyLastToolCall), 10, 64)
events, err := sc.pg.NewToolCallsSince(last)
if err != nil || len(events) == 0 {
return
}
// Advance the watermark even with no active trigger — see fireFindings.
maxID := last
for _, e := range events {
if e.NodeID > maxID {
maxID = e.NodeID
}
if len(want) == 0 {
continue
}
errTag := ""
if e.ToolIsErr {
errTag = "[error] "
+46
View File
@@ -0,0 +1,46 @@
package server
import (
"context"
"net/http"
"net/http/httptest"
"strings"
"testing"
)
func TestWorkerControlRoutes(t *testing.T) {
m, err := NewManager(t.TempDir(), "")
if err != nil {
t.Skipf("postgres unavailable (%v) - skipping", err)
}
defer m.Close()
td := t.TempDir()
s := New(context.Background(), m, td, td, td)
h := s.Handler()
token, err := signJWT(s.jwtKey)
if err != nil {
t.Fatalf("sign jwt: %v", err)
}
request := func(path string) *httptest.ResponseRecorder {
req := httptest.NewRequest(http.MethodPost, path, strings.NewReader(`{"action":"pause"}`))
req.Header.Set("Authorization", "Bearer "+token)
req.Header.Set("Content-Type", "application/json")
rec := httptest.NewRecorder()
h.ServeHTTP(rec, req)
return rec
}
// The single-worker route is still registered and reaches its JSON handler.
single := request("/api/tasks/missing/intents/1/control")
if single.Code != http.StatusNotFound || !strings.Contains(single.Body.String(), `"error":"task not found"`) {
t.Fatalf("single control route unavailable: status=%d body=%s", single.Code, single.Body.String())
}
// The former batch route must fall through the mux instead of reaching a task handler.
batch := request("/api/tasks/missing/intents/control/batch")
if batch.Code != http.StatusNotFound || !strings.Contains(batch.Body.String(), "404 page not found") {
t.Fatalf("batch control route still registered: status=%d body=%s", batch.Code, batch.Body.String())
}
}
+364
View File
@@ -0,0 +1,364 @@
package server
import (
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/Autumn-27/artex/agent"
)
func TestEngineDeleteBarrierRejectsNewTaskOperations(t *testing.T) {
e := NewEngine(nil)
const taskID = "42"
if !e.beginTaskOperation(taskID) {
t.Fatal("first task operation should be admitted")
}
if !e.BeginDelete(taskID) {
t.Fatal("first delete should install the barrier")
}
if !e.IsDeleting(taskID) {
t.Fatal("delete barrier should be visible")
}
if !e.IsPaused(taskID) {
t.Fatal("delete should temporarily pause task execution")
}
if e.beginTaskOperation(taskID) {
t.Fatal("operation started after BeginDelete returned")
}
if got := e.inflightCount(taskID); got != 1 {
t.Fatalf("existing operation must remain drainable, inflight=%d", got)
}
e.decInflight(taskID)
if e.BeginDelete(taskID) {
t.Fatal("duplicate delete must not acquire the barrier")
}
e.AbortDelete(taskID, false)
if e.IsPaused(taskID) {
t.Fatal("aborted delete should restore a previously running task")
}
if !e.beginTaskOperation(taskID) {
t.Fatal("aborting deletion should reopen operation admission")
}
e.decInflight(taskID)
}
func TestAbortDeletePreservesExistingPause(t *testing.T) {
e := NewEngine(nil)
e.Pause("42", agent.AbortPausedByUser)
if !e.BeginDelete("42") {
t.Fatal("delete barrier should be installed")
}
e.AbortDelete("42", true)
if !e.IsPaused("42") {
t.Fatal("aborted delete must preserve a pre-existing user pause")
}
}
func TestBeginTaskDeleteSerializesWithLifecycleLock(t *testing.T) {
s := &Server{engine: NewEngine(nil)}
s.concMu.Lock()
done := make(chan bool, 1)
go func() { done <- s.beginTaskDelete("42") }()
select {
case <-done:
t.Fatal("delete barrier bypassed the lifecycle lock")
case <-time.After(20 * time.Millisecond):
}
if s.engine.IsDeleting("42") {
t.Fatal("delete barrier was installed before concMu was released")
}
s.concMu.Unlock()
select {
case ok := <-done:
if !ok {
t.Fatal("delete barrier was not installed after lifecycle lock release")
}
case <-time.After(time.Second):
t.Fatal("timed out waiting for delete barrier")
}
s.engine.AbortDelete("42", false)
}
func TestDeletingRunningTaskKeepsConcurrencySlotUntilCommit(t *testing.T) {
task := &Task{ID: "42", Status: "running"}
s := &Server{engine: NewEngine(nil)}
if !s.engine.BeginDelete(task.ID) {
t.Fatal("install delete barrier")
}
if !s.occupiesConcurrencySlot(task) {
t.Fatal("temporary delete pause released the running task's slot before commit")
}
s.engine.AbortDelete(task.ID, false)
}
func TestTaskLifecycleRechecksDeleteBarrierAfterConcLock(t *testing.T) {
task := &Task{ID: "42", notify: make(chan struct{}, 1)}
m := &Manager{tasks: map[string]*Task{task.ID: task}}
s := &Server{m: m, engine: NewEngine(m)}
s.concMu.Lock()
started := make(chan struct{})
pauseDone := make(chan error, 1)
go func() {
close(started)
_, err := s.applyTaskControl(task, "pause")
pauseDone <- err
}()
<-started
if !s.engine.BeginDelete(task.ID) {
t.Fatal("install delete barrier")
}
s.concMu.Unlock()
if err := <-pauseDone; err == nil || !strings.Contains(err.Error(), "正在删除") {
t.Fatalf("pause passed a delete barrier installed while waiting for concMu: %v", err)
}
s.engine.AbortDelete(task.ID, false)
s.concMu.Lock()
started = make(chan struct{})
admitDone := make(chan error, 1)
go func() {
close(started)
_, err := s.admitTask(task, "resume")
admitDone <- err
}()
<-started
if !s.engine.BeginDelete(task.ID) {
t.Fatal("install second delete barrier")
}
s.concMu.Unlock()
if err := <-admitDone; err == nil || !strings.Contains(err.Error(), "being deleted") {
t.Fatalf("admission passed a delete barrier installed while waiting for concMu: %v", err)
}
s.engine.AbortDelete(task.ID, false)
}
func TestAbortTaskDeleteUsesPersistedPauseAndQueueState(t *testing.T) {
m, err := NewManager(t.TempDir(), "")
if err != nil {
t.Skipf("postgres unavailable (%v) - skipping", err)
}
defer m.Close()
task, err := m.CreateTask("delete abort state", "restore persisted barrier", nil, 0, 0)
if err != nil {
t.Fatal(err)
}
defer func() { _, _ = m.DeleteTask(task.ID, DeleteTaskOptions{}) }()
s := &Server{m: m, engine: NewEngine(m)}
// An engine-only pause is not authoritative. The committed task is running,
// so aborting deletion must clear this stale in-memory barrier.
s.engine.Pause(task.ID, agent.AbortPausedByUser)
if !s.beginTaskDelete(task.ID) {
t.Fatal("install first delete barrier")
}
s.abortTaskDelete(task.ID)
if s.engine.IsPaused(task.ID) {
t.Fatal("stale engine pause survived although the persisted task is running")
}
// A queued row must retain its execution barrier even when BeginDelete saw the
// engine as running. This specifically guards against restoring a stale snapshot.
if err := m.EnqueueTask(task.ID, "resume"); err != nil {
t.Fatal(err)
}
if !s.beginTaskDelete(task.ID) {
t.Fatal("install second delete barrier")
}
s.abortTaskDelete(task.ID)
if !s.engine.IsPaused(task.ID) {
t.Fatal("persisted queued task lost its execution barrier after abort")
}
}
func TestStopTaskCancelsRuntimeAndClearsLifecycleState(t *testing.T) {
e := NewEngine(nil)
const taskID = "42"
e.deleteMu.RLock()
rt := e.registerTaskRoutines(context.Background(), taskID, 1)
e.started.Store(taskID, true)
e.lastAct.Store(taskID, int64(1))
e.paused.Store(taskID, true)
e.plannerRound.Store(taskID, 3)
e.coordStarted.Store(taskID, true)
e.deleteMu.RUnlock()
done := make(chan struct{})
runTaskRoutine(rt, func(ctx context.Context) {
<-ctx.Done()
close(done)
})
e.StopTask(taskID)
select {
case <-done:
default:
t.Fatal("task runtime was not cancelled before StopTask returned")
}
if e.Started(taskID) || e.IsPaused(taskID) || e.IsDeleting(taskID) {
t.Fatalf("task lifecycle maps were not cleared: started=%v paused=%v deleting=%v",
e.Started(taskID), e.IsPaused(taskID), e.IsDeleting(taskID))
}
if _, ok := e.lastAct.Load(taskID); ok {
t.Fatal("last activity state was retained")
}
if _, ok := e.plannerRound.Load(taskID); ok {
t.Fatal("planner round state was retained")
}
if _, ok := e.coordStarted.Load(taskID); ok {
t.Fatal("deadline coordinator state was retained")
}
e.runtimeMu.Lock()
_, retained := e.runtimes[taskID]
e.runtimeMu.Unlock()
if retained {
t.Fatal("task runtime retained after StopTask")
}
}
func TestCanonicalTaskID(t *testing.T) {
for raw, want := range map[string]string{"42": "42", "00042": "42", "+42": "42"} {
if got, ok := canonicalTaskID(raw); !ok || got != want {
t.Fatalf("canonicalTaskID(%q)=(%q,%v), want (%q,true)", raw, got, ok, want)
}
}
for _, raw := range []string{"", "0", "-1", "abc", "42x"} {
if got, ok := canonicalTaskID(raw); ok {
t.Fatalf("canonicalTaskID(%q)=(%q,true), want invalid", raw, got)
}
}
}
func TestCancelChatThenWaitTaskQuiescent(t *testing.T) {
e := NewEngine(nil)
chatCtx, chatCancel := context.WithCancelCause(context.Background())
s := &Server{
engine: e,
chatBusy: map[string]bool{"7": true},
chatCancel: map[string]context.CancelCauseFunc{"7": chatCancel},
}
if !e.beginTaskOperation("7") {
t.Fatal("task operation should be admitted")
}
cancelObserved := make(chan struct{})
releaseChat := make(chan struct{})
go func() {
<-chatCtx.Done()
close(cancelObserved)
<-releaseChat
s.finishTaskChat("7", chatCancel)
}()
if !s.cancelTaskChat("7", agent.AbortTaskDeleted) {
t.Fatal("expected active chat to be cancelled")
}
<-cancelObserved
waitDone := make(chan error, 1)
waitCtx, waitCancel := context.WithTimeout(context.Background(), time.Second)
defer waitCancel()
go func() { waitDone <- s.waitTaskQuiescent(waitCtx, "7") }()
select {
case err := <-waitDone:
t.Fatalf("wait returned while chat and engine operation were active: %v", err)
case <-time.After(20 * time.Millisecond):
}
close(releaseChat)
select {
case err := <-waitDone:
t.Fatalf("wait returned while engine operation was active: %v", err)
case <-time.After(20 * time.Millisecond):
}
e.decInflight("7")
if err := <-waitDone; err != nil {
t.Fatalf("wait for quiescence: %v", err)
}
}
func TestWaitTaskQuiescentHonorsContext(t *testing.T) {
s := &Server{
engine: NewEngine(nil),
chatBusy: map[string]bool{"9": true},
chatCancel: map[string]context.CancelCauseFunc{},
}
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
defer cancel()
if err := s.waitTaskQuiescent(ctx, "9"); !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("expected deadline exceeded, got %v", err)
}
}
func TestTaskChatStatusUsesAuthoritativeBusyState(t *testing.T) {
m := &Manager{tasks: map[string]*Task{"7": {ID: "7"}}}
s := &Server{
m: m,
chatBusy: map[string]bool{"7": true},
chatCancel: map[string]context.CancelCauseFunc{},
}
req := httptest.NewRequest(http.MethodGet, "/api/tasks/7/chat/status", nil)
req.SetPathValue("id", "7")
rec := httptest.NewRecorder()
s.taskChatStatus(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status=%d body=%s, want 200", rec.Code, rec.Body.String())
}
var got struct {
Running bool `json:"running"`
}
if err := json.NewDecoder(rec.Body).Decode(&got); err != nil {
t.Fatal(err)
}
if !got.Running {
t.Fatal("busy main-agent turn was reported idle")
}
}
func TestTaskChatUploadRejectsDeleteBarrierBeforeCreatingDirectory(t *testing.T) {
dataDir := t.TempDir()
m := &Manager{dir: dataDir, tasks: map[string]*Task{"7": {ID: "7"}}}
s := &Server{m: m, engine: NewEngine(m)}
if !s.engine.BeginDelete("7") {
t.Fatal("delete barrier should be installed")
}
req := httptest.NewRequest(http.MethodPost, "/api/chat/upload?scope=task&id=7", nil)
rec := httptest.NewRecorder()
s.chatUpload(rec, req)
if rec.Code != http.StatusConflict {
t.Fatalf("expected 409 while deleting, got %d: %s", rec.Code, rec.Body.String())
}
if _, err := os.Stat(filepath.Join(dataDir, "tasks", "7")); !os.IsNotExist(err) {
t.Fatalf("upload must not recreate task directory, stat err=%v", err)
}
}
func TestCommittedTaskDeleteReturnsSuccessWithCleanupWarning(t *testing.T) {
rec := httptest.NewRecorder()
result := DeleteTaskResult{Deleted: "7", FilesDeleted: true}
err := &taskDeleteCommittedError{err: errors.New("remove staged task files: permission denied")}
writeCommittedTaskDelete(rec, result, err)
if rec.Code != http.StatusOK {
t.Fatalf("status=%d body=%s, want 200", rec.Code, rec.Body.String())
}
var got DeleteTaskResult
if decodeErr := json.NewDecoder(rec.Body).Decode(&got); decodeErr != nil {
t.Fatal(decodeErr)
}
if got.Deleted != "7" || !got.FilesDeleted || !strings.Contains(got.CleanupWarning, "cleanup incomplete") {
t.Fatalf("unexpected response: %+v", got)
}
}