feat(tasks): add runtime controls and inherited context
This commit is contained in:
1 parent
18926c129f
commit
4392b4fee1
50 files changed
+8291
-663
No files matched your search
@@ -45,9 +45,11 @@ type chatAttachment struct {
|
||||
// 文件先落这里,前端按返回的 abs 绝对路径写进任务描述)
|
||||
func (s *Server) chatUpload(w http.ResponseWriter, r *http.Request) {
|
||||
var sub string
|
||||
taskScoped := false
|
||||
switch r.URL.Query().Get("scope") {
|
||||
case "task":
|
||||
sub = "tasks"
|
||||
taskScoped = true
|
||||
case "session":
|
||||
sub = "sessions"
|
||||
case "staging":
|
||||
@@ -61,6 +63,17 @@ func (s *Server) chatUpload(w http.ResponseWriter, r *http.Request) {
|
||||
writeErr(w, 400, "非法 id")
|
||||
return
|
||||
}
|
||||
if taskScoped {
|
||||
if s.m.ResolveTask(id) == nil {
|
||||
writeErr(w, 404, "task not found")
|
||||
return
|
||||
}
|
||||
if !s.engine.beginTaskOperation(id) {
|
||||
writeErr(w, http.StatusConflict, "任务正在删除,无法上传附件")
|
||||
return
|
||||
}
|
||||
defer s.engine.decInflight(id)
|
||||
}
|
||||
dir := filepath.Join(s.m.dir, sub, id, "uploads")
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
writeErr(w, 500, "建目录失败: "+err.Error())
|
||||
|
||||
+523
-137
@@ -71,8 +71,18 @@ func preview(s string, n int) string {
|
||||
const (
|
||||
modelErrorRetries = 2 // model_error 收场后额外重试的次数
|
||||
modelErrorRetryBackoff = 3 * time.Second // 每次重试前的退避
|
||||
workControlWaitTimeout = 30 * time.Second
|
||||
)
|
||||
|
||||
var errWorkControlConflict = errors.New("work control conflict")
|
||||
|
||||
// retryableWorkerModelError excludes errors already handled by the task router.
|
||||
// In particular, a quota error after partial streaming advances the task cursor
|
||||
// for the next LLM call but must not replay this whole intent on the backup.
|
||||
func retryableWorkerModelError(reason harness.TerminalReason, err error) bool {
|
||||
return reason == harness.ReasonModelError && !isTaskLLMRuntimeError(err)
|
||||
}
|
||||
|
||||
// Engine drives the event-driven exploration loop with real LLM agents
|
||||
// (docs §4.3/§4.4): on asset/exploration-graph change (debounced) it wakes the
|
||||
// planner, which reads the route, queries assets, judges goals and emits intents;
|
||||
@@ -86,25 +96,40 @@ type Engine struct {
|
||||
|
||||
bc *Broadcaster // live activity pub/sub (SSE)
|
||||
|
||||
mu sync.RWMutex
|
||||
planner *agent.Planner
|
||||
worker *agent.Worker
|
||||
started sync.Map // taskID -> bool, so Run is idempotent per task
|
||||
lastAct sync.Map // taskID -> int64 unix, last planner/worker activity (heartbeat)
|
||||
paused sync.Map // taskID -> bool, user-paused (planner + workers idle but loops alive)
|
||||
dropCnt sync.Map // taskID -> *int64, running count of dropped (unpersistable) activity records
|
||||
mu sync.RWMutex
|
||||
planner *agent.Planner
|
||||
worker *agent.Worker
|
||||
started sync.Map // taskID -> bool, so Run is idempotent per task
|
||||
lastAct sync.Map // taskID -> int64 unix, last planner/worker activity (heartbeat)
|
||||
llmCalls sync.Map // taskID -> *int64, actual planner/worker/main-agent LLM calls
|
||||
paused sync.Map // taskID -> bool, user-paused (planner + workers idle but loops alive)
|
||||
deleting sync.Map // taskID -> bool, delete barrier (no new task-owned writes)
|
||||
dropCnt sync.Map // taskID -> *int64, running count of dropped (unpersistable) activity records
|
||||
|
||||
// deleteMu makes installing the delete barrier atomic with registering a new
|
||||
// task operation. Once BeginDelete returns, every admitted writer is reflected
|
||||
// in inflight and every later writer is rejected.
|
||||
deleteMu sync.RWMutex
|
||||
|
||||
// Every long-lived task goroutine (planner, workers and deadline coordinator)
|
||||
// runs under one task-scoped context. Successful deletion cancels that context,
|
||||
// waits for all goroutines, then releases every task-level Engine reference.
|
||||
runtimeMu sync.Mutex
|
||||
runtimes map[string]*taskRuntime
|
||||
|
||||
// per-task execution context: each planner.Plan / worker.Execute runs under it,
|
||||
// so pausing can CANCEL an in-flight run (not just skip the next one). Recreated
|
||||
// on resume since cancelling is one-shot.
|
||||
// on resume since cancelling is one-shot. Every cancellation carries a named
|
||||
// cause so the activity trace can identify the initiating control path.
|
||||
execMu sync.Mutex
|
||||
execCancel map[string]context.CancelFunc
|
||||
execCancel map[string]context.CancelCauseFunc
|
||||
execCtx map[string]context.Context
|
||||
|
||||
// per-work (per running intent) cancel, so the planner's kill_work can stop a
|
||||
// single worker without pausing the whole task. Keyed by globally-unique intent id.
|
||||
workMu sync.Mutex
|
||||
workCancel map[int64]context.CancelFunc
|
||||
// Per-work control lets the planner kill a worker and lets the UI pause/cancel
|
||||
// one intent without pausing the whole task. The done channel closes only after
|
||||
// runWorkerStep has stopped writing and committed its final state.
|
||||
workMu sync.Mutex
|
||||
work map[int64]*workExecution
|
||||
|
||||
// steerBox queues planner course-corrections for a running work (keyed by intent
|
||||
// id). The worker's PreToolUse hook drains it before its next tool call and hands
|
||||
@@ -121,9 +146,23 @@ type Engine struct {
|
||||
inflight sync.Map // taskID -> *int64, 在跑的 planner.Plan + worker.Execute 计数(用于 drain)
|
||||
coordStarted sync.Map // taskID -> bool, deadline 协调器是否已启动(Run/reload 去重)
|
||||
|
||||
// resolve, if set, returns a task's dedicated planner/worker when it pins a
|
||||
// specific LLM profile (nil,nil = use the global active pair). Set once at startup.
|
||||
resolve func(t *Task) (*agent.Planner, *agent.Worker)
|
||||
// resolve, if set, returns a task's dedicated planner/worker. In legacy mode
|
||||
// nil,nil falls back to the global pair; an authoritative resolver uses nil,nil
|
||||
// to mean this task is deliberately unavailable (for example an exhausted chain).
|
||||
resolve func(t *Task) (*agent.Planner, *agent.Worker)
|
||||
resolveAuthoritative bool
|
||||
}
|
||||
|
||||
type taskRuntime struct {
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
type workExecution struct {
|
||||
cancel context.CancelCauseFunc
|
||||
done chan error
|
||||
action string // user action: pause | cancel
|
||||
}
|
||||
|
||||
// nextPlannerRound returns the next planner round number for a task (1-based).
|
||||
@@ -136,18 +175,128 @@ func (e *Engine) nextPlannerRound(taskID string) int {
|
||||
|
||||
// Pause stops a task: marks it paused AND cancels any in-flight planner/worker run
|
||||
// for it (a long worker.Execute would otherwise keep going until it finishes).
|
||||
func (e *Engine) Pause(taskID string) {
|
||||
func (e *Engine) Pause(taskID string, cause error) {
|
||||
e.paused.Store(taskID, true)
|
||||
e.cancelExec(taskID)
|
||||
e.cancelExec(taskID, cause)
|
||||
}
|
||||
|
||||
// BeginDelete installs an execution barrier before task data/files are removed.
|
||||
// The temporary pause is not a user pause. The server serializes this transition
|
||||
// with lifecycle admission and tells AbortDelete whether the persisted task is
|
||||
// paused/queued if cleanup fails.
|
||||
func (e *Engine) BeginDelete(taskID string) bool {
|
||||
e.deleteMu.Lock()
|
||||
if _, loaded := e.deleting.LoadOrStore(taskID, true); loaded {
|
||||
e.deleteMu.Unlock()
|
||||
return false
|
||||
}
|
||||
e.paused.Store(taskID, true)
|
||||
e.deleteMu.Unlock()
|
||||
e.cancelExec(taskID, agent.AbortTaskDeleted)
|
||||
return true
|
||||
}
|
||||
|
||||
func (e *Engine) AbortDelete(taskID string, keepPaused bool) {
|
||||
e.deleteMu.Lock()
|
||||
if !e.IsDeleting(taskID) {
|
||||
e.deleteMu.Unlock()
|
||||
return
|
||||
}
|
||||
e.deleting.Delete(taskID)
|
||||
if !keepPaused {
|
||||
e.paused.Delete(taskID)
|
||||
}
|
||||
e.deleteMu.Unlock()
|
||||
if !keepPaused && e.m != nil {
|
||||
if t, ok := e.m.Task(taskID); ok {
|
||||
t.Notify()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) IsDeleting(taskID string) bool {
|
||||
_, ok := e.deleting.Load(taskID)
|
||||
return ok
|
||||
}
|
||||
|
||||
// registerTaskRoutines reserves count goroutines in the task runtime. Callers
|
||||
// hold deleteMu for reading so StopTask cannot race WaitGroup.Add with Wait.
|
||||
func (e *Engine) registerTaskRoutines(parent context.Context, taskID string, count int) *taskRuntime {
|
||||
e.runtimeMu.Lock()
|
||||
defer e.runtimeMu.Unlock()
|
||||
rt := e.runtimes[taskID]
|
||||
if rt == nil {
|
||||
ctx, cancel := context.WithCancel(parent)
|
||||
rt = &taskRuntime{ctx: ctx, cancel: cancel}
|
||||
e.runtimes[taskID] = rt
|
||||
}
|
||||
rt.wg.Add(count)
|
||||
return rt
|
||||
}
|
||||
|
||||
func runTaskRoutine(rt *taskRuntime, fn func(context.Context)) {
|
||||
go func() {
|
||||
defer rt.wg.Done()
|
||||
fn(rt.ctx)
|
||||
}()
|
||||
}
|
||||
|
||||
// StopTask permanently stops every long-lived goroutine and removes all Engine
|
||||
// state for a successfully deleted task. The delete barrier remains installed
|
||||
// until cleanup finishes, so no new task operation can race the teardown.
|
||||
func (e *Engine) StopTask(taskID string) {
|
||||
e.deleteMu.Lock()
|
||||
e.deleting.Store(taskID, true)
|
||||
e.deleteMu.Unlock()
|
||||
|
||||
e.cancelExec(taskID, agent.AbortTaskDeleted)
|
||||
e.runtimeMu.Lock()
|
||||
rt := e.runtimes[taskID]
|
||||
if rt != nil {
|
||||
rt.cancel()
|
||||
}
|
||||
e.runtimeMu.Unlock()
|
||||
if rt != nil {
|
||||
rt.wg.Wait()
|
||||
}
|
||||
|
||||
e.execMu.Lock()
|
||||
if cancel := e.execCancel[taskID]; cancel != nil {
|
||||
cancel(agent.AbortTaskDeleted)
|
||||
}
|
||||
delete(e.execCancel, taskID)
|
||||
delete(e.execCtx, taskID)
|
||||
e.execMu.Unlock()
|
||||
|
||||
e.runtimeMu.Lock()
|
||||
if e.runtimes[taskID] == rt {
|
||||
delete(e.runtimes, taskID)
|
||||
}
|
||||
e.runtimeMu.Unlock()
|
||||
|
||||
e.started.Delete(taskID)
|
||||
e.lastAct.Delete(taskID)
|
||||
e.llmCalls.Delete(taskID)
|
||||
e.paused.Delete(taskID)
|
||||
e.dropCnt.Delete(taskID)
|
||||
e.plannerRound.Delete(taskID)
|
||||
e.settling.Delete(taskID)
|
||||
e.deadline.Delete(taskID)
|
||||
e.stamped.Delete(taskID)
|
||||
e.inflight.Delete(taskID)
|
||||
e.coordStarted.Delete(taskID)
|
||||
e.deleteMu.Lock()
|
||||
e.deleting.Delete(taskID)
|
||||
e.deleteMu.Unlock()
|
||||
}
|
||||
|
||||
// cancelExec cancels a task's current per-task exec context (any in-flight
|
||||
// planner.Plan / worker.Execute), if present. Shared by Pause and the settle
|
||||
// sequence's hard-drain backstop.
|
||||
func (e *Engine) cancelExec(taskID string) {
|
||||
func (e *Engine) cancelExec(taskID string, cause error) {
|
||||
e.execMu.Lock()
|
||||
if cancel := e.execCancel[taskID]; cancel != nil {
|
||||
cancel()
|
||||
cancel(cause)
|
||||
}
|
||||
e.execMu.Unlock()
|
||||
}
|
||||
@@ -155,6 +304,17 @@ func (e *Engine) cancelExec(taskID string) {
|
||||
// Resume un-pauses a task and nudges a fresh planning round. The next exec under
|
||||
// it gets a fresh (uncancelled) context.
|
||||
func (e *Engine) Resume(t *Task) {
|
||||
// BeginDelete owns the pause barrier once deletion starts. A concurrent
|
||||
// resume must never clear it and let a planner/worker re-enter while cleanup
|
||||
// is waiting for task operations to drain.
|
||||
if t == nil {
|
||||
return
|
||||
}
|
||||
e.deleteMu.RLock()
|
||||
defer e.deleteMu.RUnlock()
|
||||
if e.IsDeleting(t.ID) {
|
||||
return
|
||||
}
|
||||
e.paused.Delete(t.ID)
|
||||
t.Notify()
|
||||
}
|
||||
@@ -166,14 +326,14 @@ func (e *Engine) execContextFor(parent context.Context, taskID string) context.C
|
||||
defer e.execMu.Unlock()
|
||||
if e.IsPaused(taskID) {
|
||||
// never hand out a live context while paused (guards the claim→Execute race)
|
||||
c, cancel := context.WithCancel(parent)
|
||||
cancel()
|
||||
c, cancel := context.WithCancelCause(parent)
|
||||
cancel(agent.AbortPausedRaceGuard)
|
||||
return c
|
||||
}
|
||||
if c := e.execCtx[taskID]; c != nil && c.Err() == nil {
|
||||
return c
|
||||
}
|
||||
c, cancel := context.WithCancel(parent)
|
||||
c, cancel := context.WithCancelCause(parent)
|
||||
e.execCtx[taskID] = c
|
||||
e.execCancel[taskID] = cancel
|
||||
return c
|
||||
@@ -200,32 +360,131 @@ func (e *Engine) LastActivity(taskID string) int64 {
|
||||
return 0
|
||||
}
|
||||
|
||||
// BeginLLMCall/EndLLMCall track actual provider calls separately from the
|
||||
// scheduler's task-operation counter. A task can have live loops while all of
|
||||
// them are waiting for a trigger; that state must remain idle in the UI.
|
||||
func (e *Engine) BeginLLMCall(taskID string) {
|
||||
v, _ := e.llmCalls.LoadOrStore(taskID, new(int64))
|
||||
atomic.AddInt64(v.(*int64), 1)
|
||||
}
|
||||
|
||||
func (e *Engine) EndLLMCall(taskID string) {
|
||||
if v, ok := e.llmCalls.Load(taskID); ok {
|
||||
p := v.(*int64)
|
||||
if atomic.AddInt64(p, -1) <= 0 {
|
||||
atomic.StoreInt64(p, 0)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) ActiveLLMCalls(taskID string) int64 {
|
||||
if v, ok := e.llmCalls.Load(taskID); ok {
|
||||
return atomic.LoadInt64(v.(*int64))
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (e *Engine) touch(taskID string) { e.lastAct.Store(taskID, time.Now().Unix()) }
|
||||
|
||||
func NewEngine(m *Manager) *Engine {
|
||||
return &Engine{m: m, debounce: 800 * time.Millisecond, bc: NewBroadcaster(),
|
||||
execCancel: map[string]context.CancelFunc{}, execCtx: map[string]context.Context{},
|
||||
workCancel: map[int64]context.CancelFunc{}, steerBox: map[int64][]string{}}
|
||||
execCancel: map[string]context.CancelCauseFunc{}, execCtx: map[string]context.Context{},
|
||||
work: map[int64]*workExecution{}, steerBox: map[int64][]string{},
|
||||
runtimes: map[string]*taskRuntime{}}
|
||||
}
|
||||
|
||||
// registerWork records the cancel for the work currently running intentID.
|
||||
func (e *Engine) registerWork(intentID int64, cancel context.CancelFunc) {
|
||||
func (e *Engine) registerWork(intentID int64, cancel context.CancelCauseFunc) {
|
||||
e.workMu.Lock()
|
||||
e.workCancel[intentID] = cancel
|
||||
e.work[intentID] = &workExecution{cancel: cancel, done: make(chan error, 1)}
|
||||
e.workMu.Unlock()
|
||||
}
|
||||
|
||||
// unregisterWork removes and releases the work's cancel (called when Execute returns).
|
||||
func (e *Engine) unregisterWork(intentID int64) {
|
||||
// detachWork removes the live control handle once Execute has returned. complete
|
||||
// must be called after the final intent state write so a waiting cancel handler can
|
||||
// safely delete the worker's blackboard output without racing a late write.
|
||||
func (e *Engine) detachWork(intentID int64) (action string, complete func(error)) {
|
||||
e.workMu.Lock()
|
||||
if c := e.workCancel[intentID]; c != nil {
|
||||
delete(e.workCancel, intentID)
|
||||
c() // release context resources (no-op if already cancelled)
|
||||
run := e.work[intentID]
|
||||
if run != nil {
|
||||
delete(e.work, intentID)
|
||||
action = run.action
|
||||
run.cancel(agent.AbortWorkFinished) // release resources (no-op if already cancelled)
|
||||
}
|
||||
e.workMu.Unlock()
|
||||
e.steerMu.Lock()
|
||||
delete(e.steerBox, intentID) // drop any undelivered steering for a finished work
|
||||
e.steerMu.Unlock()
|
||||
if run == nil {
|
||||
return action, func(error) {}
|
||||
}
|
||||
return action, func(err error) { run.done <- err }
|
||||
}
|
||||
|
||||
// ControlWork requests a user-visible pause or cancellation and waits until the
|
||||
// worker has fully stopped writing. Cancellation cleanup is performed by the API
|
||||
// handler after this returns; pause state is committed by runWorkerStep itself.
|
||||
func (e *Engine) ControlWork(ctx context.Context, intentID int64, action string) error {
|
||||
if action != "pause" && action != "cancel" {
|
||||
return fmt.Errorf("unsupported work action %q", action)
|
||||
}
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
e.workMu.Lock()
|
||||
run := e.work[intentID]
|
||||
if run == nil {
|
||||
e.workMu.Unlock()
|
||||
return fmt.Errorf("%w: 意图 %d 当前没有运行中的 work(可能已结束或未被领取)", errWorkControlConflict, intentID)
|
||||
}
|
||||
if run.action != "" {
|
||||
e.workMu.Unlock()
|
||||
return fmt.Errorf("%w: 意图 %d 正在执行 %s 操作", errWorkControlConflict, intentID, run.action)
|
||||
}
|
||||
run.action = action
|
||||
done := run.done
|
||||
cause := error(agent.AbortWorkPausedByUser)
|
||||
if action == "cancel" {
|
||||
cause = agent.AbortWorkCancelledByUser
|
||||
}
|
||||
run.cancel(cause)
|
||||
e.workMu.Unlock()
|
||||
|
||||
timer := time.NewTimer(workControlWaitTimeout)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case err := <-done:
|
||||
return err
|
||||
case <-ctx.Done():
|
||||
e.releaseWorkControl(intentID, run, action)
|
||||
return fmt.Errorf("等待意图 %d %s 收尾: %w", intentID, action, ctx.Err())
|
||||
case <-timer.C:
|
||||
e.releaseWorkControl(intentID, run, action)
|
||||
return fmt.Errorf("等待意图 %d %s 收尾: %w", intentID, action, context.DeadlineExceeded)
|
||||
}
|
||||
}
|
||||
|
||||
// releaseWorkControl drops only this caller's reservation after its wait is
|
||||
// cancelled. The work context stays cancelled; runWorkerStep recognizes the
|
||||
// named cancellation cause and settles the intent into the recoverable paused
|
||||
// state even if the HTTP caller has gone away.
|
||||
func (e *Engine) releaseWorkControl(intentID int64, run *workExecution, action string) {
|
||||
e.workMu.Lock()
|
||||
if current := e.work[intentID]; current == run && current.action == action {
|
||||
current.action = ""
|
||||
}
|
||||
e.workMu.Unlock()
|
||||
}
|
||||
|
||||
func transitionIntentState(store *db.ExplorationStore, intentID int64, expected, state string) error {
|
||||
changed, err := store.CompareAndSetIntentState(intentID, expected, state)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !changed {
|
||||
return fmt.Errorf("%w: 意图 %d 不再是 %s 状态", db.ErrIntentStateConflict, intentID, expected)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SteerWork queues a mid-run course-correction for the work running intentID (the
|
||||
@@ -236,7 +495,7 @@ func (e *Engine) SteerWork(intentID int64, msg string) error {
|
||||
return fmt.Errorf("纠偏消息不能为空")
|
||||
}
|
||||
e.workMu.Lock()
|
||||
running := e.workCancel[intentID] != nil
|
||||
running := e.work[intentID] != nil
|
||||
e.workMu.Unlock()
|
||||
if !running {
|
||||
return fmt.Errorf("意图 %d 当前没有运行中的 work(可能已结束或未被领取)", intentID)
|
||||
@@ -301,12 +560,12 @@ func (h steerHooks) Stop(ctx context.Context, messages []llm.Message) (bool, []s
|
||||
// The work's agent-core session honors ctx cancellation and aborts promptly.
|
||||
func (e *Engine) KillWork(intentID int64) error {
|
||||
e.workMu.Lock()
|
||||
c := e.workCancel[intentID]
|
||||
run := e.work[intentID]
|
||||
e.workMu.Unlock()
|
||||
if c == nil {
|
||||
if run == nil {
|
||||
return fmt.Errorf("意图 %d 当前没有运行中的 work(可能已结束或未被领取)", intentID)
|
||||
}
|
||||
c()
|
||||
run.cancel(agent.AbortKilledByPlanner)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -315,7 +574,7 @@ func (e *Engine) Broadcaster() *Broadcaster { return e.bc }
|
||||
|
||||
// emitActivity persists one captured step AND fans it out to live subscribers,
|
||||
// from a single point so storage and the SSE stream never diverge.
|
||||
func (e *Engine) emitActivity(t *Task, r db.Activity) {
|
||||
func (e *Engine) emitActivity(t *Task, r db.Activity) db.Activity {
|
||||
id, err := e.appendActivity(t, r)
|
||||
if err != nil {
|
||||
// NO LONGER SILENT: dropping a record breaks command↔result pairing in the
|
||||
@@ -340,7 +599,7 @@ func (e *Engine) emitActivity(t *Task, r db.Activity) {
|
||||
log.Printf("[activity] task %s 丢弃活动记录(该任务累计第 %d 条) worker=%s kind=%s tool=%s tuid=%s reason=%s summary=%q: %v%s",
|
||||
t.ID, n, r.Worker, r.Kind, r.Tool, r.ToolUseID, dropReason(err), preview(r.Summary, 80), err, diag)
|
||||
e.touch(t.ID)
|
||||
return
|
||||
return r
|
||||
}
|
||||
r.ID = id
|
||||
if r.CreatedAt.IsZero() {
|
||||
@@ -348,6 +607,7 @@ func (e *Engine) emitActivity(t *Task, r db.Activity) {
|
||||
}
|
||||
e.bc.Publish(t.ID, r)
|
||||
e.touch(t.ID)
|
||||
return r
|
||||
}
|
||||
|
||||
// appendActivity persists one activity row, retrying briefly on write failure.
|
||||
@@ -390,6 +650,15 @@ func (e *Engine) snapshot() (*agent.Planner, *agent.Worker) {
|
||||
// Called once at startup before any task loop runs, so no lock is needed on reads.
|
||||
func (e *Engine) SetAgentResolver(fn func(t *Task) (*agent.Planner, *agent.Worker)) {
|
||||
e.resolve = fn
|
||||
e.resolveAuthoritative = false
|
||||
}
|
||||
|
||||
// SetAuthoritativeAgentResolver installs a resolver whose nil result must not
|
||||
// fall through to the global provider. Task-level failover chains use this so a
|
||||
// fully exhausted chain cannot silently bypass its configured boundary.
|
||||
func (e *Engine) SetAuthoritativeAgentResolver(fn func(t *Task) (*agent.Planner, *agent.Worker)) {
|
||||
e.resolve = fn
|
||||
e.resolveAuthoritative = true
|
||||
}
|
||||
|
||||
// snapshotFor returns the planner/worker a task should run on: the task's dedicated
|
||||
@@ -397,7 +666,8 @@ func (e *Engine) SetAgentResolver(fn func(t *Task) (*agent.Planner, *agent.Worke
|
||||
// pair. The resolver returning nil means "no override" (fall back to global).
|
||||
func (e *Engine) snapshotFor(t *Task) (*agent.Planner, *agent.Worker) {
|
||||
if e.resolve != nil {
|
||||
if p, w := e.resolve(t); p != nil && w != nil {
|
||||
p, w := e.resolve(t)
|
||||
if (p != nil && w != nil) || e.resolveAuthoritative {
|
||||
return p, w
|
||||
}
|
||||
}
|
||||
@@ -410,6 +680,14 @@ func (e *Engine) Ready() bool {
|
||||
return p != nil && w != nil
|
||||
}
|
||||
|
||||
// ReadyFor reports whether a specific task can resolve a planner/worker pair.
|
||||
// An explicit task profile chain can be runnable even when no global default
|
||||
// provider is configured, so task status must not rely on Ready alone.
|
||||
func (e *Engine) ReadyFor(t *Task) bool {
|
||||
p, w := e.snapshotFor(t)
|
||||
return p != nil && w != nil
|
||||
}
|
||||
|
||||
// Mode reports "llm" when ready, else "idle".
|
||||
func (e *Engine) Mode() string {
|
||||
if e.Ready() {
|
||||
@@ -422,15 +700,24 @@ func (e *Engine) Mode() string {
|
||||
// but no-op until an LLM is configured (so a task created while idle picks up
|
||||
// automatically once LLM is set from the UI).
|
||||
func (e *Engine) Run(ctx context.Context, t *Task) {
|
||||
workers := e.m.Workers()
|
||||
e.deleteMu.RLock()
|
||||
if e.IsDeleting(t.ID) {
|
||||
e.deleteMu.RUnlock()
|
||||
return
|
||||
}
|
||||
if _, loaded := e.started.LoadOrStore(t.ID, true); loaded {
|
||||
e.deleteMu.RUnlock()
|
||||
t.Notify() // already running — just nudge a planning round
|
||||
return
|
||||
}
|
||||
rt := e.registerTaskRoutines(ctx, t.ID, 1+workers)
|
||||
e.deleteMu.RUnlock()
|
||||
e.touch(t.ID)
|
||||
go e.plannerLoop(ctx, t)
|
||||
workers := e.m.Workers()
|
||||
runTaskRoutine(rt, func(loopCtx context.Context) { e.plannerLoop(loopCtx, t) })
|
||||
for i := 0; i < workers; i++ {
|
||||
go e.workerLoop(ctx, t, fmt.Sprintf("work#%d", i+1))
|
||||
name := fmt.Sprintf("work#%d", i+1)
|
||||
runTaskRoutine(rt, func(loopCtx context.Context) { e.workerLoop(loopCtx, t, name) })
|
||||
}
|
||||
e.startDeadlineCoordinator(ctx, t) // 任务级超时定时器(仅 timeout>0;去重)
|
||||
// 仅在「完全没有活动意图(open+running)」时才 kick 首轮规划。带种子意图的任务:种子已
|
||||
@@ -491,10 +778,13 @@ func (e *Engine) plannerLoop(ctx context.Context, t *Task) {
|
||||
if e.IsPaused(t.ID) {
|
||||
return // user-paused: don't plan
|
||||
}
|
||||
if e.IsDeleting(t.ID) {
|
||||
return
|
||||
}
|
||||
// terminal task (goals all met → done, or failed): the run is over. A
|
||||
// resume/nudge — e.g. auto-resume of the active task on restart — must NOT
|
||||
// re-plan (it would burn an LLM round and re-confirm a settled result).
|
||||
if isTerminalStatus(t.Status) {
|
||||
if isTerminalStatus(t.lifecycleSnapshot().Status) {
|
||||
return
|
||||
}
|
||||
// 任务级超时收尾中:丢弃普通唤醒——worker 收尾写回、Resume 的 Notify 都不再
|
||||
@@ -502,10 +792,17 @@ func (e *Engine) plannerLoop(ctx context.Context, t *Task) {
|
||||
if e.isSettling(t.ID) {
|
||||
return
|
||||
}
|
||||
if !e.beginTaskOperation(t.ID) {
|
||||
return
|
||||
}
|
||||
defer e.decInflight(t.ID)
|
||||
e.stampFirstRun(t) // 首次真正规划 → 盖 first_run_at + 算 deadline(仅带 timeout 的任务)
|
||||
e.touch(t.ID)
|
||||
emit := func(r db.Activity) { e.emitActivity(t, r) }
|
||||
ectx := e.clockCtx(e.execContextFor(ctx, t.ID), t, false) // cancellable by Pause; 带任务 deadline
|
||||
if ectx.Err() != nil || e.IsDeleting(t.ID) {
|
||||
return
|
||||
}
|
||||
log.Printf("[planner] task %s 规划中…(%s 触发)", t.ID, src)
|
||||
// round marker: each Plan() is one planner round; emit a boundary so the
|
||||
// UI can separate rounds in the transcript (kind='round').
|
||||
@@ -514,10 +811,10 @@ func (e *Engine) plannerLoop(ctx context.Context, t *Task) {
|
||||
// what fired this round (worker done / finding; may be several — debounce
|
||||
// coalesces a burst; empty for time/heartbeat wakes).
|
||||
triggers := t.drainTriggers()
|
||||
e.incInflight(t.ID)
|
||||
taskIDInt, _ := strconv.ParseInt(t.ID, 10, 64)
|
||||
e.BeginLLMCall(t.ID)
|
||||
met, reason, err := planner.Plan(ectx, taskIDInt, e.m.assets, t.Store, t.Goal, triggers, emit)
|
||||
e.decInflight(t.ID)
|
||||
e.EndLLMCall(t.ID)
|
||||
switch {
|
||||
case err != nil && ectx.Err() == nil:
|
||||
log.Printf("[planner] task %s 规划出错: %v", t.ID, err)
|
||||
@@ -530,7 +827,7 @@ func (e *Engine) plannerLoop(ctx context.Context, t *Task) {
|
||||
// 任务已判完成 → 立刻取消在跑的 worker:它们手头的意图跑出来也没意义了。
|
||||
// 下一轮 worker 循环撞终态门就不再领新意图;被取消的这批走下方"任务已完成"分支
|
||||
// 归为 stopped(而非 blocked)。
|
||||
e.cancelExec(t.ID)
|
||||
e.cancelExec(t.ID, agent.AbortGoalMet)
|
||||
default:
|
||||
log.Printf("[planner] task %s 规划完成", t.ID)
|
||||
}
|
||||
@@ -572,6 +869,9 @@ func (e *Engine) workerLoop(ctx context.Context, t *Task, name string) {
|
||||
}
|
||||
continue // user-paused: don't claim/execute intents
|
||||
}
|
||||
if e.IsDeleting(t.ID) {
|
||||
return
|
||||
}
|
||||
if e.isSettling(t.ID) {
|
||||
if sleepCtx(ctx, 1000*time.Millisecond) {
|
||||
return
|
||||
@@ -584,105 +884,191 @@ func (e *Engine) workerLoop(ctx context.Context, t *Task, name string) {
|
||||
}
|
||||
continue // 任务已终态(done/failed/timeout):停止领取遗留意图,别在完成后空跑 frontier
|
||||
}
|
||||
intent := e.claimNext(t, name)
|
||||
if intent == nil {
|
||||
if sleepCtx(ctx, 800*time.Millisecond) {
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
log.Printf("[worker %s] task %s 领取意图 #%d", name, t.ID, intent.ID)
|
||||
e.stampFirstRun(t) // 首次真正执行 → 盖 first_run_at + 算 deadline(仅带 timeout 的任务)
|
||||
e.touch(t.ID)
|
||||
emit := func(r db.Activity) { e.emitActivity(t, r) }
|
||||
ectx := e.clockCtx(e.execContextFor(ctx, t.ID), t, false) // cancellable by Pause; 带任务 deadline
|
||||
// per-work child context so the planner's kill_work can stop just this work.
|
||||
workCtx, workCancel := context.WithCancel(ectx)
|
||||
e.registerWork(intent.ID, workCancel)
|
||||
// wrap the guard hooks so steer_work can inject a mid-run course-correction
|
||||
// for THIS intent (drained before the worker's next tool call).
|
||||
iid := intent.ID
|
||||
taskEmit := func(a db.Activity) {
|
||||
nid := iid
|
||||
a.NodeID, a.Worker = &nid, name
|
||||
emit(a)
|
||||
}
|
||||
workCtx = intercept.WithTaskContext(workCtx, t.ID, fmt.Sprintf("%s · #%d", name, iid), taskEmit)
|
||||
hooks := steerHooks{inner: t.Guard.Hooks(), drain: func() (string, bool) { return e.drainSteer(iid) }}
|
||||
wTaskID, _ := strconv.ParseInt(t.ID, 10, 64)
|
||||
e.incInflight(t.ID) // 计入在跑,供收尾时序 drain 等待
|
||||
reason, wrote, err := worker.Execute(workCtx, name, wTaskID, e.m.assets, t.Store, intent, hooks, emit, e.m.enrich, t.NotifyFinding)
|
||||
// model_error 收场 → 额外重跑几次(退避后再试)。仅在意图仍属本 work、任务
|
||||
// 未暂停/未终止/未取消【且未进入收尾】时重试;否则让位给对应分支处理(收尾期不
|
||||
// 再重试,避免退避挤占其他 worker 的优雅收尾窗口)。
|
||||
for attempt := 1; attempt <= modelErrorRetries &&
|
||||
reason == harness.ReasonModelError &&
|
||||
workCtx.Err() == nil && ectx.Err() == nil && !e.IsPaused(t.ID) && !e.isSettling(t.ID); attempt++ {
|
||||
log.Printf("[worker %s] task %s 意图 #%d model_error 收场,%v 后重试 (%d/%d)",
|
||||
name, t.ID, intent.ID, modelErrorRetryBackoff, attempt, modelErrorRetries)
|
||||
if sleepCtx(workCtx, modelErrorRetryBackoff) {
|
||||
break // 退避期间被取消(终止/暂停)→ 交给下方分支处理
|
||||
}
|
||||
reason, wrote, err = worker.Execute(workCtx, name, wTaskID, e.m.assets, t.Store, intent, hooks, emit, e.m.enrich, t.NotifyFinding)
|
||||
if !e.beginTaskOperation(t.ID) {
|
||||
return
|
||||
}
|
||||
claimed := e.runWorkerStep(ctx, t, name, worker)
|
||||
e.decInflight(t.ID)
|
||||
// CAPTURE kill state BEFORE unregisterWork cancels workCtx. kill = this work's
|
||||
// ctx was cancelled (planner kill_work) while the TASK ctx kept running; a
|
||||
// pause cancels the task ctx (ectx) instead. Checking workCtx.Err() AFTER
|
||||
// unregister would always be true (unregister cancels it) → every completed
|
||||
// work would be wrongly marked stopped.
|
||||
killed := workCtx.Err() != nil && ectx.Err() == nil
|
||||
e.unregisterWork(intent.ID)
|
||||
// if a pause cancelled this run mid-flight, return the intent to the frontier
|
||||
// so it is re-claimed (and re-run from scratch) on resume — not marked done.
|
||||
if ectx.Err() != nil && e.IsPaused(t.ID) {
|
||||
_ = t.Store.SetIntentState(intent.ID, "open")
|
||||
continue
|
||||
if !claimed && sleepCtx(ctx, 800*time.Millisecond) {
|
||||
return
|
||||
}
|
||||
// 任务超时收尾的硬兜底 cancel(非 pause、非 kill)取消了本 run → 归为 exhausted(已收尾),
|
||||
// 不要误标 blocked。此时 worker 通常已在 settlement 阶段把结果写回。
|
||||
if ectx.Err() != nil && e.isSettling(t.ID) {
|
||||
_ = t.Store.SetIntentState(intent.ID, "exhausted")
|
||||
log.Printf("[worker %s] task %s 意图 #%d 因任务超时收尾结束(exhausted),写回 %s", name, t.ID, intent.ID, wrote)
|
||||
e.touch(t.ID)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// runWorkerStep claims and fully settles at most one intent. Its caller holds one
|
||||
// task-operation admission for the whole claim/execute/state-write sequence so a
|
||||
// delete cannot observe quiescence between the LLM return and the final DB writes.
|
||||
func (e *Engine) runWorkerStep(ctx context.Context, t *Task, name string, worker *agent.Worker) bool {
|
||||
intent := e.claimNext(t, name)
|
||||
if intent == nil {
|
||||
return false
|
||||
}
|
||||
log.Printf("[worker %s] task %s 领取意图 #%d", name, t.ID, intent.ID)
|
||||
e.stampFirstRun(t) // 首次真正执行 → 盖 first_run_at + 算 deadline(仅带 timeout 的任务)
|
||||
e.touch(t.ID)
|
||||
emit := func(r db.Activity) { e.emitActivity(t, r) }
|
||||
ectx := e.clockCtx(e.execContextFor(ctx, t.ID), t, false) // cancellable by Pause; 带任务 deadline
|
||||
if ectx.Err() != nil || e.IsDeleting(t.ID) {
|
||||
if err := transitionIntentState(t.Store, intent.ID, "running", "open"); err != nil {
|
||||
log.Printf("[worker %s] task %s 意图 #%d 领取后回退失败: %v", name, t.ID, intent.ID, err)
|
||||
}
|
||||
// 任务已判完成(done via 常规路径)→ 上面 cancelExec 取消了本 run。意图结果已无意义,
|
||||
// 标 stopped(不是 blocked),别污染已完成任务的意图状态。
|
||||
if ectx.Err() != nil && isTerminalStatus(e.m.TaskStatus(t.ID)) {
|
||||
_ = t.Store.SetIntentState(intent.ID, "stopped")
|
||||
log.Printf("[worker %s] task %s 意图 #%d 因任务已完成而取消(stopped)", name, t.ID, intent.ID)
|
||||
e.touch(t.ID)
|
||||
continue
|
||||
return true
|
||||
}
|
||||
// per-work child context so the planner's kill_work can stop just this work.
|
||||
workCtx, workCancel := context.WithCancelCause(ectx)
|
||||
e.registerWork(intent.ID, workCancel)
|
||||
// wrap the guard hooks so steer_work can inject a mid-run course-correction
|
||||
// for THIS intent (drained before the worker's next tool call).
|
||||
iid := intent.ID
|
||||
taskEmit := func(a db.Activity) {
|
||||
nid := iid
|
||||
a.NodeID, a.Worker = &nid, name
|
||||
emit(a)
|
||||
}
|
||||
workCtx = intercept.WithTaskContext(workCtx, t.ID, fmt.Sprintf("%s · #%d", name, iid), taskEmit)
|
||||
hooks := steerHooks{inner: t.Guard.Hooks(), drain: func() (string, bool) { return e.drainSteer(iid) }}
|
||||
wTaskID, _ := strconv.ParseInt(t.ID, 10, 64)
|
||||
e.BeginLLMCall(t.ID)
|
||||
reason, wrote, err := worker.Execute(workCtx, name, wTaskID, e.m.assets, t.Store, intent, hooks, emit, e.m.enrich, t.NotifyFinding)
|
||||
e.EndLLMCall(t.ID)
|
||||
// model_error 收场 → 额外重跑几次(退避后再试)。仅在意图仍属本 work、任务
|
||||
// 未暂停/未终止/未取消【且未进入收尾】时重试;否则让位给对应分支处理(收尾期不
|
||||
// 再重试,避免退避挤占其他 worker 的优雅收尾窗口)。
|
||||
for attempt := 1; attempt <= modelErrorRetries &&
|
||||
retryableWorkerModelError(reason, err) &&
|
||||
workCtx.Err() == nil && ectx.Err() == nil && !e.IsPaused(t.ID) && !e.isSettling(t.ID); attempt++ {
|
||||
log.Printf("[worker %s] task %s 意图 #%d model_error 收场,%v 后重试 (%d/%d)",
|
||||
name, t.ID, intent.ID, modelErrorRetryBackoff, attempt, modelErrorRetries)
|
||||
if sleepCtx(workCtx, modelErrorRetryBackoff) {
|
||||
break // 退避期间被取消(终止/暂停)→ 交给下方分支处理
|
||||
}
|
||||
// killed by the planner: mark stopped (don't write back results, don't auto-reclaim).
|
||||
if killed {
|
||||
_ = t.Store.SetIntentState(intent.ID, "stopped")
|
||||
log.Printf("[worker %s] task %s 意图 #%d 被终止(stopped)", name, t.ID, intent.ID)
|
||||
e.touch(t.ID)
|
||||
t.Notify()
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
log.Printf("[worker %s] intent %d: %v", name, intent.ID, err)
|
||||
}
|
||||
// terminal分流:撞步数上限 ≠ 完成。max_turns→exhausted(规划者据此知道这个方向
|
||||
// 试过但没真正做完、需换角度,而非当成已覆盖永久跳过);出错→blocked;正常→done。
|
||||
state := "done"
|
||||
e.BeginLLMCall(t.ID)
|
||||
reason, wrote, err = worker.Execute(workCtx, name, wTaskID, e.m.assets, t.Store, intent, hooks, emit, e.m.enrich, t.NotifyFinding)
|
||||
e.EndLLMCall(t.ID)
|
||||
}
|
||||
// Capture kill state before detachWork cancels workCtx. kill = this work's
|
||||
// ctx was cancelled (planner kill_work) while the TASK ctx kept running; a
|
||||
// pause cancels the task ctx (ectx) instead. Checking workCtx.Err() AFTER
|
||||
// unregister would always be true (unregister cancels it) → every completed
|
||||
// work would be wrongly marked stopped.
|
||||
workCause := context.Cause(workCtx)
|
||||
killed := workCtx.Err() != nil && ectx.Err() == nil
|
||||
action, completeWork := e.detachWork(intent.ID)
|
||||
// A caller may stop waiting and release its in-memory reservation before the
|
||||
// agent honors cancellation. The named context cause remains authoritative and
|
||||
// still settles the stopped run into a recoverable state.
|
||||
if action == "" {
|
||||
switch {
|
||||
case err != nil:
|
||||
state = "blocked"
|
||||
case reason == harness.ReasonMaxTurns:
|
||||
state = "exhausted"
|
||||
log.Printf("[worker %s] intent %d 撞步数上限(exhausted),本次写回 %s", name, intent.ID, wrote)
|
||||
case reason == harness.ReasonTimeout:
|
||||
state = "exhausted"
|
||||
log.Printf("[worker %s] intent %d 运行超时(exhausted),收尾后写回 %s", name, intent.ID, wrote)
|
||||
case errors.Is(workCause, agent.AbortWorkPausedByUser):
|
||||
action = "pause"
|
||||
case errors.Is(workCause, agent.AbortWorkCancelledByUser):
|
||||
action = "cancel"
|
||||
}
|
||||
_ = t.Store.SetIntentState(intent.ID, state)
|
||||
log.Printf("[worker %s] task %s 意图 #%d 结束: %s (写回 %s)", name, t.ID, intent.ID, state, wrote)
|
||||
}
|
||||
var controlErr error
|
||||
defer func() { completeWork(controlErr) }()
|
||||
if action == "pause" {
|
||||
controlErr = transitionIntentState(t.Store, intent.ID, "running", "paused")
|
||||
if controlErr != nil {
|
||||
log.Printf("[worker %s] task %s 意图 #%d 暂停状态落库失败: %v", name, t.ID, intent.ID, controlErr)
|
||||
return true
|
||||
}
|
||||
log.Printf("[worker %s] task %s 意图 #%d 已暂停", name, t.ID, intent.ID)
|
||||
e.touch(t.ID)
|
||||
t.NotifyDone(intent.ID) // results changed the graph -> wake the planner (with the just-finished intent id)
|
||||
return true
|
||||
}
|
||||
if action == "cancel" {
|
||||
// Park the stopped run in paused before handing cleanup to the API. If the
|
||||
// request disconnects after cancellation, the intent remains recoverable and
|
||||
// a later cancel can finish cleanup instead of leaving a phantom running row.
|
||||
controlErr = transitionIntentState(t.Store, intent.ID, "running", "paused")
|
||||
if controlErr != nil {
|
||||
log.Printf("[worker %s] task %s 意图 #%d 取消栅栏落库失败: %v", name, t.ID, intent.ID, controlErr)
|
||||
return true
|
||||
}
|
||||
log.Printf("[worker %s] task %s 意图 #%d 已停止,等待取消清理", name, t.ID, intent.ID)
|
||||
e.touch(t.ID)
|
||||
return true
|
||||
}
|
||||
// if a pause cancelled this run mid-flight, return the intent to the frontier
|
||||
// so it is re-claimed on resume — the worker will resume the prior LLM
|
||||
// conversation from its transcript instead of restarting from scratch.
|
||||
if ectx.Err() != nil && taskExecutionPaused(context.Cause(ectx)) {
|
||||
if err := transitionIntentState(t.Store, intent.ID, "running", "open"); err != nil {
|
||||
log.Printf("[worker %s] task %s 意图 #%d 任务暂停回退失败: %v", name, t.ID, intent.ID, err)
|
||||
}
|
||||
return true
|
||||
}
|
||||
// 任务超时收尾的硬兜底 cancel(非 pause、非 kill)取消了本 run → 归为 exhausted(已收尾),
|
||||
// 不要误标 blocked。此时 worker 通常已在 settlement 阶段把结果写回。
|
||||
if ectx.Err() != nil && e.isSettling(t.ID) {
|
||||
if err := transitionIntentState(t.Store, intent.ID, "running", "exhausted"); err != nil {
|
||||
log.Printf("[worker %s] task %s 意图 #%d 超时收尾状态落库失败: %v", name, t.ID, intent.ID, err)
|
||||
}
|
||||
log.Printf("[worker %s] task %s 意图 #%d 因任务超时收尾结束(exhausted),写回 %s", name, t.ID, intent.ID, wrote)
|
||||
e.touch(t.ID)
|
||||
return true
|
||||
}
|
||||
// 任务已判完成(done via 常规路径)→ 上面 cancelExec 取消了本 run。意图结果已无意义,
|
||||
// 标 stopped(不是 blocked),别污染已完成任务的意图状态。
|
||||
if ectx.Err() != nil && isTerminalStatus(e.m.TaskStatus(t.ID)) {
|
||||
if err := transitionIntentState(t.Store, intent.ID, "running", "stopped"); err != nil {
|
||||
log.Printf("[worker %s] task %s 意图 #%d 终态停止落库失败: %v", name, t.ID, intent.ID, err)
|
||||
}
|
||||
log.Printf("[worker %s] task %s 意图 #%d 因任务已完成而取消(stopped)", name, t.ID, intent.ID)
|
||||
e.touch(t.ID)
|
||||
return true
|
||||
}
|
||||
// killed by the planner: mark stopped (don't write back results, don't auto-reclaim).
|
||||
if killed {
|
||||
if err := transitionIntentState(t.Store, intent.ID, "running", "stopped"); err != nil {
|
||||
log.Printf("[worker %s] task %s 意图 #%d planner 停止落库失败: %v", name, t.ID, intent.ID, err)
|
||||
}
|
||||
log.Printf("[worker %s] task %s 意图 #%d 被终止(stopped)", name, t.ID, intent.ID)
|
||||
e.touch(t.ID)
|
||||
t.Notify()
|
||||
return true
|
||||
}
|
||||
if err != nil {
|
||||
log.Printf("[worker %s] intent %d: %v", name, intent.ID, err)
|
||||
}
|
||||
// terminal分流:撞步数上限 ≠ 完成。max_turns→exhausted(规划者据此知道这个方向
|
||||
// 试过但没真正做完、需换角度,而非当成已覆盖永久跳过);出错→blocked;正常→done。
|
||||
state := "done"
|
||||
switch {
|
||||
case err != nil:
|
||||
state = "blocked"
|
||||
case reason == harness.ReasonMaxTurns:
|
||||
state = "exhausted"
|
||||
log.Printf("[worker %s] intent %d 撞步数上限(exhausted),本次写回 %s", name, intent.ID, wrote)
|
||||
case reason == harness.ReasonTimeout:
|
||||
state = "exhausted"
|
||||
log.Printf("[worker %s] intent %d 运行超时(exhausted),收尾后写回 %s", name, intent.ID, wrote)
|
||||
}
|
||||
if state == "blocked" && isTaskLLMChainExhausted(err) {
|
||||
_ = t.Store.SetIntentBlockedReason(intent.ID, db.IntentBlockedLLMQuota)
|
||||
} else {
|
||||
if stateErr := transitionIntentState(t.Store, intent.ID, "running", state); stateErr != nil {
|
||||
log.Printf("[worker %s] task %s 意图 #%d 终态 %s 落库失败: %v", name, t.ID, intent.ID, state, stateErr)
|
||||
}
|
||||
}
|
||||
log.Printf("[worker %s] task %s 意图 #%d 结束: %s (写回 %s)", name, t.ID, intent.ID, state, wrote)
|
||||
e.touch(t.ID)
|
||||
t.NotifyDone(intent.ID) // results changed the graph -> wake the planner (with the just-finished intent id)
|
||||
return true
|
||||
}
|
||||
|
||||
func taskExecutionPaused(cause error) bool {
|
||||
var abort *agent.AbortCause
|
||||
if !errors.As(cause, &abort) {
|
||||
return false
|
||||
}
|
||||
switch abort.Code {
|
||||
case "paused_by_user", "paused_by_orchestrator", "paused_on_reload", "paused_race_guard",
|
||||
"queued_for_admission", "llm_unavailable_queued", "task_deleted":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Autumn-27/artex/agent"
|
||||
)
|
||||
|
||||
func TestPauseCancelsWithNamedCause(t *testing.T) {
|
||||
e := NewEngine(nil)
|
||||
execCtx := e.execContextFor(context.Background(), "t1")
|
||||
workCtx, workCancel := context.WithCancelCause(execCtx)
|
||||
defer workCancel(nil)
|
||||
e.Pause("t1", agent.AbortPausedByUser)
|
||||
if code, _, _, ok := agent.AbortReason(workCtx); !ok || code != "paused_by_user" {
|
||||
t.Fatalf("code=%q ok=%v, want paused_by_user", code, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKillWorkCancelsOnlyThatWorkWithNamedCause(t *testing.T) {
|
||||
e := NewEngine(nil)
|
||||
execCtx := e.execContextFor(context.Background(), "t1")
|
||||
workCtx, workCancel := context.WithCancelCause(execCtx)
|
||||
e.registerWork(42, workCancel)
|
||||
if err := e.KillWork(42); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if code, _, _, ok := agent.AbortReason(workCtx); !ok || code != "killed_by_planner" {
|
||||
t.Fatalf("code=%q ok=%v, want killed_by_planner", code, ok)
|
||||
}
|
||||
if execCtx.Err() != nil {
|
||||
t.Fatal("kill_work cancelled the entire task context")
|
||||
}
|
||||
_, complete := e.detachWork(42)
|
||||
complete(nil)
|
||||
}
|
||||
|
||||
func TestControlWorkCarriesUserActionCause(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
action string
|
||||
code string
|
||||
}{
|
||||
{action: "pause", code: "work_paused_by_user"},
|
||||
{action: "cancel", code: "work_cancelled_by_user"},
|
||||
} {
|
||||
t.Run(tc.action, func(t *testing.T) {
|
||||
e := NewEngine(nil)
|
||||
ctx, cancel := context.WithCancelCause(context.Background())
|
||||
e.registerWork(42, cancel)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- e.ControlWork(context.Background(), 42, tc.action) }()
|
||||
<-ctx.Done()
|
||||
if code, _, _, _ := agent.AbortReason(ctx); code != tc.code {
|
||||
t.Fatalf("code=%q, want %q", code, tc.code)
|
||||
}
|
||||
action, complete := e.detachWork(42)
|
||||
if action != tc.action {
|
||||
t.Fatalf("action=%q, want %q", action, tc.action)
|
||||
}
|
||||
complete(nil)
|
||||
if err := <-done; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestControlWorkReturnsSettlementError(t *testing.T) {
|
||||
e := NewEngine(nil)
|
||||
ctx, cancel := context.WithCancelCause(context.Background())
|
||||
e.registerWork(42, cancel)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- e.ControlWork(context.Background(), 42, "pause") }()
|
||||
<-ctx.Done()
|
||||
_, complete := e.detachWork(42)
|
||||
want := errors.New("pause persistence failed")
|
||||
complete(want)
|
||||
if err := <-done; !errors.Is(err, want) {
|
||||
t.Fatalf("ControlWork error=%v, want %v", err, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestControlWorkWaitHonorsContextAndReleasesReservation(t *testing.T) {
|
||||
e := NewEngine(nil)
|
||||
_, cancel := context.WithCancelCause(context.Background())
|
||||
e.registerWork(42, cancel)
|
||||
waitCtx, stop := context.WithTimeout(context.Background(), 20*time.Millisecond)
|
||||
defer stop()
|
||||
if err := e.ControlWork(waitCtx, 42, "pause"); !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("ControlWork error=%v, want deadline exceeded", err)
|
||||
}
|
||||
|
||||
e.workMu.Lock()
|
||||
run := e.work[42]
|
||||
if run == nil || run.action != "" {
|
||||
t.Fatalf("timed-out control left reservation: %+v", run)
|
||||
}
|
||||
e.workMu.Unlock()
|
||||
_, complete := e.detachWork(42)
|
||||
complete(nil)
|
||||
}
|
||||
|
||||
func TestControlWorkRejectsConcurrentController(t *testing.T) {
|
||||
e := NewEngine(nil)
|
||||
ctx, cancel := context.WithCancelCause(context.Background())
|
||||
e.registerWork(42, cancel)
|
||||
first := make(chan error, 1)
|
||||
go func() { first <- e.ControlWork(context.Background(), 42, "pause") }()
|
||||
<-ctx.Done()
|
||||
if err := e.ControlWork(context.Background(), 42, "cancel"); !errors.Is(err, errWorkControlConflict) {
|
||||
t.Fatalf("second control error=%v, want conflict", err)
|
||||
}
|
||||
action, complete := e.detachWork(42)
|
||||
if action != "pause" {
|
||||
t.Fatalf("winning action=%q, want pause", action)
|
||||
}
|
||||
complete(nil)
|
||||
if err := <-first; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSettleDrainAndGoalMetCausesAreDistinct(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
cause *agent.AbortCause
|
||||
code string
|
||||
}{
|
||||
{agent.AbortSettleDrainTimeout, "settle_drain_timeout"},
|
||||
{agent.AbortGoalMet, "goal_met"},
|
||||
} {
|
||||
e := NewEngine(nil)
|
||||
ctx := e.execContextFor(context.Background(), "t1")
|
||||
e.cancelExec("t1", tc.cause)
|
||||
if code, _, _, _ := agent.AbortReason(ctx); code != tc.code {
|
||||
t.Fatalf("code=%q, want %q", code, tc.code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecContextWhilePausedCarriesRaceGuardCause(t *testing.T) {
|
||||
e := NewEngine(nil)
|
||||
e.Pause("t1", agent.AbortPausedByUser)
|
||||
ctx := e.execContextFor(context.Background(), "t1")
|
||||
if ctx.Err() == nil {
|
||||
t.Fatal("paused task received a live execution context")
|
||||
}
|
||||
if code, _, _, _ := agent.AbortReason(ctx); code != "paused_race_guard" {
|
||||
t.Fatalf("code=%q, want paused_race_guard", code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResumeYieldsFreshContext(t *testing.T) {
|
||||
e := NewEngine(nil)
|
||||
old := e.execContextFor(context.Background(), "t1")
|
||||
e.Pause("t1", agent.AbortPausedByUser)
|
||||
if old.Err() == nil {
|
||||
t.Fatal("Pause did not cancel old context")
|
||||
}
|
||||
e.paused.Delete("t1")
|
||||
if fresh := e.execContextFor(context.Background(), "t1"); fresh.Err() != nil {
|
||||
t.Fatal("resume did not create a fresh execution context")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTaskPauseCauseSurvivesImmediateResume(t *testing.T) {
|
||||
e := NewEngine(nil)
|
||||
task := &Task{ID: "t1", notify: make(chan struct{}, 1)}
|
||||
old := e.execContextFor(context.Background(), task.ID)
|
||||
e.Pause(task.ID, agent.AbortPausedByUser)
|
||||
e.Resume(task)
|
||||
if e.IsPaused(task.ID) {
|
||||
t.Fatal("resume did not clear the task pause flag")
|
||||
}
|
||||
if old.Err() == nil || !taskExecutionPaused(context.Cause(old)) {
|
||||
t.Fatalf("old execution cause=%v was not retained as a task pause", context.Cause(old))
|
||||
}
|
||||
if taskExecutionPaused(agent.AbortGoalMet) {
|
||||
t.Fatal("goal completion was misclassified as a task pause")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteCancellationReturnsRunningIntentToFrontier(t *testing.T) {
|
||||
if !taskExecutionPaused(agent.AbortTaskDeleted) {
|
||||
t.Fatal("delete cancellation must settle a running intent as open so AbortDelete can resume it")
|
||||
}
|
||||
if taskExecutionPaused(agent.AbortGoalMet) {
|
||||
t.Fatal("terminal task cancellation must not reopen completed work")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResumeDoesNotClearDeleteBarrier(t *testing.T) {
|
||||
e := NewEngine(nil)
|
||||
task := &Task{ID: "t1", notify: make(chan struct{}, 1)}
|
||||
e.paused.Store(task.ID, true)
|
||||
e.deleting.Store(task.ID, true)
|
||||
e.Resume(task)
|
||||
if !e.IsPaused(task.ID) {
|
||||
t.Fatal("resume cleared the pause barrier owned by task deletion")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBootstrapAdmissionClearsQueuedPauseBarrier(t *testing.T) {
|
||||
e := NewEngine(nil)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
task := &Task{ID: "t1", notify: make(chan struct{}, 1)}
|
||||
s := &Server{engine: e, ctx: ctx}
|
||||
e.Pause(task.ID, agent.AbortPausedByUser)
|
||||
|
||||
s.startAdmittedTask(task, "bootstrap")
|
||||
if e.IsPaused(task.ID) {
|
||||
t.Fatal("bootstrap admission left the queued task behind the pause barrier")
|
||||
}
|
||||
}
|
||||
+59
-11
@@ -43,7 +43,20 @@ func (e *Engine) inflightCounter(taskID string) *int64 {
|
||||
v, _ := e.inflight.LoadOrStore(taskID, new(int64))
|
||||
return v.(*int64)
|
||||
}
|
||||
func (e *Engine) incInflight(taskID string) { atomic.AddInt64(e.inflightCounter(taskID), 1) }
|
||||
|
||||
// beginTaskOperation atomically registers a task-owned operation unless deletion
|
||||
// has already installed its barrier. The delete handler can therefore wait for
|
||||
// inflight==0 without a check-then-start race recreating files after cleanup.
|
||||
func (e *Engine) beginTaskOperation(taskID string) bool {
|
||||
e.deleteMu.RLock()
|
||||
defer e.deleteMu.RUnlock()
|
||||
if e.IsDeleting(taskID) {
|
||||
return false
|
||||
}
|
||||
atomic.AddInt64(e.inflightCounter(taskID), 1)
|
||||
return true
|
||||
}
|
||||
|
||||
func (e *Engine) decInflight(taskID string) { atomic.AddInt64(e.inflightCounter(taskID), -1) }
|
||||
func (e *Engine) inflightCount(taskID string) int64 {
|
||||
return atomic.LoadInt64(e.inflightCounter(taskID))
|
||||
@@ -58,13 +71,26 @@ func (e *Engine) taskDeadline(t *Task) int64 {
|
||||
if v, ok := e.deadline.Load(t.ID); ok {
|
||||
return v.(int64)
|
||||
}
|
||||
if t.DeadlineAt > 0 {
|
||||
e.deadline.Store(t.ID, t.DeadlineAt)
|
||||
return t.DeadlineAt
|
||||
deadlineAt := t.lifecycleSnapshot().DeadlineAt
|
||||
if deadlineAt > 0 {
|
||||
e.deadline.Store(t.ID, deadlineAt)
|
||||
return deadlineAt
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// resetTimeoutRevival clears only the per-run timeout state after PostgreSQL has
|
||||
// atomically committed timeout -> running and reset first_run_at/deadline_at. The
|
||||
// configured TimeoutSeconds remains on Task, so the next real Planner/Worker run
|
||||
// stamps a fresh full budget. coordStarted is reset because the coordinator that
|
||||
// produced the timeout has already completed (or is in its final return path).
|
||||
func (e *Engine) resetTimeoutRevival(taskID string) {
|
||||
e.settling.Delete(taskID)
|
||||
e.deadline.Delete(taskID)
|
||||
e.stamped.Delete(taskID)
|
||||
e.coordStarted.Delete(taskID)
|
||||
}
|
||||
|
||||
// stampFirstRun records first_run_at + deadline_at on the FIRST real run (LLM ready)
|
||||
// of a timeout task, once per process. No-op when the task has no timeout.
|
||||
func (e *Engine) stampFirstRun(t *Task) {
|
||||
@@ -106,10 +132,18 @@ func (e *Engine) startDeadlineCoordinator(ctx context.Context, t *Task) {
|
||||
if t == nil || t.TimeoutSeconds <= 0 {
|
||||
return
|
||||
}
|
||||
if _, loaded := e.coordStarted.LoadOrStore(t.ID, true); loaded {
|
||||
e.deleteMu.RLock()
|
||||
if e.IsDeleting(t.ID) {
|
||||
e.deleteMu.RUnlock()
|
||||
return
|
||||
}
|
||||
go e.deadlineCoordinator(ctx, t)
|
||||
if _, loaded := e.coordStarted.LoadOrStore(t.ID, true); loaded {
|
||||
e.deleteMu.RUnlock()
|
||||
return
|
||||
}
|
||||
rt := e.registerTaskRoutines(ctx, t.ID, 1)
|
||||
e.deleteMu.RUnlock()
|
||||
runTaskRoutine(rt, func(loopCtx context.Context) { e.deadlineCoordinator(loopCtx, t) })
|
||||
}
|
||||
|
||||
// deadlineCoordinator waits until the task's absolute deadline, then runs the settle
|
||||
@@ -159,7 +193,7 @@ func (e *Engine) settleTask(ctx context.Context, t *Task) {
|
||||
for e.inflightCount(t.ID) > 0 {
|
||||
if time.Now().After(hardStop) {
|
||||
log.Printf("[deadline] task %s drain 超时(%s),硬取消在跑 run", t.ID, settleDrainGrace)
|
||||
e.cancelExec(t.ID)
|
||||
e.cancelExec(t.ID, agent.AbortSettleDrainTimeout)
|
||||
_ = sleepCtx(ctx, 3*time.Second) // 给 worker 分支一点时间落库/归类
|
||||
break
|
||||
}
|
||||
@@ -170,6 +204,10 @@ func (e *Engine) settleTask(ctx context.Context, t *Task) {
|
||||
|
||||
// ⑤ 终局一轮 planner(任务超时词,最后目标判定,不产新意图)。
|
||||
met := e.runFinalPlannerRound(ctx, t)
|
||||
if !e.beginTaskOperation(t.ID) {
|
||||
return
|
||||
}
|
||||
defer e.decInflight(t.ID)
|
||||
|
||||
// ⑥ 定终态(带守卫):met → done(completed);否则 timeout。若常规路径已先落 done,
|
||||
// 守卫(SetTaskStatusGuarded)会拒绝覆盖,保留 completed 语义。
|
||||
@@ -192,6 +230,9 @@ func (e *Engine) settleTask(ctx context.Context, t *Task) {
|
||||
// task-timeout planner words (final goal judgment; no new intents). Waits for the
|
||||
// LLM to be ready (bounded by ctx) so a completable task isn't mis-judged timeout.
|
||||
func (e *Engine) runFinalPlannerRound(ctx context.Context, t *Task) (met bool) {
|
||||
if e.IsDeleting(t.ID) {
|
||||
return false
|
||||
}
|
||||
planner, _ := e.snapshotFor(t)
|
||||
for planner == nil {
|
||||
if sleepCtx(ctx, deadlinePollInterval) {
|
||||
@@ -200,17 +241,24 @@ func (e *Engine) runFinalPlannerRound(ctx context.Context, t *Task) (met bool) {
|
||||
if isTerminalStatus(e.m.TaskStatus(t.ID)) {
|
||||
return false
|
||||
}
|
||||
if e.IsDeleting(t.ID) {
|
||||
return false
|
||||
}
|
||||
planner, _ = e.snapshotFor(t)
|
||||
}
|
||||
// 独立 ctx(不挂 execCancel,避免 pause/硬 cancel 打断这最后一轮),带 Final 注入任务超时词。
|
||||
fctx := e.clockCtx(ctx, t, true)
|
||||
if !e.beginTaskOperation(t.ID) {
|
||||
return false
|
||||
}
|
||||
defer e.decInflight(t.ID)
|
||||
emit := func(r db.Activity) { e.emitActivity(t, r) }
|
||||
e.emitActivity(t, db.Activity{Worker: "planner", Kind: "round",
|
||||
Summary: fmt.Sprintf("任务超时收尾·终局判定(第 %d 轮)", e.nextPlannerRound(t.ID))})
|
||||
// 独立 ctx(不挂 execCancel,避免 pause/硬 cancel 打断这最后一轮),带 Final 注入任务超时词。
|
||||
fctx := e.clockCtx(ctx, t, true)
|
||||
e.incInflight(t.ID)
|
||||
tTaskID, _ := strconv.ParseInt(t.ID, 10, 64)
|
||||
e.BeginLLMCall(t.ID)
|
||||
met, reason, err := planner.Plan(fctx, tTaskID, e.m.assets, t.Store, t.Goal, t.drainTriggers(), emit)
|
||||
e.decInflight(t.ID)
|
||||
e.EndLLMCall(t.ID)
|
||||
if err != nil {
|
||||
log.Printf("[deadline] task %s 终局规划出错: %v", t.ID, err)
|
||||
} else if met {
|
||||
|
||||
+330
-59
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
@@ -25,25 +26,324 @@ type goalSpec struct {
|
||||
//
|
||||
// 异步(goroutine)所以调用方立即返回,两条路径行为一致:秒建任务、后台拆目标。
|
||||
func (s *Server) launchTask(t *Task, seedText string, seedFirstIntent bool) {
|
||||
if !s.engine.beginTaskOperation(t.ID) {
|
||||
return
|
||||
}
|
||||
s.seed(t, seedText)
|
||||
if seedFirstIntent {
|
||||
s.seedFirstIntent(t)
|
||||
}
|
||||
go func() {
|
||||
s.engine.emitActivity(t, db.Activity{Worker: "planner", Kind: "round",
|
||||
Summary: "第 0 轮目标拆解"})
|
||||
goals := s.createGoals(s.ctx, t, func(r db.Activity) {
|
||||
s.engine.emitActivity(t, r)
|
||||
})
|
||||
for _, g := range goals {
|
||||
summary := g.Text
|
||||
if g.VulnClass != "" {
|
||||
summary = fmt.Sprintf("[%s] %s", g.VulnClass, g.Text)
|
||||
}
|
||||
s.engine.emitActivity(t, db.Activity{Worker: "planner", Kind: "text", Summary: summary})
|
||||
s.engine.decInflight(t.ID)
|
||||
if _, err := s.admitTask(t, "bootstrap"); err != nil {
|
||||
log.Printf("[concurrency] task %s 启动失败: %v", t.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) startTaskEngine(t *Task) {
|
||||
ctx := s.engine.execContextFor(s.ctx, t.ID)
|
||||
if ctx.Err() != nil || s.engine.IsDeleting(t.ID) {
|
||||
return
|
||||
}
|
||||
s.engine.emitActivity(t, db.Activity{Worker: "planner", Kind: "round",
|
||||
Summary: "第 0 轮目标拆解"})
|
||||
goals := s.createGoals(ctx, t, func(r db.Activity) {
|
||||
s.engine.emitActivity(t, r)
|
||||
})
|
||||
if ctx.Err() != nil || s.engine.IsDeleting(t.ID) {
|
||||
return
|
||||
}
|
||||
for _, g := range goals {
|
||||
summary := g.Text
|
||||
if g.VulnClass != "" {
|
||||
summary = fmt.Sprintf("[%s] %s", g.VulnClass, g.Text)
|
||||
}
|
||||
s.engine.Run(s.ctx, t)
|
||||
}()
|
||||
s.engine.emitActivity(t, db.Activity{Worker: "planner", Kind: "text", Summary: summary})
|
||||
}
|
||||
if ctx.Err() != nil || s.engine.IsDeleting(t.ID) {
|
||||
return
|
||||
}
|
||||
s.engine.Run(s.ctx, t)
|
||||
}
|
||||
|
||||
func (s *Server) occupiesConcurrencySlot(t *Task) bool {
|
||||
if t == nil {
|
||||
return false
|
||||
}
|
||||
lifecycle := t.lifecycleSnapshot()
|
||||
if lifecycle.Queued || lifecycle.Paused || isTerminalStatus(lifecycle.Status) {
|
||||
return false
|
||||
}
|
||||
// Deletion temporarily pauses the Engine but has not committed yet. Preserve
|
||||
// the task's slot until PostgreSQL deletion succeeds; otherwise FIFO promotion
|
||||
// during the drain window could over-admit if deletion later aborts and the
|
||||
// persisted running task is restored.
|
||||
if s.engine.IsDeleting(t.ID) {
|
||||
return true
|
||||
}
|
||||
return !s.engine.IsPaused(t.ID) && (s.engine.ReadyFor(t) || s.engine.ActiveLLMCalls(t.ID) > 0)
|
||||
}
|
||||
|
||||
func (s *Server) runningTaskCount(excludeID string) int {
|
||||
count := 0
|
||||
for _, task := range s.m.List() {
|
||||
if task.ID != excludeID && s.occupiesConcurrencySlot(task) {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
// admitTask is the single admission path for new, resumed, rerun and follow-up
|
||||
// work. It atomically either starts the task or appends it to the persistent FIFO
|
||||
// queue. mode is bootstrap for a freshly-created task and resume otherwise.
|
||||
func (s *Server) admitTask(t *Task, mode string) (queued bool, err error) {
|
||||
return s.admitTaskWhen(t, mode, false)
|
||||
}
|
||||
|
||||
// admitPausedTask is the task-control resume path. The paused precondition is
|
||||
// checked under the same scheduler lock as admission so a concurrent pause,
|
||||
// dequeue or FIFO promotion cannot leave database and Engine state divergent.
|
||||
func (s *Server) admitPausedTask(t *Task) (queued bool, err error) {
|
||||
return s.admitTaskWhen(t, "resume", true)
|
||||
}
|
||||
|
||||
func (s *Server) admitTaskWhen(t *Task, mode string, requirePaused bool) (queued bool, err error) {
|
||||
if t == nil {
|
||||
return false, fmt.Errorf("task not found")
|
||||
}
|
||||
if mode != "bootstrap" {
|
||||
mode = "resume"
|
||||
}
|
||||
s.concMu.Lock()
|
||||
defer s.concMu.Unlock()
|
||||
// Delete installs its barrier under concMu as well. Re-resolve after acquiring
|
||||
// the lock so a request that captured a task pointer before successful deletion
|
||||
// cannot revive that stale handle after StopTask clears its Engine maps.
|
||||
current, exists := s.m.Task(t.ID)
|
||||
if !exists || current != t || s.engine.IsDeleting(t.ID) {
|
||||
return false, fmt.Errorf("task is being deleted")
|
||||
}
|
||||
if !s.engine.beginTaskOperation(t.ID) {
|
||||
return false, fmt.Errorf("task is being deleted")
|
||||
}
|
||||
defer s.engine.decInflight(t.ID)
|
||||
lifecycle := t.lifecycleSnapshot()
|
||||
if requirePaused {
|
||||
if isTerminalStatus(lifecycle.Status) {
|
||||
return false, fmt.Errorf("终态任务不能执行继续")
|
||||
}
|
||||
if !lifecycle.Paused {
|
||||
return false, fmt.Errorf("仅已暂停的任务可以继续")
|
||||
}
|
||||
mode = s.resumeAdmissionMode(t)
|
||||
}
|
||||
wasTerminal := isTerminalStatus(lifecycle.Status)
|
||||
wasPaused := lifecycle.Paused || s.engine.IsPaused(t.ID)
|
||||
wasQueued := lifecycle.Queued
|
||||
engineWasPaused := s.engine.IsPaused(t.ID)
|
||||
enabled, limit := s.m.ConcurrencyLimit()
|
||||
ready := s.engine.ReadyFor(t)
|
||||
|
||||
// Work added to a task that is already admitted only needs a wake-up. This
|
||||
// matters when the configured limit was lowered below the current running
|
||||
// count: an existing task must not suddenly mark itself queued while its
|
||||
// planner/workers are still live. Still compare-and-commit the persisted
|
||||
// status: a concurrent terminal transition must win instead of being silently
|
||||
// reported as a successful follow-up admission.
|
||||
if !wasTerminal && !wasPaused && !wasQueued && s.engine.Started(t.ID) && (!enabled || ready) {
|
||||
if err := s.m.ApplyTaskAdmission(t.ID, lifecycle.Status, lifecycle.Status, false, "resume", false); err != nil {
|
||||
return false, err
|
||||
}
|
||||
s.startAdmittedTask(t, "resume")
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// Preserve the original first-run mode across repeated admissions. Legacy
|
||||
// queued rows have an empty queue_mode, so infer bootstrap from their graph.
|
||||
if wasQueued {
|
||||
switch lifecycle.QueueMode {
|
||||
case "bootstrap":
|
||||
mode = "bootstrap"
|
||||
case "":
|
||||
mode = s.resumeAdmissionMode(t)
|
||||
}
|
||||
}
|
||||
|
||||
readyBacklog := enabled && s.hasReadyQueuedTask(t.ID)
|
||||
atCapacity := enabled && s.runningTaskCount(t.ID) >= limit
|
||||
shouldQueue := enabled && (!ready || readyBacklog || atCapacity)
|
||||
|
||||
// Install the execution barrier before reviving a terminal/paused task. Without
|
||||
// this ordering, its already-running worker loops can claim the newly-opened
|
||||
// intent in the gap between status=running and queued=true.
|
||||
if shouldQueue || wasTerminal || wasPaused || wasQueued {
|
||||
s.engine.Pause(t.ID, agent.Causef("queued_for_admission", "任务等待运行准入",
|
||||
"任务正在等待并发队列或准入状态提交,本次执行已停止;只有获得运行槽后才会重新领取意图"))
|
||||
}
|
||||
|
||||
status := lifecycle.Status
|
||||
if wasTerminal {
|
||||
status = "running"
|
||||
}
|
||||
if err := s.m.ApplyTaskAdmission(t.ID, lifecycle.Status, status, shouldQueue, mode, wasQueued); err != nil {
|
||||
if !engineWasPaused && !wasPaused && !wasQueued {
|
||||
s.engine.Resume(t)
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
if lifecycle.Status == "timeout" && status == "running" {
|
||||
// ApplyTaskAdmission reset this timed-out run's persisted clock. Clear the
|
||||
// matching Engine gates before either parking it in FIFO or starting work;
|
||||
// otherwise the old settling flag would make every worker skip forever.
|
||||
s.engine.resetTimeoutRevival(t.ID)
|
||||
}
|
||||
if shouldQueue {
|
||||
if !wasQueued {
|
||||
summary := fmt.Sprintf("已排队:达到并发上限 %d,等待空位后自动开始", limit)
|
||||
switch {
|
||||
case !ready:
|
||||
summary = "已排队:当前没有可运行的 LLM 配置,配置恢复后自动开始"
|
||||
case readyBacklog:
|
||||
summary = "已排队:已有更早的任务等待运行,将按 FIFO 顺序自动开始"
|
||||
}
|
||||
s.engine.emitActivity(t, db.Activity{Worker: "system", Kind: "text", Summary: summary})
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
s.startAdmittedTask(t, mode)
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (s *Server) hasReadyQueuedTask(excludeID string) bool {
|
||||
for _, task := range s.m.List() {
|
||||
lifecycle := task.lifecycleSnapshot()
|
||||
if task.ID != excludeID && lifecycle.Queued && !lifecycle.Paused && !isTerminalStatus(lifecycle.Status) && s.engine.ReadyFor(task) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (s *Server) startAdmittedTask(t *Task, mode string) {
|
||||
if mode == "bootstrap" {
|
||||
if !s.engine.beginTaskOperation(t.ID) {
|
||||
return
|
||||
}
|
||||
// A first-run task may have kept the Engine pause barrier while waiting
|
||||
// in the concurrency queue. Clear it only after operation admission, or
|
||||
// startTaskEngine's first execContextFor call would return a cancelled
|
||||
// context and silently strand the dequeued task.
|
||||
if s.engine.IsPaused(t.ID) {
|
||||
s.engine.Resume(t)
|
||||
}
|
||||
go func() {
|
||||
defer s.engine.decInflight(t.ID)
|
||||
s.startTaskEngine(t)
|
||||
}()
|
||||
return
|
||||
}
|
||||
s.engine.Run(s.ctx, t)
|
||||
s.engine.Resume(t)
|
||||
// Run returns early for an already-started task. Explicitly ensure a timeout
|
||||
// coordinator exists after resume; resetTimeoutRevival cleared the completed
|
||||
// coordinator's marker and the next real call will stamp a fresh deadline.
|
||||
s.engine.startDeadlineCoordinator(s.ctx, t)
|
||||
}
|
||||
|
||||
func (s *Server) reconcileConcurrency() {
|
||||
s.concMu.Lock()
|
||||
defer s.concMu.Unlock()
|
||||
enabled, limit := s.m.ConcurrencyLimit()
|
||||
|
||||
// A task whose provider chain becomes unavailable cannot do useful work and
|
||||
// must not reserve a limited running slot forever. Park it persistently so a
|
||||
// later profile edit/recovery can re-enter through the same FIFO path.
|
||||
if enabled {
|
||||
for _, task := range s.m.List() {
|
||||
lifecycle := task.lifecycleSnapshot()
|
||||
if lifecycle.Queued || lifecycle.Paused || s.engine.IsPaused(task.ID) || s.engine.IsDeleting(task.ID) ||
|
||||
isTerminalStatus(lifecycle.Status) || s.engine.ReadyFor(task) || s.engine.ActiveLLMCalls(task.ID) > 0 {
|
||||
continue
|
||||
}
|
||||
mode := s.resumeAdmissionMode(task)
|
||||
s.engine.Pause(task.ID, agent.Causef("llm_unavailable_queued", "LLM 不可用,任务进入等待队列",
|
||||
"任务当前无法解析可运行的 Planner/Worker LLM,已释放并发槽;配置恢复后按队列顺序继续"))
|
||||
if err := s.m.EnqueueTask(task.ID, mode); err != nil {
|
||||
s.engine.Resume(task)
|
||||
log.Printf("[concurrency] task %s 因 LLM 不可用入队失败: %v", task.ID, err)
|
||||
continue
|
||||
}
|
||||
s.engine.emitActivity(task, db.Activity{Worker: "system", Kind: "text",
|
||||
Summary: "已排队:当前没有可运行的 LLM 配置,配置恢复后自动开始"})
|
||||
}
|
||||
}
|
||||
|
||||
type queuedTask struct {
|
||||
task *Task
|
||||
lifecycle taskLifecycleState
|
||||
}
|
||||
queued := []queuedTask{}
|
||||
for _, task := range s.m.List() {
|
||||
lifecycle := task.lifecycleSnapshot()
|
||||
if lifecycle.Queued && !isTerminalStatus(lifecycle.Status) {
|
||||
queued = append(queued, queuedTask{task: task, lifecycle: lifecycle})
|
||||
}
|
||||
}
|
||||
sort.SliceStable(queued, func(i, j int) bool {
|
||||
left, right := queued[i].lifecycle.QueuedAt, queued[j].lifecycle.QueuedAt
|
||||
if left == 0 {
|
||||
left = queued[i].task.CreatedAt * int64(1e9)
|
||||
}
|
||||
if right == 0 {
|
||||
right = queued[j].task.CreatedAt * int64(1e9)
|
||||
}
|
||||
if left != right {
|
||||
return left < right
|
||||
}
|
||||
// Old rows may not have queued_at and task creation timestamps are only
|
||||
// kept to second precision in memory. Task ids are monotonic, providing a
|
||||
// deterministic oldest-first fallback for those ties.
|
||||
leftID, leftErr := strconv.ParseInt(queued[i].task.ID, 10, 64)
|
||||
rightID, rightErr := strconv.ParseInt(queued[j].task.ID, 10, 64)
|
||||
if leftErr == nil && rightErr == nil {
|
||||
return leftID < rightID
|
||||
}
|
||||
return queued[i].task.ID < queued[j].task.ID
|
||||
})
|
||||
slots := len(queued)
|
||||
if enabled {
|
||||
slots = limit - s.runningTaskCount("")
|
||||
}
|
||||
for _, entry := range queued {
|
||||
if slots <= 0 {
|
||||
break
|
||||
}
|
||||
task := entry.task
|
||||
lifecycle := task.lifecycleSnapshot()
|
||||
// Keep FIFO order, but do not consume a concurrency slot for a task
|
||||
// whose explicit chain/global provider is unavailable. It will be retried
|
||||
// after the user configures or resets its LLM chain.
|
||||
if lifecycle.Paused || (enabled && !s.engine.ReadyFor(task)) {
|
||||
continue
|
||||
}
|
||||
mode := lifecycle.QueueMode
|
||||
if mode != "bootstrap" && mode != "resume" {
|
||||
mode = s.resumeAdmissionMode(task)
|
||||
}
|
||||
if mode != "bootstrap" {
|
||||
mode = "resume"
|
||||
}
|
||||
if !s.engine.beginTaskOperation(task.ID) {
|
||||
continue
|
||||
}
|
||||
if err := s.m.ApplyTaskAdmission(task.ID, lifecycle.Status, lifecycle.Status, false, mode, false); err != nil {
|
||||
s.engine.decInflight(task.ID)
|
||||
continue
|
||||
}
|
||||
s.startAdmittedTask(task, mode)
|
||||
s.engine.decInflight(task.ID)
|
||||
slots--
|
||||
}
|
||||
}
|
||||
|
||||
// reviveTask 让一个已停下的任务重新跑起来:把终态(done/failed/timeout)拉回 running、
|
||||
@@ -56,22 +356,8 @@ func (s *Server) reviveTask(t *Task) {
|
||||
if t == nil {
|
||||
return
|
||||
}
|
||||
// 终态 → 拉回 running(SetTaskStatus 会清 completed_at 并同步内存 handle 的 Status,
|
||||
// 使 planner/worker 循环的终态门放行)。
|
||||
if isTerminalStatus(t.Status) {
|
||||
if err := s.m.SetTaskStatus(t.ID, "running"); err != nil {
|
||||
log.Printf("[revive] task %s 置 running 失败: %v", t.ID, err)
|
||||
}
|
||||
}
|
||||
// 确保引擎循环存活:已 started 只会 Notify;未 started(重启后未恢复的终态任务)则重起循环。
|
||||
s.engine.Run(s.ctx, t)
|
||||
// 暂停中 → 解除暂停(清引擎内存态 + Notify),并持久化 paused=false(存活过重启)。
|
||||
if t.Paused || s.engine.IsPaused(t.ID) {
|
||||
s.engine.Resume(t)
|
||||
t.Paused = false
|
||||
if err := s.m.SetTaskPaused(t.ID, false); err != nil {
|
||||
log.Printf("[revive] task %s 解除暂停失败: %v", t.ID, err)
|
||||
}
|
||||
if _, err := s.admitTask(t, "resume"); err != nil {
|
||||
log.Printf("[revive] task %s 恢复失败: %v", t.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -83,24 +369,25 @@ func (s *Server) reviveTask(t *Task) {
|
||||
// Returns the seeded specs so callers can emit activity records for them.
|
||||
// emit, when non-nil, is forwarded to DecomposeGoals so LLM steps are visible in the UI.
|
||||
func (s *Server) createGoals(ctx context.Context, t *Task, emit func(db.Activity)) []goalSpec {
|
||||
if t == nil {
|
||||
return nil
|
||||
}
|
||||
// Use the SAME LLM the task runs on (its pinned profile, else the active profile),
|
||||
// NOT agent.FromEnv() — the LLM is configured via the UI (DB profile), not env vars,
|
||||
// so FromEnv returned empty and every task silently fell back to the crude rule
|
||||
// splitter (which shredded URLs / made meaningless 2-way splits).
|
||||
var specs []goalSpec
|
||||
if cfg, ok := s.goalsLLMConfig(t); ok {
|
||||
var as *db.AssetStore
|
||||
if s.m != nil {
|
||||
as = s.m.Assets()
|
||||
}
|
||||
taskID, _ := strconv.ParseInt(t.ID, 10, 64)
|
||||
// DecomposeGoals persists each decomposed goal via the set_goals tool straight
|
||||
// into t.Store; it returns the specs it wrote so we can emit activity + spot the
|
||||
// empty (LLM error / no provider) case below. No write-back needed here anymore.
|
||||
for _, g := range agent.DecomposeGoals(ctx, cfg, s.m.dir, t.Goal, t.Description, as, t.Store, taskID, emit) {
|
||||
if strings.TrimSpace(g.Text) != "" {
|
||||
specs = append(specs, goalSpec{Text: g.Text, VulnClass: g.VulnClass})
|
||||
}
|
||||
var as *db.AssetStore
|
||||
if s.m != nil {
|
||||
as = s.m.Assets()
|
||||
}
|
||||
taskID, _ := strconv.ParseInt(t.ID, 10, 64)
|
||||
s.engine.BeginLLMCall(t.ID)
|
||||
decomposed := agent.DecomposeGoalsWithProvider(ctx, s.agentsForTask(t).runtime, s.m.dir, t.Goal, t.Description, as, t.Store, taskID, emit)
|
||||
s.engine.EndLLMCall(t.ID)
|
||||
for _, g := range decomposed {
|
||||
if strings.TrimSpace(g.Text) != "" {
|
||||
specs = append(specs, goalSpec{Text: g.Text, VulnClass: g.VulnClass})
|
||||
}
|
||||
}
|
||||
if len(specs) == 0 {
|
||||
@@ -119,19 +406,3 @@ func (s *Server) createGoals(ctx context.Context, t *Task, emit func(db.Activity
|
||||
}
|
||||
return specs
|
||||
}
|
||||
|
||||
// goalsLLMConfig resolves the LLM config for goal decomposition by the standard
|
||||
// precedence: the goals agent's own binding → the task's pinned profile → the global
|
||||
// active profile (same source the engine runs on).
|
||||
func (s *Server) goalsLLMConfig(t *Task) (agent.Config, bool) {
|
||||
var pin *int64
|
||||
if t != nil {
|
||||
pin = t.LLMProfileID
|
||||
}
|
||||
if eff := s.effectiveProfileForAgent("goals", pin); eff != nil {
|
||||
if cfg, ok := s.loadProfileConfig(*eff); ok {
|
||||
return cfg, true
|
||||
}
|
||||
}
|
||||
return s.loadLLMConfig()
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCreateGoalsNilTask(t *testing.T) {
|
||||
if got := (&Server{}).createGoals(context.Background(), nil, nil); len(got) != 0 {
|
||||
t.Fatalf("nil task produced goals: %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/Autumn-27/artex/db"
|
||||
)
|
||||
|
||||
func TestInheritedActivityDetailAndRelationDeletion(t *testing.T) {
|
||||
m, err := NewManager(t.TempDir(), "")
|
||||
if err != nil {
|
||||
t.Skipf("postgres unavailable (%v) - skipping", err)
|
||||
}
|
||||
defer m.Close()
|
||||
|
||||
source, err := m.CreateTaskWithOptions("detail source", "source goal", db.TaskCreateOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sourceID := mustTaskID(t, source.ID)
|
||||
current, err := m.CreateTaskWithOptions("detail current", "current goal", db.TaskCreateOptions{SourceTaskIDs: []int64{sourceID}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_, _ = m.DeleteTask(current.ID, DeleteTaskOptions{})
|
||||
_, _ = m.DeleteTask(source.ID, DeleteTaskOptions{})
|
||||
})
|
||||
|
||||
plannerStep, err := source.Store.AppendActivity(db.Activity{
|
||||
Worker: "planner", Kind: "text", Summary: "private planner", Detail: "private planner detail",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
intentID, err := source.Store.AddIntent(map[string]any{"summary": "completed source work"}, 1, nil, "planner")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := source.Store.SetIntentState(intentID, "done"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
workerStep, err := source.Store.AppendActivity(db.Activity{
|
||||
NodeID: &intentID, Worker: "worker", Kind: "result", Summary: "shared result", Detail: "shared worker detail",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
s := New(context.Background(), m, t.TempDir(), t.TempDir(), t.TempDir())
|
||||
token, err := signJWT(s.jwtKey)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fetchDetail := func(stepID int64) string {
|
||||
req := httptest.NewRequest(http.MethodGet,
|
||||
fmt.Sprintf("/api/exploration/activity/%d?task=%s", stepID, current.ID), nil)
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
rec := httptest.NewRecorder()
|
||||
s.Handler().ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("activity detail %d: status=%d body=%s", stepID, rec.Code, rec.Body.String())
|
||||
}
|
||||
var body struct {
|
||||
Detail string `json:"detail"`
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return body.Detail
|
||||
}
|
||||
if got := fetchDetail(plannerStep); got != "" {
|
||||
t.Fatalf("source planner transcript leaked through API: %q", got)
|
||||
}
|
||||
if got := fetchDetail(workerStep); got != "shared worker detail" {
|
||||
t.Fatalf("terminal inherited worker detail missing: %q", got)
|
||||
}
|
||||
|
||||
if _, err := m.DeleteTask(source.ID, DeleteTaskOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
currentAfter, ok := m.Task(current.ID)
|
||||
if !ok || len(currentAfter.SourceTaskIDs) != 0 {
|
||||
t.Fatalf("deleted source remained in live task DTO state: %+v", currentAfter)
|
||||
}
|
||||
}
|
||||
|
||||
func mustTaskID(t *testing.T, raw string) int64 {
|
||||
t.Helper()
|
||||
var id int64
|
||||
if _, err := fmt.Sscan(raw, &id); err != nil || id <= 0 {
|
||||
t.Fatalf("bad task id %q: %v", raw, err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Autumn-27/artex/db"
|
||||
)
|
||||
|
||||
func TestInheritedDTOProvenance(t *testing.T) {
|
||||
createdAt := time.Date(2026, time.August, 19, 12, 0, 0, 0, time.UTC)
|
||||
node := &db.Node{
|
||||
ID: 41,
|
||||
Kind: db.KindFinding,
|
||||
Payload: json.RawMessage(`{"vulnclass":"xss","severity":"high","summary":"source finding"}`),
|
||||
State: "confirmed",
|
||||
CreatedAt: createdAt,
|
||||
SourceTaskID: 7,
|
||||
Inherited: true,
|
||||
}
|
||||
|
||||
nodeDTO := taskNodeDTO(node)
|
||||
if !nodeDTO.Inherited || nodeDTO.SourceTaskID != "7" {
|
||||
t.Fatalf("node provenance lost: %+v", nodeDTO)
|
||||
}
|
||||
finding := findingDTOsForOwner("7", "source", []*db.Node{node}, nil, nil)
|
||||
if len(finding) != 1 || !finding[0].Inherited || finding[0].SourceTaskID != "7" || finding[0].TaskID != "7" {
|
||||
t.Fatalf("finding provenance lost: %+v", finding)
|
||||
}
|
||||
|
||||
intentID := int64(44)
|
||||
activity := activityDTO(db.Activity{
|
||||
ID: 45,
|
||||
NodeID: &intentID,
|
||||
CreatedAt: createdAt,
|
||||
SourceTaskID: 7,
|
||||
Inherited: true,
|
||||
})
|
||||
if !activity.Inherited || activity.SourceTaskID != "7" || activity.IntentID != "44" {
|
||||
t.Fatalf("activity provenance lost: %+v", activity)
|
||||
}
|
||||
assetRef := coverageAssetRefDTO(db.AssetRef{
|
||||
ID: 46, Kind: db.KindFact, State: "confirmed", Summary: "source fact", SourceTaskID: 7, Inherited: true,
|
||||
})
|
||||
if !assetRef.Inherited || assetRef.SourceTaskID != "7" {
|
||||
t.Fatalf("asset ref provenance lost: %+v", assetRef)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInheritedIntentResultOnlyAllowsHistory(t *testing.T) {
|
||||
for _, state := range []string{"done", "blocked", "exhausted", "stopped"} {
|
||||
if !inheritedIntentResult(state) {
|
||||
t.Errorf("terminal state %q should be inherited history", state)
|
||||
}
|
||||
}
|
||||
for _, state := range []string{"open", "running", ""} {
|
||||
if inheritedIntentResult(state) {
|
||||
t.Errorf("live state %q must not enter inherited history", state)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestInheritedGraphSnapshotFiltersLiveIntentsAndEdges(t *testing.T) {
|
||||
nodes := []*db.Node{
|
||||
{ID: 1, Kind: db.KindFact, State: "confirmed"},
|
||||
{ID: 2, Kind: db.KindIntent, State: "running"},
|
||||
{ID: 3, Kind: db.KindIntent, State: "done"},
|
||||
{ID: 4, Kind: db.KindFinding, State: "confirmed"},
|
||||
}
|
||||
edges := []db.Edge{
|
||||
{From: 1, Rel: db.RelDerivedFrom, To: 2},
|
||||
{From: 1, Rel: db.RelDerivedFrom, To: 3},
|
||||
{From: 3, Rel: db.RelYields, To: 4},
|
||||
}
|
||||
|
||||
gotNodes, gotEdges := inheritedGraphSnapshot(nodes, edges, 77)
|
||||
if len(gotNodes) != 3 {
|
||||
t.Fatalf("expected fact, terminal intent and finding; got %+v", gotNodes)
|
||||
}
|
||||
for _, node := range gotNodes {
|
||||
if node.ID == 2 {
|
||||
t.Fatal("running inherited intent leaked into graph snapshot")
|
||||
}
|
||||
if !node.Inherited || node.SourceTaskID != 77 {
|
||||
t.Fatalf("source provenance missing: %+v", node)
|
||||
}
|
||||
}
|
||||
if len(gotEdges) != 2 {
|
||||
t.Fatalf("edge referencing hidden live intent was not removed: %+v", gotEdges)
|
||||
}
|
||||
for _, edge := range gotEdges {
|
||||
if edge.From == 2 || edge.To == 2 {
|
||||
t.Fatalf("live intent id leaked through edge: %+v", edge)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindingProvenanceInTask(t *testing.T) {
|
||||
task := &Task{ID: "10", SourceTaskIDs: []int64{7, 8}}
|
||||
|
||||
local := int64(10)
|
||||
if source, inherited, allowed := findingProvenanceInTask(task, &local); !allowed || inherited || source != "" {
|
||||
t.Fatalf("local finding: source=%q inherited=%v allowed=%v", source, inherited, allowed)
|
||||
}
|
||||
sourceID := int64(7)
|
||||
if source, inherited, allowed := findingProvenanceInTask(task, &sourceID); !allowed || !inherited || source != "7" {
|
||||
t.Fatalf("direct source finding: source=%q inherited=%v allowed=%v", source, inherited, allowed)
|
||||
}
|
||||
unrelated := int64(6)
|
||||
if _, _, allowed := findingProvenanceInTask(task, &unrelated); allowed {
|
||||
t.Fatal("unrelated finding must not be readable through task context")
|
||||
}
|
||||
if _, _, allowed := findingProvenanceInTask(task, nil); allowed {
|
||||
t.Fatal("detached finding must not be readable through task context")
|
||||
}
|
||||
}
|
||||
+850
-63
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,280 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
pgdb "github.com/Autumn-27/artex/db"
|
||||
"github.com/Autumn-27/artex/traffic"
|
||||
)
|
||||
|
||||
func TestSeedAssociatesTargetAssetWithTask(t *testing.T) {
|
||||
m, err := NewManager(t.TempDir(), "")
|
||||
if err != nil {
|
||||
t.Skipf("postgres unavailable (%v) - skipping", err)
|
||||
}
|
||||
defer m.Close()
|
||||
task, err := m.CreateTask("seed ownership", "seed ownership", nil, 0, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_, _ = m.DeleteTask(task.ID, DeleteTaskOptions{DeleteAssets: true})
|
||||
})
|
||||
|
||||
host := fmt.Sprintf("seed-%s.example.test", task.ID)
|
||||
s := &Server{m: m}
|
||||
s.seed(task, "https://"+host)
|
||||
taskID := mustTaskID(t, task.ID)
|
||||
assets, err := m.Assets().QueryByTask(taskID, "", 20, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
found := false
|
||||
for _, asset := range assets {
|
||||
if asset.Domain == host || asset.URL == "https://"+host {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("seed target was not associated with task %s: %+v", task.ID, assets)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteTaskFilesRemovesOnlyOwnedWorkspaceAndTranscripts(t *testing.T) {
|
||||
dataDir := t.TempDir()
|
||||
|
||||
removed := []struct {
|
||||
path string
|
||||
isDir bool
|
||||
}{
|
||||
{path: filepath.Join("tasks", "77"), isDir: true},
|
||||
{path: filepath.Join("transcripts", "exp12-main.jsonl")},
|
||||
{path: filepath.Join("transcripts", "exp12-planner.jsonl")},
|
||||
{path: filepath.Join("transcripts", "exp12-worker-i9.jsonl")},
|
||||
{path: filepath.Join("transcripts", "exp12-main"), isDir: true},
|
||||
{path: filepath.Join("transcripts", "exp12-worker-i9"), isDir: true},
|
||||
}
|
||||
kept := []struct {
|
||||
path string
|
||||
isDir bool
|
||||
}{
|
||||
{path: filepath.Join("tasks", "78"), isDir: true},
|
||||
{path: filepath.Join("transcripts", "exp123-main.jsonl")},
|
||||
{path: filepath.Join("transcripts", "exp123-main"), isDir: true},
|
||||
{path: filepath.Join("transcripts", "exp1-main.jsonl")},
|
||||
{path: filepath.Join("transcripts", "exp12main.jsonl")},
|
||||
{path: filepath.Join("transcripts", "exp12-main.jsonl.bak")},
|
||||
{path: filepath.Join("transcripts", "exp12-main.txt")},
|
||||
{path: filepath.Join("transcripts", "conv-12.jsonl")},
|
||||
{path: filepath.Join("transcripts", "unrelated-session"), isDir: true},
|
||||
}
|
||||
|
||||
for _, fixture := range append(removed, kept...) {
|
||||
createDeleteFixture(t, dataDir, fixture.path, fixture.isDir)
|
||||
}
|
||||
|
||||
deleted, err := deleteTaskFiles(dataDir, "77", 12)
|
||||
if err != nil {
|
||||
t.Fatalf("delete task files: %v", err)
|
||||
}
|
||||
if !deleted {
|
||||
t.Fatal("expected deletion to be reported")
|
||||
}
|
||||
for _, fixture := range removed {
|
||||
assertPathMissing(t, filepath.Join(dataDir, fixture.path))
|
||||
}
|
||||
for _, fixture := range kept {
|
||||
assertPathExists(t, filepath.Join(dataDir, fixture.path))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteTaskFilesReportsTranscriptOnlyDeletion(t *testing.T) {
|
||||
dataDir := t.TempDir()
|
||||
path := filepath.Join("transcripts", "exp44-main.jsonl")
|
||||
createDeleteFixture(t, dataDir, path, false)
|
||||
|
||||
deleted, err := deleteTaskFiles(dataDir, "missing-task", 44)
|
||||
if err != nil {
|
||||
t.Fatalf("delete task files: %v", err)
|
||||
}
|
||||
if !deleted {
|
||||
t.Fatal("expected transcript deletion to be reported")
|
||||
}
|
||||
assertPathMissing(t, filepath.Join(dataDir, path))
|
||||
}
|
||||
|
||||
func TestDeleteTaskFilesMissingTargetsIsIdempotent(t *testing.T) {
|
||||
deleted, err := deleteTaskFiles(t.TempDir(), "77", 12)
|
||||
if err != nil {
|
||||
t.Fatalf("delete missing task files: %v", err)
|
||||
}
|
||||
if deleted {
|
||||
t.Fatal("missing targets must not be reported as deleted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStageTaskFilesRollbackRestoresWorkspaceAndTranscripts(t *testing.T) {
|
||||
dataDir := t.TempDir()
|
||||
paths := []string{
|
||||
filepath.Join("tasks", "77", "notes.txt"),
|
||||
filepath.Join("transcripts", "exp12-main.jsonl"),
|
||||
filepath.Join("transcripts", "exp12-worker-i9", "sidechain.jsonl"),
|
||||
}
|
||||
for _, path := range paths {
|
||||
createDeleteFixture(t, dataDir, path, false)
|
||||
}
|
||||
|
||||
stage, err := stageTaskFiles(dataDir, "77", 12)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !stage.deleted {
|
||||
t.Fatal("expected files to be staged")
|
||||
}
|
||||
for _, path := range paths {
|
||||
assertPathMissing(t, filepath.Join(dataDir, path))
|
||||
}
|
||||
if err := stage.rollback(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, path := range paths {
|
||||
assertPathExists(t, filepath.Join(dataDir, path))
|
||||
}
|
||||
}
|
||||
|
||||
func TestStageTaskFilesRollbackReportsRestoreFailure(t *testing.T) {
|
||||
dataDir := t.TempDir()
|
||||
workspace := filepath.Join(dataDir, "tasks", "77")
|
||||
createDeleteFixture(t, dataDir, filepath.Join("tasks", "77"), true)
|
||||
|
||||
stage, err := stageTaskFiles(dataDir, "77", 12)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(workspace, []byte("conflict"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cause := errors.New("forced PostgreSQL delete failure")
|
||||
err = rollbackTaskDelete(cause, nil, stage)
|
||||
if err == nil || !errors.Is(err, cause) || !strings.Contains(err.Error(), "destination already exists") {
|
||||
t.Fatalf("rollback err=%v, want original and restore errors", err)
|
||||
}
|
||||
if _, statErr := os.Stat(stage.stageDir); statErr != nil {
|
||||
t.Fatalf("staging was removed after failed restore: %v", statErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagerDeleteTaskRestoresFilesAndTrafficWhenPostgresDeleteFails(t *testing.T) {
|
||||
dataDir := t.TempDir()
|
||||
m, err := NewManager(dataDir, "")
|
||||
if err != nil {
|
||||
t.Skipf("postgres unavailable (%v) - skipping", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = m.Close() })
|
||||
|
||||
task, err := m.CreateTask("delete rollback", "delete rollback", nil, 0, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
taskID := mustTaskID(t, task.ID)
|
||||
triggerName := fmt.Sprintf("test_fail_task_delete_%d", taskID)
|
||||
functionName := triggerName + "_fn"
|
||||
cleanupDB := func() {
|
||||
_, _ = m.pg.Exec(fmt.Sprintf(`DROP TRIGGER IF EXISTS %s ON tasks`, triggerName))
|
||||
_, _ = m.pg.Exec(fmt.Sprintf(`DROP FUNCTION IF EXISTS %s()`, functionName))
|
||||
_, _ = m.DeleteTask(task.ID, DeleteTaskOptions{DeleteAssets: true, DeleteTraffic: true, DeleteFiles: true})
|
||||
}
|
||||
t.Cleanup(cleanupDB)
|
||||
|
||||
host := fmt.Sprintf("delete-rollback-%d.example.test", taskID)
|
||||
if _, err := m.Assets().UpsertHTTPService(pgdb.UpsertHTTPServiceReq{URL: "https://" + host, TaskID: taskID}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
createDeleteFixture(t, dataDir, filepath.Join("tasks", task.ID), true)
|
||||
transcriptPath := filepath.Join(dataDir, "transcripts", fmt.Sprintf("exp%d-main.jsonl", task.ExpID))
|
||||
createDeleteFixture(t, dataDir, filepath.Join("transcripts", filepath.Base(transcriptPath)), false)
|
||||
|
||||
tr, err := traffic.Open(filepath.Join(dataDir, "traffic"), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.traffic = tr
|
||||
trafficTree := filepath.Join(dataDir, "traffic", host, "GET", "1-0001")
|
||||
if err := os.MkdirAll(trafficTree, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
trafficMarker := filepath.Join(trafficTree, "request.http")
|
||||
if err := os.WriteFile(trafficMarker, []byte("original traffic"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := tr.DB().Exec(`INSERT INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
|
||||
VALUES(?,?,?,?,?,?,?,?,?,?,?)`, "1-0001", 1, host, "GET", "/", "https://"+host+"/", 200, "text/plain", 0, 0, host+"/GET/1-0001"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if _, err := m.pg.Exec(fmt.Sprintf(`CREATE FUNCTION %s() RETURNS trigger LANGUAGE plpgsql AS $body$
|
||||
BEGIN RAISE EXCEPTION 'forced task delete failure'; END $body$`, functionName)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := m.pg.Exec(fmt.Sprintf(`CREATE TRIGGER %s BEFORE DELETE ON tasks
|
||||
FOR EACH ROW WHEN (OLD.id = %d) EXECUTE FUNCTION %s()`, triggerName, taskID, functionName)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, err = m.DeleteTask(task.ID, DeleteTaskOptions{DeleteTraffic: true, DeleteFiles: true})
|
||||
if err == nil || !strings.Contains(err.Error(), "forced task delete failure") {
|
||||
t.Fatalf("delete err=%v, want injected PostgreSQL failure", err)
|
||||
}
|
||||
if got, err := m.pg.GetTask(taskID); err != nil || got == nil {
|
||||
t.Fatalf("task row was lost after failed delete: task=%+v err=%v", got, err)
|
||||
}
|
||||
assertPathExists(t, filepath.Join(dataDir, "tasks", task.ID))
|
||||
assertPathExists(t, transcriptPath)
|
||||
if got, err := os.ReadFile(trafficMarker); err != nil || string(got) != "original traffic" {
|
||||
t.Fatalf("traffic tree was not restored: content=%q err=%v", got, err)
|
||||
}
|
||||
var trafficRows int
|
||||
if err := tr.DB().QueryRow(`SELECT COUNT(*) FROM exchanges WHERE host=?`, host).Scan(&trafficRows); err != nil || trafficRows != 1 {
|
||||
t.Fatalf("traffic index was not restored: rows=%d err=%v", trafficRows, err)
|
||||
}
|
||||
}
|
||||
|
||||
func createDeleteFixture(t *testing.T, root, relative string, isDir bool) {
|
||||
t.Helper()
|
||||
path := filepath.Join(root, relative)
|
||||
if isDir {
|
||||
if err := os.MkdirAll(filepath.Join(path, "subagents"), 0o755); err != nil {
|
||||
t.Fatalf("create fixture directory %s: %v", path, err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(path, "subagents", "agent-test.jsonl"), []byte("test"), 0o644); err != nil {
|
||||
t.Fatalf("create fixture sidechain %s: %v", path, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatalf("create fixture parent %s: %v", path, err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte("test"), 0o644); err != nil {
|
||||
t.Fatalf("create fixture file %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
func assertPathMissing(t *testing.T, path string) {
|
||||
t.Helper()
|
||||
if _, err := os.Lstat(path); !os.IsNotExist(err) {
|
||||
t.Fatalf("expected %s to be removed, stat err=%v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
func assertPathExists(t *testing.T, path string) {
|
||||
t.Helper()
|
||||
if _, err := os.Lstat(path); err != nil {
|
||||
t.Fatalf("expected %s to remain: %v", path, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestTaskLifecycleSnapshotCopiesContextSlices(t *testing.T) {
|
||||
task := &Task{
|
||||
Status: "running",
|
||||
SourceTaskIDs: []int64{11, 12},
|
||||
CompanyIDs: []int64{21, 22},
|
||||
}
|
||||
|
||||
snapshot := task.lifecycleSnapshot()
|
||||
snapshot.SourceTaskIDs[0] = 99
|
||||
snapshot.CompanyIDs[0] = 98
|
||||
|
||||
fresh := task.lifecycleSnapshot()
|
||||
if got := fresh.SourceTaskIDs[0]; got != 11 {
|
||||
t.Fatalf("snapshot source IDs alias task storage: got %d", got)
|
||||
}
|
||||
if got := fresh.CompanyIDs[0]; got != 21 {
|
||||
t.Fatalf("snapshot company IDs alias task storage: got %d", got)
|
||||
}
|
||||
|
||||
sources := []int64{31, 32}
|
||||
companies := []int64{41, 42}
|
||||
task.updateLifecycle(func(state *taskLifecycleState) {
|
||||
state.SourceTaskIDs = sources
|
||||
state.CompanyIDs = companies
|
||||
})
|
||||
sources[0] = 97
|
||||
companies[0] = 96
|
||||
|
||||
fresh = task.lifecycleSnapshot()
|
||||
if got := fresh.SourceTaskIDs[0]; got != 31 {
|
||||
t.Fatalf("lifecycle update retained caller source slice: got %d", got)
|
||||
}
|
||||
if got := fresh.CompanyIDs[0]; got != 41 {
|
||||
t.Fatalf("lifecycle update retained caller company slice: got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTaskLifecycleSnapshotConcurrentConsistency(t *testing.T) {
|
||||
task := &Task{}
|
||||
writeState := func(generation int64, queued bool) {
|
||||
task.updateLifecycle(func(state *taskLifecycleState) {
|
||||
state.Status = map[bool]string{true: "queued", false: "paused"}[queued]
|
||||
state.Queued = queued
|
||||
state.Paused = !queued
|
||||
state.QueuedAt = generation
|
||||
state.QueueMode = map[bool]string{true: "resume", false: "bootstrap"}[queued]
|
||||
state.CompletedAt = generation
|
||||
state.FirstRunAt = generation
|
||||
state.DeadlineAt = generation
|
||||
state.SourceTaskIDs = []int64{generation, generation + 1}
|
||||
state.CompanyIDs = []int64{generation, generation + 1}
|
||||
})
|
||||
}
|
||||
writeState(1, true)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for writer := 0; writer < 4; writer++ {
|
||||
writer := writer
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for i := 1; i <= 1000; i++ {
|
||||
generation := int64(writer*1000 + i)
|
||||
writeState(generation, generation%2 == 0)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
errCh := make(chan error, 4)
|
||||
for reader := 0; reader < 4; reader++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for i := 0; i < 1000; i++ {
|
||||
state := task.lifecycleSnapshot()
|
||||
if state.Queued == state.Paused {
|
||||
errCh <- fmt.Errorf("queued=%v paused=%v", state.Queued, state.Paused)
|
||||
return
|
||||
}
|
||||
wantStatus := map[bool]string{true: "queued", false: "paused"}[state.Queued]
|
||||
wantMode := map[bool]string{true: "resume", false: "bootstrap"}[state.Queued]
|
||||
if state.Status != wantStatus || state.QueueMode != wantMode {
|
||||
errCh <- fmt.Errorf("inconsistent status/mode: %+v", state)
|
||||
return
|
||||
}
|
||||
generation := state.QueuedAt
|
||||
if state.CompletedAt != generation || state.FirstRunAt != generation || state.DeadlineAt != generation ||
|
||||
len(state.SourceTaskIDs) != 2 || len(state.CompanyIDs) != 2 ||
|
||||
state.SourceTaskIDs[0] != generation || state.CompanyIDs[0] != generation {
|
||||
errCh <- fmt.Errorf("mixed lifecycle snapshot: %+v", state)
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
close(errCh)
|
||||
for err := range errCh {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
+14
-9
@@ -199,12 +199,13 @@ func (s *Server) toolListTasks() actool.CoreTool {
|
||||
if t.ParentRef != "" {
|
||||
row["parent_ref"] = t.ParentRef
|
||||
}
|
||||
if t.LLMProfileID == nil {
|
||||
llmState := t.llmStateSnapshot()
|
||||
if llmState.ProfileID == nil {
|
||||
row["llm_profile"] = "(激活配置)"
|
||||
} else if n, ok := profName[*t.LLMProfileID]; ok {
|
||||
} else if n, ok := profName[*llmState.ProfileID]; ok {
|
||||
row["llm_profile"] = n
|
||||
} else {
|
||||
row["llm_profile"] = fmt.Sprintf("#%d(已删除)", *t.LLMProfileID)
|
||||
row["llm_profile"] = fmt.Sprintf("#%d(已删除)", *llmState.ProfileID)
|
||||
}
|
||||
out = append(out, row)
|
||||
}
|
||||
@@ -301,10 +302,13 @@ func (s *Server) toolPauseTask() actool.CoreTool {
|
||||
TaskID string `json:"task_id"`
|
||||
}
|
||||
_ = json.Unmarshal(in, &a)
|
||||
if _, ok := s.m.Task(a.TaskID); !ok {
|
||||
t, ok := s.m.Task(a.TaskID)
|
||||
if !ok {
|
||||
return actool.Errorf("task 不存在: " + a.TaskID), nil
|
||||
}
|
||||
s.engine.Pause(a.TaskID)
|
||||
if _, err := s.applyTaskControlWithCause(t, "pause", agent.AbortPausedByOrchestrator); err != nil {
|
||||
return actool.Errorf(err.Error()), nil
|
||||
}
|
||||
return actool.Text("task paused: " + a.TaskID), nil
|
||||
})
|
||||
}
|
||||
@@ -414,12 +418,13 @@ func (s *Server) toolUpdateFindingReport() actool.CoreTool {
|
||||
|
||||
// deriveTaskStatus mirrors listTasks' status derivation for the list_tasks tool.
|
||||
func (s *Server) deriveTaskStatus(t *Task) string {
|
||||
lifecycle := t.lifecycleSnapshot()
|
||||
switch {
|
||||
case isTerminalStatus(t.Status):
|
||||
return t.Status
|
||||
case t.Paused || s.engine.IsPaused(t.ID):
|
||||
case isTerminalStatus(lifecycle.Status):
|
||||
return lifecycle.Status
|
||||
case lifecycle.Paused || s.engine.IsPaused(t.ID):
|
||||
return "paused"
|
||||
case s.engine.Ready() && s.engine.Started(t.ID):
|
||||
case s.engine.ReadyFor(t) && s.engine.Started(t.ID):
|
||||
return "running"
|
||||
}
|
||||
return "created"
|
||||
|
||||
+24
-15
@@ -50,6 +50,7 @@ func (sc *Scheduler) Run(ctx context.Context) {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
sc.s.reconcileConcurrency() // 并发上限:有空位就把排队任务补位启动
|
||||
sc.step()
|
||||
}
|
||||
}
|
||||
@@ -155,19 +156,22 @@ func (sc *Scheduler) fireFindings(triggers []*db.AgentTrigger) {
|
||||
want = append(want, tr)
|
||||
}
|
||||
}
|
||||
if len(want) == 0 {
|
||||
return
|
||||
}
|
||||
last, _ := strconv.ParseInt(sc.mustState(schedKeyLastFinding), 10, 64)
|
||||
events, err := sc.pg.NewFindingsSince(last)
|
||||
if err != nil || len(events) == 0 {
|
||||
return
|
||||
}
|
||||
// Advance the watermark whether or not any on_finding trigger is active: a
|
||||
// finding fires only for triggers live at the moment it appears. Otherwise a
|
||||
// trigger enabled later would replay the entire historical backlog at once.
|
||||
maxID := last
|
||||
for _, e := range events {
|
||||
if e.NodeID > maxID {
|
||||
maxID = e.NodeID
|
||||
}
|
||||
if len(want) == 0 {
|
||||
continue
|
||||
}
|
||||
msgCtx := fmt.Sprintf("\n\n【本次由任务发现 finding 触发】\n任务: #%d %s(目标:%s)\n发现: [%s/%s] %s",
|
||||
e.TaskID, e.TaskDesc, e.TaskGoal, e.VulnClass, e.Severity, e.Summary)
|
||||
for _, tr := range want {
|
||||
@@ -185,13 +189,12 @@ func (sc *Scheduler) fireGoals(triggers []*db.AgentTrigger) {
|
||||
want = append(want, tr)
|
||||
}
|
||||
}
|
||||
if len(want) == 0 {
|
||||
return
|
||||
}
|
||||
events, err := sc.pg.MetGoals()
|
||||
if err != nil || len(events) == 0 {
|
||||
return
|
||||
}
|
||||
// Mark goals as consumed whether or not a trigger is active, so enabling an
|
||||
// on_goal_met trigger later doesn't replay every already-met goal.
|
||||
fired := sc.firedGoalSet()
|
||||
changed := false
|
||||
for _, e := range events {
|
||||
@@ -200,6 +203,9 @@ func (sc *Scheduler) fireGoals(triggers []*db.AgentTrigger) {
|
||||
}
|
||||
fired[e.NodeID] = true
|
||||
changed = true
|
||||
if len(want) == 0 {
|
||||
continue
|
||||
}
|
||||
msgCtx := fmt.Sprintf("\n\n【本次由任务完成目标触发】\n任务: #%d %s(目标:%s)\n达成目标: %s",
|
||||
e.TaskID, e.TaskDesc, e.TaskGoal, e.Summary)
|
||||
for _, tr := range want {
|
||||
@@ -220,19 +226,20 @@ func (sc *Scheduler) fireTaskTimeouts(triggers []*db.AgentTrigger) {
|
||||
want = append(want, tr)
|
||||
}
|
||||
}
|
||||
if len(want) == 0 {
|
||||
return
|
||||
}
|
||||
last, _ := strconv.ParseInt(sc.mustState(schedKeyLastTimeout), 10, 64)
|
||||
events, err := sc.pg.TimedOutTasksSince(last)
|
||||
if err != nil || len(events) == 0 {
|
||||
return
|
||||
}
|
||||
// Advance the watermark even with no active trigger — see fireFindings.
|
||||
maxID := last
|
||||
for _, e := range events {
|
||||
if e.NodeID > maxID {
|
||||
maxID = e.NodeID
|
||||
}
|
||||
if len(want) == 0 {
|
||||
continue
|
||||
}
|
||||
msgCtx := fmt.Sprintf("\n\n【本次由任务超时触发】\n任务: #%d %s(目标:%s)",
|
||||
e.TaskID, e.TaskDesc, e.TaskGoal)
|
||||
for _, tr := range want {
|
||||
@@ -251,19 +258,20 @@ func (sc *Scheduler) fireTaskCreates(triggers []*db.AgentTrigger) {
|
||||
want = append(want, tr)
|
||||
}
|
||||
}
|
||||
if len(want) == 0 {
|
||||
return
|
||||
}
|
||||
last, _ := strconv.ParseInt(sc.mustState(schedKeyLastTaskCreate), 10, 64)
|
||||
events, err := sc.pg.NewTasksSince(last)
|
||||
if err != nil || len(events) == 0 {
|
||||
return
|
||||
}
|
||||
// Advance the watermark even with no active trigger — see fireFindings.
|
||||
maxID := last
|
||||
for _, e := range events {
|
||||
if e.NodeID > maxID {
|
||||
maxID = e.NodeID
|
||||
}
|
||||
if len(want) == 0 {
|
||||
continue
|
||||
}
|
||||
msgCtx := fmt.Sprintf("\n\n【本次由任务创建触发】\n任务ID: #%d\n任务描述: %s\n任务目标: %s",
|
||||
e.TaskID, e.TaskDesc, e.TaskGoal)
|
||||
for _, tr := range want {
|
||||
@@ -283,19 +291,20 @@ func (sc *Scheduler) fireToolCalls(triggers []*db.AgentTrigger) {
|
||||
want = append(want, tr)
|
||||
}
|
||||
}
|
||||
if len(want) == 0 {
|
||||
return
|
||||
}
|
||||
last, _ := strconv.ParseInt(sc.mustState(schedKeyLastToolCall), 10, 64)
|
||||
events, err := sc.pg.NewToolCallsSince(last)
|
||||
if err != nil || len(events) == 0 {
|
||||
return
|
||||
}
|
||||
// Advance the watermark even with no active trigger — see fireFindings.
|
||||
maxID := last
|
||||
for _, e := range events {
|
||||
if e.NodeID > maxID {
|
||||
maxID = e.NodeID
|
||||
}
|
||||
if len(want) == 0 {
|
||||
continue
|
||||
}
|
||||
errTag := ""
|
||||
if e.ToolIsErr {
|
||||
errTag = "[error] "
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestWorkerControlRoutes(t *testing.T) {
|
||||
m, err := NewManager(t.TempDir(), "")
|
||||
if err != nil {
|
||||
t.Skipf("postgres unavailable (%v) - skipping", err)
|
||||
}
|
||||
defer m.Close()
|
||||
|
||||
td := t.TempDir()
|
||||
s := New(context.Background(), m, td, td, td)
|
||||
h := s.Handler()
|
||||
token, err := signJWT(s.jwtKey)
|
||||
if err != nil {
|
||||
t.Fatalf("sign jwt: %v", err)
|
||||
}
|
||||
|
||||
request := func(path string) *httptest.ResponseRecorder {
|
||||
req := httptest.NewRequest(http.MethodPost, path, strings.NewReader(`{"action":"pause"}`))
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
rec := httptest.NewRecorder()
|
||||
h.ServeHTTP(rec, req)
|
||||
return rec
|
||||
}
|
||||
|
||||
// The single-worker route is still registered and reaches its JSON handler.
|
||||
single := request("/api/tasks/missing/intents/1/control")
|
||||
if single.Code != http.StatusNotFound || !strings.Contains(single.Body.String(), `"error":"task not found"`) {
|
||||
t.Fatalf("single control route unavailable: status=%d body=%s", single.Code, single.Body.String())
|
||||
}
|
||||
|
||||
// The former batch route must fall through the mux instead of reaching a task handler.
|
||||
batch := request("/api/tasks/missing/intents/control/batch")
|
||||
if batch.Code != http.StatusNotFound || !strings.Contains(batch.Body.String(), "404 page not found") {
|
||||
t.Fatalf("batch control route still registered: status=%d body=%s", batch.Code, batch.Body.String())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,364 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Autumn-27/artex/agent"
|
||||
)
|
||||
|
||||
func TestEngineDeleteBarrierRejectsNewTaskOperations(t *testing.T) {
|
||||
e := NewEngine(nil)
|
||||
const taskID = "42"
|
||||
|
||||
if !e.beginTaskOperation(taskID) {
|
||||
t.Fatal("first task operation should be admitted")
|
||||
}
|
||||
if !e.BeginDelete(taskID) {
|
||||
t.Fatal("first delete should install the barrier")
|
||||
}
|
||||
if !e.IsDeleting(taskID) {
|
||||
t.Fatal("delete barrier should be visible")
|
||||
}
|
||||
if !e.IsPaused(taskID) {
|
||||
t.Fatal("delete should temporarily pause task execution")
|
||||
}
|
||||
if e.beginTaskOperation(taskID) {
|
||||
t.Fatal("operation started after BeginDelete returned")
|
||||
}
|
||||
if got := e.inflightCount(taskID); got != 1 {
|
||||
t.Fatalf("existing operation must remain drainable, inflight=%d", got)
|
||||
}
|
||||
e.decInflight(taskID)
|
||||
|
||||
if e.BeginDelete(taskID) {
|
||||
t.Fatal("duplicate delete must not acquire the barrier")
|
||||
}
|
||||
e.AbortDelete(taskID, false)
|
||||
if e.IsPaused(taskID) {
|
||||
t.Fatal("aborted delete should restore a previously running task")
|
||||
}
|
||||
if !e.beginTaskOperation(taskID) {
|
||||
t.Fatal("aborting deletion should reopen operation admission")
|
||||
}
|
||||
e.decInflight(taskID)
|
||||
}
|
||||
|
||||
func TestAbortDeletePreservesExistingPause(t *testing.T) {
|
||||
e := NewEngine(nil)
|
||||
e.Pause("42", agent.AbortPausedByUser)
|
||||
if !e.BeginDelete("42") {
|
||||
t.Fatal("delete barrier should be installed")
|
||||
}
|
||||
e.AbortDelete("42", true)
|
||||
if !e.IsPaused("42") {
|
||||
t.Fatal("aborted delete must preserve a pre-existing user pause")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBeginTaskDeleteSerializesWithLifecycleLock(t *testing.T) {
|
||||
s := &Server{engine: NewEngine(nil)}
|
||||
s.concMu.Lock()
|
||||
done := make(chan bool, 1)
|
||||
go func() { done <- s.beginTaskDelete("42") }()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
t.Fatal("delete barrier bypassed the lifecycle lock")
|
||||
case <-time.After(20 * time.Millisecond):
|
||||
}
|
||||
if s.engine.IsDeleting("42") {
|
||||
t.Fatal("delete barrier was installed before concMu was released")
|
||||
}
|
||||
s.concMu.Unlock()
|
||||
select {
|
||||
case ok := <-done:
|
||||
if !ok {
|
||||
t.Fatal("delete barrier was not installed after lifecycle lock release")
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("timed out waiting for delete barrier")
|
||||
}
|
||||
s.engine.AbortDelete("42", false)
|
||||
}
|
||||
|
||||
func TestDeletingRunningTaskKeepsConcurrencySlotUntilCommit(t *testing.T) {
|
||||
task := &Task{ID: "42", Status: "running"}
|
||||
s := &Server{engine: NewEngine(nil)}
|
||||
if !s.engine.BeginDelete(task.ID) {
|
||||
t.Fatal("install delete barrier")
|
||||
}
|
||||
if !s.occupiesConcurrencySlot(task) {
|
||||
t.Fatal("temporary delete pause released the running task's slot before commit")
|
||||
}
|
||||
s.engine.AbortDelete(task.ID, false)
|
||||
}
|
||||
|
||||
func TestTaskLifecycleRechecksDeleteBarrierAfterConcLock(t *testing.T) {
|
||||
task := &Task{ID: "42", notify: make(chan struct{}, 1)}
|
||||
m := &Manager{tasks: map[string]*Task{task.ID: task}}
|
||||
s := &Server{m: m, engine: NewEngine(m)}
|
||||
|
||||
s.concMu.Lock()
|
||||
started := make(chan struct{})
|
||||
pauseDone := make(chan error, 1)
|
||||
go func() {
|
||||
close(started)
|
||||
_, err := s.applyTaskControl(task, "pause")
|
||||
pauseDone <- err
|
||||
}()
|
||||
<-started
|
||||
if !s.engine.BeginDelete(task.ID) {
|
||||
t.Fatal("install delete barrier")
|
||||
}
|
||||
s.concMu.Unlock()
|
||||
if err := <-pauseDone; err == nil || !strings.Contains(err.Error(), "正在删除") {
|
||||
t.Fatalf("pause passed a delete barrier installed while waiting for concMu: %v", err)
|
||||
}
|
||||
s.engine.AbortDelete(task.ID, false)
|
||||
|
||||
s.concMu.Lock()
|
||||
started = make(chan struct{})
|
||||
admitDone := make(chan error, 1)
|
||||
go func() {
|
||||
close(started)
|
||||
_, err := s.admitTask(task, "resume")
|
||||
admitDone <- err
|
||||
}()
|
||||
<-started
|
||||
if !s.engine.BeginDelete(task.ID) {
|
||||
t.Fatal("install second delete barrier")
|
||||
}
|
||||
s.concMu.Unlock()
|
||||
if err := <-admitDone; err == nil || !strings.Contains(err.Error(), "being deleted") {
|
||||
t.Fatalf("admission passed a delete barrier installed while waiting for concMu: %v", err)
|
||||
}
|
||||
s.engine.AbortDelete(task.ID, false)
|
||||
}
|
||||
|
||||
func TestAbortTaskDeleteUsesPersistedPauseAndQueueState(t *testing.T) {
|
||||
m, err := NewManager(t.TempDir(), "")
|
||||
if err != nil {
|
||||
t.Skipf("postgres unavailable (%v) - skipping", err)
|
||||
}
|
||||
defer m.Close()
|
||||
task, err := m.CreateTask("delete abort state", "restore persisted barrier", nil, 0, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _, _ = m.DeleteTask(task.ID, DeleteTaskOptions{}) }()
|
||||
s := &Server{m: m, engine: NewEngine(m)}
|
||||
|
||||
// An engine-only pause is not authoritative. The committed task is running,
|
||||
// so aborting deletion must clear this stale in-memory barrier.
|
||||
s.engine.Pause(task.ID, agent.AbortPausedByUser)
|
||||
if !s.beginTaskDelete(task.ID) {
|
||||
t.Fatal("install first delete barrier")
|
||||
}
|
||||
s.abortTaskDelete(task.ID)
|
||||
if s.engine.IsPaused(task.ID) {
|
||||
t.Fatal("stale engine pause survived although the persisted task is running")
|
||||
}
|
||||
|
||||
// A queued row must retain its execution barrier even when BeginDelete saw the
|
||||
// engine as running. This specifically guards against restoring a stale snapshot.
|
||||
if err := m.EnqueueTask(task.ID, "resume"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !s.beginTaskDelete(task.ID) {
|
||||
t.Fatal("install second delete barrier")
|
||||
}
|
||||
s.abortTaskDelete(task.ID)
|
||||
if !s.engine.IsPaused(task.ID) {
|
||||
t.Fatal("persisted queued task lost its execution barrier after abort")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStopTaskCancelsRuntimeAndClearsLifecycleState(t *testing.T) {
|
||||
e := NewEngine(nil)
|
||||
const taskID = "42"
|
||||
e.deleteMu.RLock()
|
||||
rt := e.registerTaskRoutines(context.Background(), taskID, 1)
|
||||
e.started.Store(taskID, true)
|
||||
e.lastAct.Store(taskID, int64(1))
|
||||
e.paused.Store(taskID, true)
|
||||
e.plannerRound.Store(taskID, 3)
|
||||
e.coordStarted.Store(taskID, true)
|
||||
e.deleteMu.RUnlock()
|
||||
|
||||
done := make(chan struct{})
|
||||
runTaskRoutine(rt, func(ctx context.Context) {
|
||||
<-ctx.Done()
|
||||
close(done)
|
||||
})
|
||||
e.StopTask(taskID)
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
default:
|
||||
t.Fatal("task runtime was not cancelled before StopTask returned")
|
||||
}
|
||||
if e.Started(taskID) || e.IsPaused(taskID) || e.IsDeleting(taskID) {
|
||||
t.Fatalf("task lifecycle maps were not cleared: started=%v paused=%v deleting=%v",
|
||||
e.Started(taskID), e.IsPaused(taskID), e.IsDeleting(taskID))
|
||||
}
|
||||
if _, ok := e.lastAct.Load(taskID); ok {
|
||||
t.Fatal("last activity state was retained")
|
||||
}
|
||||
if _, ok := e.plannerRound.Load(taskID); ok {
|
||||
t.Fatal("planner round state was retained")
|
||||
}
|
||||
if _, ok := e.coordStarted.Load(taskID); ok {
|
||||
t.Fatal("deadline coordinator state was retained")
|
||||
}
|
||||
e.runtimeMu.Lock()
|
||||
_, retained := e.runtimes[taskID]
|
||||
e.runtimeMu.Unlock()
|
||||
if retained {
|
||||
t.Fatal("task runtime retained after StopTask")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCanonicalTaskID(t *testing.T) {
|
||||
for raw, want := range map[string]string{"42": "42", "00042": "42", "+42": "42"} {
|
||||
if got, ok := canonicalTaskID(raw); !ok || got != want {
|
||||
t.Fatalf("canonicalTaskID(%q)=(%q,%v), want (%q,true)", raw, got, ok, want)
|
||||
}
|
||||
}
|
||||
for _, raw := range []string{"", "0", "-1", "abc", "42x"} {
|
||||
if got, ok := canonicalTaskID(raw); ok {
|
||||
t.Fatalf("canonicalTaskID(%q)=(%q,true), want invalid", raw, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCancelChatThenWaitTaskQuiescent(t *testing.T) {
|
||||
e := NewEngine(nil)
|
||||
chatCtx, chatCancel := context.WithCancelCause(context.Background())
|
||||
s := &Server{
|
||||
engine: e,
|
||||
chatBusy: map[string]bool{"7": true},
|
||||
chatCancel: map[string]context.CancelCauseFunc{"7": chatCancel},
|
||||
}
|
||||
if !e.beginTaskOperation("7") {
|
||||
t.Fatal("task operation should be admitted")
|
||||
}
|
||||
|
||||
cancelObserved := make(chan struct{})
|
||||
releaseChat := make(chan struct{})
|
||||
go func() {
|
||||
<-chatCtx.Done()
|
||||
close(cancelObserved)
|
||||
<-releaseChat
|
||||
s.finishTaskChat("7", chatCancel)
|
||||
}()
|
||||
if !s.cancelTaskChat("7", agent.AbortTaskDeleted) {
|
||||
t.Fatal("expected active chat to be cancelled")
|
||||
}
|
||||
<-cancelObserved
|
||||
|
||||
waitDone := make(chan error, 1)
|
||||
waitCtx, waitCancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer waitCancel()
|
||||
go func() { waitDone <- s.waitTaskQuiescent(waitCtx, "7") }()
|
||||
|
||||
select {
|
||||
case err := <-waitDone:
|
||||
t.Fatalf("wait returned while chat and engine operation were active: %v", err)
|
||||
case <-time.After(20 * time.Millisecond):
|
||||
}
|
||||
close(releaseChat)
|
||||
select {
|
||||
case err := <-waitDone:
|
||||
t.Fatalf("wait returned while engine operation was active: %v", err)
|
||||
case <-time.After(20 * time.Millisecond):
|
||||
}
|
||||
e.decInflight("7")
|
||||
if err := <-waitDone; err != nil {
|
||||
t.Fatalf("wait for quiescence: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWaitTaskQuiescentHonorsContext(t *testing.T) {
|
||||
s := &Server{
|
||||
engine: NewEngine(nil),
|
||||
chatBusy: map[string]bool{"9": true},
|
||||
chatCancel: map[string]context.CancelCauseFunc{},
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
|
||||
defer cancel()
|
||||
if err := s.waitTaskQuiescent(ctx, "9"); !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("expected deadline exceeded, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTaskChatStatusUsesAuthoritativeBusyState(t *testing.T) {
|
||||
m := &Manager{tasks: map[string]*Task{"7": {ID: "7"}}}
|
||||
s := &Server{
|
||||
m: m,
|
||||
chatBusy: map[string]bool{"7": true},
|
||||
chatCancel: map[string]context.CancelCauseFunc{},
|
||||
}
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/tasks/7/chat/status", nil)
|
||||
req.SetPathValue("id", "7")
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
s.taskChatStatus(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status=%d body=%s, want 200", rec.Code, rec.Body.String())
|
||||
}
|
||||
var got struct {
|
||||
Running bool `json:"running"`
|
||||
}
|
||||
if err := json.NewDecoder(rec.Body).Decode(&got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !got.Running {
|
||||
t.Fatal("busy main-agent turn was reported idle")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTaskChatUploadRejectsDeleteBarrierBeforeCreatingDirectory(t *testing.T) {
|
||||
dataDir := t.TempDir()
|
||||
m := &Manager{dir: dataDir, tasks: map[string]*Task{"7": {ID: "7"}}}
|
||||
s := &Server{m: m, engine: NewEngine(m)}
|
||||
if !s.engine.BeginDelete("7") {
|
||||
t.Fatal("delete barrier should be installed")
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/chat/upload?scope=task&id=7", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
s.chatUpload(rec, req)
|
||||
if rec.Code != http.StatusConflict {
|
||||
t.Fatalf("expected 409 while deleting, got %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dataDir, "tasks", "7")); !os.IsNotExist(err) {
|
||||
t.Fatalf("upload must not recreate task directory, stat err=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCommittedTaskDeleteReturnsSuccessWithCleanupWarning(t *testing.T) {
|
||||
rec := httptest.NewRecorder()
|
||||
result := DeleteTaskResult{Deleted: "7", FilesDeleted: true}
|
||||
err := &taskDeleteCommittedError{err: errors.New("remove staged task files: permission denied")}
|
||||
|
||||
writeCommittedTaskDelete(rec, result, err)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status=%d body=%s, want 200", rec.Code, rec.Body.String())
|
||||
}
|
||||
var got DeleteTaskResult
|
||||
if decodeErr := json.NewDecoder(rec.Body).Decode(&got); decodeErr != nil {
|
||||
t.Fatal(decodeErr)
|
||||
}
|
||||
if got.Deleted != "7" || !got.FilesDeleted || !strings.Contains(got.CleanupWarning, "cleanup incomplete") {
|
||||
t.Fatalf("unexpected response: %+v", got)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user