feat(tasks): add inherited context and resilient LLM execution

Add read-only task context inheritance, ordered task-level LLM failover, live Worker controls, cascading task cleanup, FIFO task concurrency, cancellation diagnostics, UI updates, tests, build script, and changelog.
This commit is contained in:
pphua committed 2026-08-20 18:34:24 +08:00
1 parent 7090b034f8
commit 9793b522d9
82 files changed
+11094 -1712

No files matched your search

+129 -53
View File
@@ -5,6 +5,7 @@ import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"log"
@@ -66,61 +67,97 @@ func decode(r *http.Request, v any) error { return json.NewDecoder(r.Body).Decod
// ---------- tasks (delete) ----------
// pgDeleteTask removes a task and, per opt-in flags in the (optional) JSON body,
// its associated data: assets (task-owned), findings, recorded traffic (by the
// task's hosts) and the per-task working directory. Empty body → all flags false →
// only the task + its exploration subgraph are removed (legacy behavior).
const taskDeleteDrainTimeout = 10 * time.Second
func canonicalTaskID(raw string) (string, bool) {
n, err := strconv.ParseInt(raw, 10, 64)
if err != nil || n <= 0 {
return "", false
}
return strconv.FormatInt(n, 10), true
}
func (s *Server) pgDeleteTask(w http.ResponseWriter, r *http.Request) {
id := r.PathValue("id")
var opts struct {
Assets bool `json:"assets"`
Findings bool `json:"findings"`
Traffic bool `json:"traffic"`
Files bool `json:"files"`
id, ok := canonicalTaskID(r.PathValue("id"))
if !ok {
writeErr(w, http.StatusBadRequest, "任务 id 无效")
return
}
_ = json.NewDecoder(r.Body).Decode(&opts) // optional body; ignore EOF/parse errors
result := map[string]any{"deleted": id}
nid, _ := strconv.ParseInt(id, 10, 64)
as := s.m.Assets()
// Traffic first: it needs the task's hosts, which come from assets that a later
// asset-delete would remove. Traffic is host-keyed (no per-task tag), so this
// purges ALL recorded traffic for those hosts, not just this task's.
if opts.Traffic && nid > 0 && as != nil {
if tr := s.m.Traffic(); tr != nil {
if hosts, err := as.HostsByTask(nid); err == nil && len(hosts) > 0 {
if n, err := tr.DeleteHostsExact(hosts); err == nil {
result["traffic_deleted"] = n
}
}
var opts DeleteTaskOptions
if err := decode(r, &opts); err != nil && err != io.EOF {
writeErr(w, 400, "invalid JSON: "+err.Error())
return
}
if !s.engine.BeginDelete(id) {
writeErr(w, http.StatusConflict, "任务正在删除")
return
}
deleted := false
defer func() {
if !deleted {
s.engine.AbortDelete(id)
}
}
if opts.Findings && nid > 0 {
if n, err := s.m.pg.DeleteFindingsByTask(nid); err == nil {
result["findings_deleted"] = n
}
}
if opts.Assets && nid > 0 && as != nil {
if n, err := as.DeleteByTaskID(nid); err == nil {
result["assets_deleted"] = n
}
}
}()
s.engine.Pause(id, agent.AbortTaskDeleted) // cancel running loops before removing
if err := s.m.DeleteTask(id); err != nil {
writeErr(w, 500, err.Error())
// Main-agent runs use a separate context from planner/workers. Cancel it, then
// wait until both execution domains have returned before removing transcripts.
s.cancelTaskChat(id, agent.AbortTaskDeleted)
drainCtx, cancelDrain := context.WithTimeout(r.Context(), taskDeleteDrainTimeout)
defer cancelDrain()
if err := s.waitTaskQuiescent(drainCtx, id); err != nil {
writeErr(w, http.StatusConflict, "任务仍有运行中的 Agent,删除已取消")
return
}
// Files last: the per-task working dir <workDir>/tasks/<id> (worker artifacts +
// uploads). Confined to the work dir; id is numeric so no traversal risk.
if opts.Files && nid > 0 {
if err := os.RemoveAll(filepath.Join(s.m.dir, "tasks", id)); err == nil {
result["files_deleted"] = true
result, err := s.m.DeleteTask(id, opts)
if err != nil {
var committed *taskDeleteCommittedError
if errors.As(err, &committed) {
// PostgreSQL is already gone. Complete runtime teardown and return the
// auditable counts together with the post-commit cleanup warning.
s.engine.StopTask(id)
s.taskAgentMu.Lock()
delete(s.taskAgents, id)
s.taskAgentMu.Unlock()
deleted = true
writeCommittedTaskDelete(w, result, err)
return
}
writeErr(w, 500, err.Error())
return
}
// Manager has removed the task from the registry, so no new API operation can
// resolve it. Now stop and join every task-owned Engine goroutine and clear its
// lifecycle maps before releasing the delete barrier.
s.engine.StopTask(id)
s.taskAgentMu.Lock()
delete(s.taskAgents, id)
s.taskAgentMu.Unlock()
deleted = true
writeJSON(w, 200, result)
}
func writeCommittedTaskDelete(w http.ResponseWriter, result DeleteTaskResult, err error) {
result.CleanupWarning = err.Error()
writeJSON(w, http.StatusOK, result)
}
func (s *Server) waitTaskQuiescent(ctx context.Context, taskID string) error {
ticker := time.NewTicker(10 * time.Millisecond)
defer ticker.Stop()
for {
s.chatMu.Lock()
chatBusy := s.chatBusy[taskID]
s.chatMu.Unlock()
if s.engine.inflightCount(taskID) == 0 && !chatBusy {
return nil
}
select {
case <-ctx.Done():
return ctx.Err()
case <-ticker.C:
}
}
writeJSON(w, 200, result)
}
// ---------- agents ----------
@@ -352,6 +389,9 @@ func (s *Server) pgSaveAgentConfig(w http.ResponseWriter, r *http.Request) {
if profileChanged {
s.invalidateProfileAgents()
}
// Task bundles capture operational Agent settings (turn/time budgets, tools,
// web search). Rebuild them even when no global profile is configured.
s.invalidateTaskAgents()
// rebuild the live agents so the new max_turns/run_seconds/binding apply without a restart.
s.cfgMu.Lock()
cfg, on := s.llmCfg, s.llmOn
@@ -396,8 +436,8 @@ func (s *Server) pgGetAgent(w http.ResponseWriter, r *http.Request) {
}
writeJSON(w, 200, map[string]any{
"agent": agentDTO(a), "prompt": cur, "variables": vars, "versions": vers,
"visibility": map[string]any{"mcp": mcp, "skill": sk},
"llm_profiles": llmProfiles, // 可绑定的 LLM 配置候选
"visibility": map[string]any{"mcp": mcp, "skill": sk},
"llm_profiles": llmProfiles, // 可绑定的 LLM 配置候选
"wrapup_prompt": a.WrapupPrompt, // 已保存的收尾提示词(空=用内置默认)
"wrapup_default": agent.WrapupDefault(a.Key), // 内置默认(供占位/恢复默认)
@@ -1607,7 +1647,9 @@ func (s *Server) pgSaveProfile(w http.ResponseWriter, r *http.Request) {
writeErr(w, 500, err.Error())
return
}
// Editing a profile rebuilds any task pinned to it on its next round.
// Editing a profile rebuilds any task pinned to it on its next round. Reapply
// the active profile too: the global fallback and explicit task chains must
// repopulate the same provider-cache entry and therefore share one limiter.
s.invalidateProfileAgents()
// Hot-apply. Not just when the ACTIVE profile changed: with failover on, any
// profile's key/model/priority/pool_exclude edit reshapes the chain, so the
@@ -1622,16 +1664,50 @@ func (s *Server) pgDeleteProfile(w http.ResponseWriter, r *http.Request) {
return
}
id, _ := pathInt(r, "id")
if err := pg.DeleteProfile(id); err != nil {
writeErr(w, 500, err.Error())
if err := pg.DeleteProfileContext(r.Context(), id); err != nil {
switch {
case errors.Is(err, db.ErrActiveLLMProfileDelete):
writeErr(w, 409, "当前激活的 LLM 配置不能删除,请先激活其他配置")
case errors.Is(err, db.ErrLLMProfileReferencesChanged):
writeErr(w, 409, "LLM 配置正在被任务或会话修改,请重试")
case errors.Is(err, context.DeadlineExceeded):
writeErr(w, 409, "等待 LLM 配置引用释放超时,请重试")
case errors.Is(err, db.ErrLLMProfileNotFound):
writeErr(w, 404, err.Error())
default:
writeErr(w, 500, err.Error())
}
return
}
s.invalidateProfileAgents() // drop cached agents for the removed profile
s.llmHealth.Reset(id) // its breaker state is meaningless now (row is FK-cascaded away)
s.reapplyActiveProfile() // the deleted profile may have been in the failover chain
s.llmHealth.Reset(id) // its breaker state is meaningless now (row is FK-cascaded away)
s.restoreTasksAfterProfileDelete(pg)
// Cache invalidation also removed the active profile's shared provider entry.
// Reapply after task-state sync so the wake-up observes the post-delete chain.
s.reapplyActiveProfile()
writeJSON(w, 200, map[string]any{"deleted": id})
}
func (s *Server) restoreTasksAfterProfileDelete(pg *db.DB) {
for _, task := range s.m.List() {
if taskID, err := strconv.ParseInt(task.ID, 10, 64); err == nil {
if pt, err := pg.GetTask(taskID); err == nil && pt != nil {
s.syncTaskLLMState(pt)
// Deleting the active entry may advance the task to a ready successor,
// or clear the explicit chain and restore its Agent/global fallback.
// Resume only intents that were blocked by the exhausted chain; a
// paused task remains paused and terminal tasks remain immutable.
if !isTerminalStatus(task.Status) && s.taskRuntimeAvailable(task, "planner", "worker") {
if _, reopenErr := task.Store.ReopenIntentsByBlockedReason(db.IntentBlockedLLMQuota); reopenErr != nil {
log.Printf("[llm-profile] task %s reopen quota-blocked intents after profile delete: %v", task.ID, reopenErr)
}
}
task.Notify()
}
}
}
}
func (s *Server) pgActivateProfile(w http.ResponseWriter, r *http.Request) {
pg := s.pg(w)
if pg == nil {