feat(tasks): add inherited context and resilient LLM execution
Add read-only task context inheritance, ordered task-level LLM failover, live Worker controls, cascading task cleanup, FIFO task concurrency, cancellation diagnostics, UI updates, tests, build script, and changelog.
This commit is contained in:
pphua committed
1 parent
7090b034f8
commit
9793b522d9
82 files changed
+11094
-1712
No files matched your search
@@ -0,0 +1,462 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"iter"
|
||||
"log"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Autumn-27/artex/agent"
|
||||
"github.com/Autumn-27/artex/db"
|
||||
"github.com/Autumn-27/artex/llmrec"
|
||||
"github.com/Autumn-27/norma/llm"
|
||||
"github.com/Autumn-27/norma/memory"
|
||||
"github.com/Autumn-27/norma/transcript"
|
||||
)
|
||||
|
||||
type taskAgentBundle struct {
|
||||
runtime *taskLLMRuntime // goal decomposition runtime
|
||||
plannerRuntime *taskLLMRuntime
|
||||
workerRuntime *taskLLMRuntime
|
||||
mainRuntime *taskLLMRuntime
|
||||
pl *agent.Planner
|
||||
wk *agent.Worker
|
||||
main *agent.MainAgent
|
||||
}
|
||||
|
||||
type llmAuditProfile struct {
|
||||
ID int64 `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Format string `json:"format"`
|
||||
Model string `json:"model"`
|
||||
}
|
||||
|
||||
type llmTransitionAudit struct {
|
||||
Mode string `json:"mode"` // automatic | manual | exhausted
|
||||
Reason string `json:"reason"`
|
||||
Previous *llmAuditProfile `json:"previous,omitempty"`
|
||||
Next *llmAuditProfile `json:"next,omitempty"`
|
||||
}
|
||||
|
||||
type llmActivityMetadata struct {
|
||||
LLMTransition llmTransitionAudit `json:"llm_transition"`
|
||||
}
|
||||
|
||||
type taskLLMRuntime struct {
|
||||
s *Server
|
||||
taskID string
|
||||
agentKey string
|
||||
}
|
||||
|
||||
type taskLLMError struct {
|
||||
taskID string
|
||||
chainExhausted bool
|
||||
cause error
|
||||
}
|
||||
|
||||
func (e *taskLLMError) Error() string {
|
||||
if e.chainExhausted {
|
||||
return fmt.Sprintf("task %s LLM profile chain exhausted: %v", e.taskID, e.cause)
|
||||
}
|
||||
return e.cause.Error()
|
||||
}
|
||||
|
||||
func (e *taskLLMError) Unwrap() error { return e.cause }
|
||||
|
||||
func isTaskLLMRuntimeError(err error) bool {
|
||||
var target *taskLLMError
|
||||
return errors.As(err, &target)
|
||||
}
|
||||
|
||||
func isTaskLLMChainExhausted(err error) bool {
|
||||
var target *taskLLMError
|
||||
return errors.As(err, &target) && target.chainExhausted
|
||||
}
|
||||
|
||||
// isQuotaExhaustedError is intentionally strict. A generic 429, auth error,
|
||||
// network failure, or 5xx does not rotate providers; the response must explicitly
|
||||
// identify quota, credits, billing balance, or payment exhaustion.
|
||||
func isQuotaExhaustedError(err error) bool {
|
||||
return err != nil && agent.IsQuotaExhaustedMessage(err.Error())
|
||||
}
|
||||
|
||||
type taskLLMSelection struct {
|
||||
task *Task
|
||||
profileID int64
|
||||
revision int64
|
||||
provider llm.Provider
|
||||
}
|
||||
|
||||
type taskLLMStreamHooks struct {
|
||||
current func() (taskLLMSelection, error)
|
||||
exhaust func(taskLLMSelection, error) (db.TaskLLMTransition, error)
|
||||
transition func(taskLLMSelection, db.TaskLLMTransition, error)
|
||||
}
|
||||
|
||||
func (r *taskLLMRuntime) current() (*Task, int64, int64, llm.Provider, error) {
|
||||
taskNum, err := parseTaskID(r.taskID)
|
||||
if err != nil {
|
||||
return nil, 0, 0, nil, err
|
||||
}
|
||||
pt, err := r.s.m.pg.GetTask(taskNum)
|
||||
if err != nil {
|
||||
return nil, 0, 0, nil, err
|
||||
}
|
||||
if pt == nil {
|
||||
return nil, 0, 0, nil, fmt.Errorf("task %s not found", r.taskID)
|
||||
}
|
||||
r.s.syncTaskLLMState(pt)
|
||||
t, _ := r.s.m.Task(r.taskID)
|
||||
if len(pt.LLMProfileIDs) == 0 {
|
||||
prov, _, ok := r.s.fallbackProviderForAgent(r.agentKey)
|
||||
if !ok {
|
||||
return t, 0, pt.LLMChainRevision, nil, fmt.Errorf("task %s has no available fallback LLM provider", r.taskID)
|
||||
}
|
||||
return t, 0, pt.LLMChainRevision, prov, nil
|
||||
}
|
||||
if pt.ActiveLLMProfileID == nil {
|
||||
return t, 0, pt.LLMChainRevision, nil, &taskLLMError{taskID: r.taskID, chainExhausted: true, cause: errors.New("all selected profiles are quota exhausted")}
|
||||
}
|
||||
prov, _, ok := r.s.providerForProfile(*pt.ActiveLLMProfileID)
|
||||
if !ok {
|
||||
return t, *pt.ActiveLLMProfileID, pt.LLMChainRevision, nil, fmt.Errorf("LLM profile #%d is missing or invalid", *pt.ActiveLLMProfileID)
|
||||
}
|
||||
return t, *pt.ActiveLLMProfileID, pt.LLMChainRevision, prov, nil
|
||||
}
|
||||
|
||||
func parseTaskID(id string) (int64, error) {
|
||||
n, err := strconv.ParseInt(id, 10, 64)
|
||||
if err != nil || n <= 0 {
|
||||
return 0, fmt.Errorf("invalid task id %q", id)
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func (r *taskLLMRuntime) Stream(ctx context.Context, req llm.CompletionRequest) iter.Seq2[llm.StreamEvent, error] {
|
||||
ctx = llmrec.WithTaskID(ctx, r.taskID)
|
||||
hooks := taskLLMStreamHooks{
|
||||
current: func() (taskLLMSelection, error) {
|
||||
task, profileID, revision, provider, err := r.current()
|
||||
return taskLLMSelection{task: task, profileID: profileID, revision: revision, provider: provider}, err
|
||||
},
|
||||
exhaust: func(selection taskLLMSelection, cause error) (db.TaskLLMTransition, error) {
|
||||
taskNum, _ := parseTaskID(r.taskID)
|
||||
transition, err := r.s.m.pg.MarkTaskLLMProfileQuotaExhaustedAtRevision(taskNum, selection.profileID, selection.revision, cause.Error())
|
||||
if err != nil {
|
||||
return transition, err
|
||||
}
|
||||
if pt, getErr := r.s.m.pg.GetTask(taskNum); getErr == nil && pt != nil {
|
||||
r.s.syncTaskLLMState(pt)
|
||||
}
|
||||
return transition, nil
|
||||
},
|
||||
transition: func(selection taskLLMSelection, transition db.TaskLLMTransition, cause error) {
|
||||
r.s.emitTaskLLMTransition(selection.task, transition, cause)
|
||||
},
|
||||
}
|
||||
return streamTaskLLM(ctx, r.taskID, req, hooks)
|
||||
}
|
||||
|
||||
func streamTaskLLM(ctx context.Context, taskID string, req llm.CompletionRequest, hooks taskLLMStreamHooks) iter.Seq2[llm.StreamEvent, error] {
|
||||
return func(yield func(llm.StreamEvent, error) bool) {
|
||||
for {
|
||||
selection, err := hooks.current()
|
||||
if err != nil {
|
||||
yield(llm.StreamEvent{}, err)
|
||||
return
|
||||
}
|
||||
committed := false
|
||||
var pending []llm.StreamEvent
|
||||
var streamErr error
|
||||
for event, err := range selection.provider.Stream(ctx, req) {
|
||||
if err != nil {
|
||||
streamErr = err
|
||||
break
|
||||
}
|
||||
if !committed && !streamEventCommitsOutput(event) {
|
||||
pending = append(pending, event)
|
||||
continue
|
||||
}
|
||||
if !committed {
|
||||
for _, buffered := range pending {
|
||||
if !yield(buffered, nil) {
|
||||
return
|
||||
}
|
||||
}
|
||||
pending = nil
|
||||
committed = true
|
||||
}
|
||||
if !yield(event, nil) {
|
||||
return
|
||||
}
|
||||
}
|
||||
if streamErr == nil {
|
||||
for _, buffered := range pending {
|
||||
if !yield(buffered, nil) {
|
||||
return
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
// profileID=0 means the explicit chain was cleared while this stable task
|
||||
// bundle was still in use. Agent/global fallback errors follow the legacy
|
||||
// behavior and never mutate task failover state.
|
||||
if selection.profileID == 0 || !isQuotaExhaustedError(streamErr) {
|
||||
for _, buffered := range pending {
|
||||
if !yield(buffered, nil) {
|
||||
return
|
||||
}
|
||||
}
|
||||
yield(llm.StreamEvent{}, streamErr)
|
||||
return
|
||||
}
|
||||
transition, markErr := hooks.exhaust(selection, streamErr)
|
||||
if markErr != nil {
|
||||
cause := fmt.Errorf("mark profile quota exhausted after %v: %w", streamErr, markErr)
|
||||
if committed {
|
||||
// Output may already have driven tool execution. Report the persistence
|
||||
// failure, but classify it as router-handled so the worker does not
|
||||
// replay the entire intent and duplicate those side effects.
|
||||
yield(llm.StreamEvent{}, &taskLLMError{taskID: taskID, cause: cause})
|
||||
} else {
|
||||
yield(llm.StreamEvent{}, cause)
|
||||
}
|
||||
return
|
||||
}
|
||||
if transition.Advanced && !transition.Stale && hooks.transition != nil {
|
||||
hooks.transition(selection, transition, streamErr)
|
||||
}
|
||||
if committed || (!transition.Stale && transition.NextProfileID == nil) {
|
||||
yield(llm.StreamEvent{}, &taskLLMError{taskID: taskID, chainExhausted: transition.ChainExhausted, cause: streamErr})
|
||||
return
|
||||
}
|
||||
// No event reached the caller, so replaying the same logical request on
|
||||
// the next profile cannot duplicate model output or tool execution.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func streamEventCommitsOutput(event llm.StreamEvent) bool {
|
||||
switch event.Type {
|
||||
case llm.SETextDelta, llm.SEThinkingDelta, llm.SEToolInputJSON:
|
||||
return event.Text != ""
|
||||
case llm.SEToolUseStart, llm.SEMessageDelta, llm.SEMessageStop:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (r *taskLLMRuntime) CompactionWindow() int {
|
||||
taskNum, err := parseTaskID(r.taskID)
|
||||
if err != nil {
|
||||
return (agent.Config{}).CompactionWindow()
|
||||
}
|
||||
chain, err := r.s.m.pg.TaskLLMProfiles(taskNum)
|
||||
if err != nil {
|
||||
return (agent.Config{}).CompactionWindow()
|
||||
}
|
||||
if len(chain) == 0 {
|
||||
if _, cfg, ok := r.s.fallbackProviderForAgent(r.agentKey); ok {
|
||||
return cfg.CompactionWindow()
|
||||
}
|
||||
return (agent.Config{}).CompactionWindow()
|
||||
}
|
||||
minimum := 0
|
||||
for _, entry := range chain {
|
||||
if cfg, ok := r.s.loadProfileConfig(entry.ProfileID); ok {
|
||||
window := cfg.CompactionWindow()
|
||||
if minimum == 0 || window < minimum {
|
||||
minimum = window
|
||||
}
|
||||
}
|
||||
}
|
||||
if minimum == 0 {
|
||||
return (agent.Config{}).CompactionWindow()
|
||||
}
|
||||
return minimum
|
||||
}
|
||||
|
||||
// fallbackProviderForAgent resolves the pre-existing precedence used by tasks
|
||||
// without an explicit chain: the Agent's saved binding first, then the current
|
||||
// global provider (which may come from the default profile or environment).
|
||||
func (s *Server) fallbackProviderForAgent(agentKey string) (llm.Provider, agent.Config, bool) {
|
||||
if id := s.effectiveProfileForAgent(agentKey, nil); id != nil {
|
||||
if prov, cfg, ok := s.providerForProfile(*id); ok {
|
||||
return s.poolForBinding(*id, prov, cfg), cfg, true
|
||||
}
|
||||
}
|
||||
s.cfgMu.Lock()
|
||||
defer s.cfgMu.Unlock()
|
||||
if !s.llmOn || s.llmProv == nil {
|
||||
return nil, agent.Config{}, false
|
||||
}
|
||||
return s.llmProv, s.llmCfg, true
|
||||
}
|
||||
|
||||
func (s *Server) taskRuntimeAvailable(t *Task, agentKeys ...string) bool {
|
||||
if t == nil {
|
||||
return false
|
||||
}
|
||||
state := t.llmStateSnapshot()
|
||||
if len(state.ProfileIDs) > 0 {
|
||||
if state.ActiveID == nil {
|
||||
return false
|
||||
}
|
||||
_, _, ok := s.providerForProfile(*state.ActiveID)
|
||||
return ok
|
||||
}
|
||||
s.cfgMu.Lock()
|
||||
globallyReady := s.llmOn && s.llmProv != nil
|
||||
s.cfgMu.Unlock()
|
||||
if globallyReady {
|
||||
return true
|
||||
}
|
||||
for _, key := range agentKeys {
|
||||
if _, _, ok := s.fallbackProviderForAgent(key); !ok {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return len(agentKeys) > 0
|
||||
}
|
||||
|
||||
func (s *Server) invalidateTaskAgents() {
|
||||
s.taskAgentMu.Lock()
|
||||
s.taskAgents = map[string]*taskAgentBundle{}
|
||||
s.taskAgentMu.Unlock()
|
||||
}
|
||||
|
||||
func (s *Server) agentsForTask(t *Task) *taskAgentBundle {
|
||||
s.taskAgentMu.Lock()
|
||||
defer s.taskAgentMu.Unlock()
|
||||
if bundle := s.taskAgents[t.ID]; bundle != nil {
|
||||
return bundle
|
||||
}
|
||||
goalRuntime := &taskLLMRuntime{s: s, taskID: t.ID, agentKey: "goals"}
|
||||
plannerRuntime := &taskLLMRuntime{s: s, taskID: t.ID, agentKey: "planner"}
|
||||
workerRuntime := &taskLLMRuntime{s: s, taskID: t.ID, agentKey: "worker"}
|
||||
mainRuntime := &taskLLMRuntime{s: s, taskID: t.ID, agentKey: "mainagent"}
|
||||
tx := transcript.NewStore(filepath.Join(s.m.dir, "transcripts"))
|
||||
window := workerRuntime.CompactionWindow()
|
||||
wk := agent.NewWorker(workerRuntime, "task-router", s.m.dir, tx, window, s.agentMaxTurns("worker"))
|
||||
wk.SetCompactionWindowResolver(workerRuntime.CompactionWindow)
|
||||
wk.SetRunTimeout(time.Duration(s.agentRunSeconds("worker")) * time.Second)
|
||||
wk.SetProxy(s.m.ProxyAddr(), s.m.ProxyCACert())
|
||||
wk.SetMemory(memory.NewStore(filepath.Join(s.m.dir, "memory")))
|
||||
wk.SetWebSearch(s.webSearchFor("worker"))
|
||||
pl := agent.NewPlanner(plannerRuntime, "task-router", s.m.dir, tx, plannerRuntime.CompactionWindow(), s.agentMaxTurns("planner"))
|
||||
pl.SetCompactionWindowResolver(plannerRuntime.CompactionWindow)
|
||||
pl.SetKillWork(s.engine.KillWork)
|
||||
pl.SetSteerWork(s.engine.SteerWork)
|
||||
pl.SetProxy(s.m.ProxyAddr(), s.m.ProxyCACert())
|
||||
pl.SetWebSearch(s.webSearchFor("planner"))
|
||||
main := agent.NewMainAgent(mainRuntime, "task-router", s.m.dir, tx, mainRuntime.CompactionWindow(), s.agentMaxTurns("mainagent"))
|
||||
main.SetCompactionWindowResolver(mainRuntime.CompactionWindow)
|
||||
main.SetProxy(s.m.ProxyAddr(), s.m.ProxyCACert())
|
||||
main.SetWebSearch(s.webSearchFor("mainagent"))
|
||||
bundle := &taskAgentBundle{
|
||||
runtime: goalRuntime, plannerRuntime: plannerRuntime, workerRuntime: workerRuntime,
|
||||
mainRuntime: mainRuntime, pl: pl, wk: wk, main: main,
|
||||
}
|
||||
s.taskAgents[t.ID] = bundle
|
||||
return bundle
|
||||
}
|
||||
|
||||
func (s *Server) syncTaskLLMState(pt *db.Task) {
|
||||
if pt == nil {
|
||||
return
|
||||
}
|
||||
id := fmt.Sprintf("%d", pt.ID)
|
||||
s.m.mu.Lock()
|
||||
if task := s.m.tasks[id]; task != nil {
|
||||
task.setLLMState(pt.LLMProfileID, pt.ActiveLLMProfileID, pt.LLMProfileIDs, pt.LLMFailoverState, pt.LLMFailoverReason)
|
||||
}
|
||||
s.m.mu.Unlock()
|
||||
}
|
||||
|
||||
func (s *Server) emitTaskLLMTransition(t *Task, transition db.TaskLLMTransition, cause error) {
|
||||
if t == nil {
|
||||
return
|
||||
}
|
||||
previous := s.llmAuditProfile(transition.PreviousProfileID)
|
||||
var next *llmAuditProfile
|
||||
if transition.NextProfileID != nil {
|
||||
next = s.llmAuditProfile(*transition.NextProfileID)
|
||||
}
|
||||
mode := "automatic"
|
||||
kind := "llm_switch"
|
||||
summary := fmt.Sprintf("%s 额度不足", llmAuditProfileLabel(previous))
|
||||
if transition.NextProfileID != nil {
|
||||
summary += fmt.Sprintf("，后续调用切换到 %s", llmAuditProfileLabel(next))
|
||||
} else {
|
||||
mode = "exhausted"
|
||||
kind = "llm_failover"
|
||||
summary += "，配置链已耗尽"
|
||||
}
|
||||
metadata, _ := json.Marshal(llmActivityMetadata{LLMTransition: llmTransitionAudit{
|
||||
Mode: mode, Reason: cause.Error(), Previous: previous, Next: next,
|
||||
}})
|
||||
s.engine.emitActivity(t, db.Activity{Worker: "system", Kind: kind, IsError: transition.ChainExhausted, Summary: summary, Detail: cause.Error(), Metadata: metadata})
|
||||
log.Printf("[llm-failover] task %s: %s", t.ID, summary)
|
||||
}
|
||||
|
||||
func (s *Server) llmAuditProfile(id int64) *llmAuditProfile {
|
||||
if id <= 0 || s.m == nil || s.m.pg == nil {
|
||||
return nil
|
||||
}
|
||||
p, err := s.m.pg.ProfileByID(id)
|
||||
if err != nil || p == nil {
|
||||
return &llmAuditProfile{ID: id, Name: fmt.Sprintf("配置 #%d", id)}
|
||||
}
|
||||
return &llmAuditProfile{ID: p.ID, Name: p.Name, Format: p.Format, Model: p.Model}
|
||||
}
|
||||
|
||||
func llmAuditProfileLabel(profile *llmAuditProfile) string {
|
||||
if profile == nil {
|
||||
return "默认配置"
|
||||
}
|
||||
name := profile.Name
|
||||
if name == "" {
|
||||
name = fmt.Sprintf("配置 #%d", profile.ID)
|
||||
}
|
||||
detail := []string{}
|
||||
if profile.Format != "" {
|
||||
detail = append(detail, profile.Format)
|
||||
}
|
||||
if profile.Model != "" {
|
||||
detail = append(detail, profile.Model)
|
||||
}
|
||||
if len(detail) == 0 {
|
||||
return name
|
||||
}
|
||||
return fmt.Sprintf("%s（%s）", name, strings.Join(detail, " / "))
|
||||
}
|
||||
|
||||
func sameOptionalID(a, b *int64) bool {
|
||||
if a == nil || b == nil {
|
||||
return a == nil && b == nil
|
||||
}
|
||||
return *a == *b
|
||||
}
|
||||
|
||||
func (s *Server) emitManualTaskLLMSwitch(t *Task, previousID, nextID *int64) db.Activity {
|
||||
previous := (*llmAuditProfile)(nil)
|
||||
next := (*llmAuditProfile)(nil)
|
||||
if previousID != nil {
|
||||
previous = s.llmAuditProfile(*previousID)
|
||||
}
|
||||
if nextID != nil {
|
||||
next = s.llmAuditProfile(*nextID)
|
||||
}
|
||||
summary := fmt.Sprintf("用户手动将任务 LLM 从 %s 切换到 %s", llmAuditProfileLabel(previous), llmAuditProfileLabel(next))
|
||||
metadata, _ := json.Marshal(llmActivityMetadata{LLMTransition: llmTransitionAudit{
|
||||
Mode: "manual", Reason: "用户手动切换任务 LLM", Previous: previous, Next: next,
|
||||
}})
|
||||
return s.engine.emitActivity(t, db.Activity{Worker: "system", Kind: "llm_switch", Summary: summary, Detail: summary, Metadata: metadata})
|
||||
}
|
||||
Reference in new issue
Block a user