pphua
9793b522d9
Add read-only task context inheritance, ordered task-level LLM failover, live Worker controls, cascading task cleanup, FIFO task concurrency, cancellation diagnostics, UI updates, tests, build script, and changelog.
463 lines
14 KiB
Go
463 lines
14 KiB
Go
package server
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"iter"
|
||
"log"
|
||
"path/filepath"
|
||
"strconv"
|
||
"strings"
|
||
"time"
|
||
|
||
"github.com/Autumn-27/artex/agent"
|
||
"github.com/Autumn-27/artex/db"
|
||
"github.com/Autumn-27/artex/llmrec"
|
||
"github.com/Autumn-27/norma/llm"
|
||
"github.com/Autumn-27/norma/memory"
|
||
"github.com/Autumn-27/norma/transcript"
|
||
)
|
||
|
||
type taskAgentBundle struct {
|
||
runtime *taskLLMRuntime // goal decomposition runtime
|
||
plannerRuntime *taskLLMRuntime
|
||
workerRuntime *taskLLMRuntime
|
||
mainRuntime *taskLLMRuntime
|
||
pl *agent.Planner
|
||
wk *agent.Worker
|
||
main *agent.MainAgent
|
||
}
|
||
|
||
type llmAuditProfile struct {
|
||
ID int64 `json:"id"`
|
||
Name string `json:"name"`
|
||
Format string `json:"format"`
|
||
Model string `json:"model"`
|
||
}
|
||
|
||
type llmTransitionAudit struct {
|
||
Mode string `json:"mode"` // automatic | manual | exhausted
|
||
Reason string `json:"reason"`
|
||
Previous *llmAuditProfile `json:"previous,omitempty"`
|
||
Next *llmAuditProfile `json:"next,omitempty"`
|
||
}
|
||
|
||
type llmActivityMetadata struct {
|
||
LLMTransition llmTransitionAudit `json:"llm_transition"`
|
||
}
|
||
|
||
type taskLLMRuntime struct {
|
||
s *Server
|
||
taskID string
|
||
agentKey string
|
||
}
|
||
|
||
type taskLLMError struct {
|
||
taskID string
|
||
chainExhausted bool
|
||
cause error
|
||
}
|
||
|
||
func (e *taskLLMError) Error() string {
|
||
if e.chainExhausted {
|
||
return fmt.Sprintf("task %s LLM profile chain exhausted: %v", e.taskID, e.cause)
|
||
}
|
||
return e.cause.Error()
|
||
}
|
||
|
||
func (e *taskLLMError) Unwrap() error { return e.cause }
|
||
|
||
func isTaskLLMRuntimeError(err error) bool {
|
||
var target *taskLLMError
|
||
return errors.As(err, &target)
|
||
}
|
||
|
||
func isTaskLLMChainExhausted(err error) bool {
|
||
var target *taskLLMError
|
||
return errors.As(err, &target) && target.chainExhausted
|
||
}
|
||
|
||
// isQuotaExhaustedError is intentionally strict. A generic 429, auth error,
|
||
// network failure, or 5xx does not rotate providers; the response must explicitly
|
||
// identify quota, credits, billing balance, or payment exhaustion.
|
||
func isQuotaExhaustedError(err error) bool {
|
||
return err != nil && agent.IsQuotaExhaustedMessage(err.Error())
|
||
}
|
||
|
||
type taskLLMSelection struct {
|
||
task *Task
|
||
profileID int64
|
||
revision int64
|
||
provider llm.Provider
|
||
}
|
||
|
||
type taskLLMStreamHooks struct {
|
||
current func() (taskLLMSelection, error)
|
||
exhaust func(taskLLMSelection, error) (db.TaskLLMTransition, error)
|
||
transition func(taskLLMSelection, db.TaskLLMTransition, error)
|
||
}
|
||
|
||
func (r *taskLLMRuntime) current() (*Task, int64, int64, llm.Provider, error) {
|
||
taskNum, err := parseTaskID(r.taskID)
|
||
if err != nil {
|
||
return nil, 0, 0, nil, err
|
||
}
|
||
pt, err := r.s.m.pg.GetTask(taskNum)
|
||
if err != nil {
|
||
return nil, 0, 0, nil, err
|
||
}
|
||
if pt == nil {
|
||
return nil, 0, 0, nil, fmt.Errorf("task %s not found", r.taskID)
|
||
}
|
||
r.s.syncTaskLLMState(pt)
|
||
t, _ := r.s.m.Task(r.taskID)
|
||
if len(pt.LLMProfileIDs) == 0 {
|
||
prov, _, ok := r.s.fallbackProviderForAgent(r.agentKey)
|
||
if !ok {
|
||
return t, 0, pt.LLMChainRevision, nil, fmt.Errorf("task %s has no available fallback LLM provider", r.taskID)
|
||
}
|
||
return t, 0, pt.LLMChainRevision, prov, nil
|
||
}
|
||
if pt.ActiveLLMProfileID == nil {
|
||
return t, 0, pt.LLMChainRevision, nil, &taskLLMError{taskID: r.taskID, chainExhausted: true, cause: errors.New("all selected profiles are quota exhausted")}
|
||
}
|
||
prov, _, ok := r.s.providerForProfile(*pt.ActiveLLMProfileID)
|
||
if !ok {
|
||
return t, *pt.ActiveLLMProfileID, pt.LLMChainRevision, nil, fmt.Errorf("LLM profile #%d is missing or invalid", *pt.ActiveLLMProfileID)
|
||
}
|
||
return t, *pt.ActiveLLMProfileID, pt.LLMChainRevision, prov, nil
|
||
}
|
||
|
||
func parseTaskID(id string) (int64, error) {
|
||
n, err := strconv.ParseInt(id, 10, 64)
|
||
if err != nil || n <= 0 {
|
||
return 0, fmt.Errorf("invalid task id %q", id)
|
||
}
|
||
return n, nil
|
||
}
|
||
|
||
func (r *taskLLMRuntime) Stream(ctx context.Context, req llm.CompletionRequest) iter.Seq2[llm.StreamEvent, error] {
|
||
ctx = llmrec.WithTaskID(ctx, r.taskID)
|
||
hooks := taskLLMStreamHooks{
|
||
current: func() (taskLLMSelection, error) {
|
||
task, profileID, revision, provider, err := r.current()
|
||
return taskLLMSelection{task: task, profileID: profileID, revision: revision, provider: provider}, err
|
||
},
|
||
exhaust: func(selection taskLLMSelection, cause error) (db.TaskLLMTransition, error) {
|
||
taskNum, _ := parseTaskID(r.taskID)
|
||
transition, err := r.s.m.pg.MarkTaskLLMProfileQuotaExhaustedAtRevision(taskNum, selection.profileID, selection.revision, cause.Error())
|
||
if err != nil {
|
||
return transition, err
|
||
}
|
||
if pt, getErr := r.s.m.pg.GetTask(taskNum); getErr == nil && pt != nil {
|
||
r.s.syncTaskLLMState(pt)
|
||
}
|
||
return transition, nil
|
||
},
|
||
transition: func(selection taskLLMSelection, transition db.TaskLLMTransition, cause error) {
|
||
r.s.emitTaskLLMTransition(selection.task, transition, cause)
|
||
},
|
||
}
|
||
return streamTaskLLM(ctx, r.taskID, req, hooks)
|
||
}
|
||
|
||
func streamTaskLLM(ctx context.Context, taskID string, req llm.CompletionRequest, hooks taskLLMStreamHooks) iter.Seq2[llm.StreamEvent, error] {
|
||
return func(yield func(llm.StreamEvent, error) bool) {
|
||
for {
|
||
selection, err := hooks.current()
|
||
if err != nil {
|
||
yield(llm.StreamEvent{}, err)
|
||
return
|
||
}
|
||
committed := false
|
||
var pending []llm.StreamEvent
|
||
var streamErr error
|
||
for event, err := range selection.provider.Stream(ctx, req) {
|
||
if err != nil {
|
||
streamErr = err
|
||
break
|
||
}
|
||
if !committed && !streamEventCommitsOutput(event) {
|
||
pending = append(pending, event)
|
||
continue
|
||
}
|
||
if !committed {
|
||
for _, buffered := range pending {
|
||
if !yield(buffered, nil) {
|
||
return
|
||
}
|
||
}
|
||
pending = nil
|
||
committed = true
|
||
}
|
||
if !yield(event, nil) {
|
||
return
|
||
}
|
||
}
|
||
if streamErr == nil {
|
||
for _, buffered := range pending {
|
||
if !yield(buffered, nil) {
|
||
return
|
||
}
|
||
}
|
||
return
|
||
}
|
||
// profileID=0 means the explicit chain was cleared while this stable task
|
||
// bundle was still in use. Agent/global fallback errors follow the legacy
|
||
// behavior and never mutate task failover state.
|
||
if selection.profileID == 0 || !isQuotaExhaustedError(streamErr) {
|
||
for _, buffered := range pending {
|
||
if !yield(buffered, nil) {
|
||
return
|
||
}
|
||
}
|
||
yield(llm.StreamEvent{}, streamErr)
|
||
return
|
||
}
|
||
transition, markErr := hooks.exhaust(selection, streamErr)
|
||
if markErr != nil {
|
||
cause := fmt.Errorf("mark profile quota exhausted after %v: %w", streamErr, markErr)
|
||
if committed {
|
||
// Output may already have driven tool execution. Report the persistence
|
||
// failure, but classify it as router-handled so the worker does not
|
||
// replay the entire intent and duplicate those side effects.
|
||
yield(llm.StreamEvent{}, &taskLLMError{taskID: taskID, cause: cause})
|
||
} else {
|
||
yield(llm.StreamEvent{}, cause)
|
||
}
|
||
return
|
||
}
|
||
if transition.Advanced && !transition.Stale && hooks.transition != nil {
|
||
hooks.transition(selection, transition, streamErr)
|
||
}
|
||
if committed || (!transition.Stale && transition.NextProfileID == nil) {
|
||
yield(llm.StreamEvent{}, &taskLLMError{taskID: taskID, chainExhausted: transition.ChainExhausted, cause: streamErr})
|
||
return
|
||
}
|
||
// No event reached the caller, so replaying the same logical request on
|
||
// the next profile cannot duplicate model output or tool execution.
|
||
}
|
||
}
|
||
}
|
||
|
||
func streamEventCommitsOutput(event llm.StreamEvent) bool {
|
||
switch event.Type {
|
||
case llm.SETextDelta, llm.SEThinkingDelta, llm.SEToolInputJSON:
|
||
return event.Text != ""
|
||
case llm.SEToolUseStart, llm.SEMessageDelta, llm.SEMessageStop:
|
||
return true
|
||
default:
|
||
return false
|
||
}
|
||
}
|
||
|
||
func (r *taskLLMRuntime) CompactionWindow() int {
|
||
taskNum, err := parseTaskID(r.taskID)
|
||
if err != nil {
|
||
return (agent.Config{}).CompactionWindow()
|
||
}
|
||
chain, err := r.s.m.pg.TaskLLMProfiles(taskNum)
|
||
if err != nil {
|
||
return (agent.Config{}).CompactionWindow()
|
||
}
|
||
if len(chain) == 0 {
|
||
if _, cfg, ok := r.s.fallbackProviderForAgent(r.agentKey); ok {
|
||
return cfg.CompactionWindow()
|
||
}
|
||
return (agent.Config{}).CompactionWindow()
|
||
}
|
||
minimum := 0
|
||
for _, entry := range chain {
|
||
if cfg, ok := r.s.loadProfileConfig(entry.ProfileID); ok {
|
||
window := cfg.CompactionWindow()
|
||
if minimum == 0 || window < minimum {
|
||
minimum = window
|
||
}
|
||
}
|
||
}
|
||
if minimum == 0 {
|
||
return (agent.Config{}).CompactionWindow()
|
||
}
|
||
return minimum
|
||
}
|
||
|
||
// fallbackProviderForAgent resolves the pre-existing precedence used by tasks
|
||
// without an explicit chain: the Agent's saved binding first, then the current
|
||
// global provider (which may come from the default profile or environment).
|
||
func (s *Server) fallbackProviderForAgent(agentKey string) (llm.Provider, agent.Config, bool) {
|
||
if id := s.effectiveProfileForAgent(agentKey, nil); id != nil {
|
||
if prov, cfg, ok := s.providerForProfile(*id); ok {
|
||
return s.poolForBinding(*id, prov, cfg), cfg, true
|
||
}
|
||
}
|
||
s.cfgMu.Lock()
|
||
defer s.cfgMu.Unlock()
|
||
if !s.llmOn || s.llmProv == nil {
|
||
return nil, agent.Config{}, false
|
||
}
|
||
return s.llmProv, s.llmCfg, true
|
||
}
|
||
|
||
func (s *Server) taskRuntimeAvailable(t *Task, agentKeys ...string) bool {
|
||
if t == nil {
|
||
return false
|
||
}
|
||
state := t.llmStateSnapshot()
|
||
if len(state.ProfileIDs) > 0 {
|
||
if state.ActiveID == nil {
|
||
return false
|
||
}
|
||
_, _, ok := s.providerForProfile(*state.ActiveID)
|
||
return ok
|
||
}
|
||
s.cfgMu.Lock()
|
||
globallyReady := s.llmOn && s.llmProv != nil
|
||
s.cfgMu.Unlock()
|
||
if globallyReady {
|
||
return true
|
||
}
|
||
for _, key := range agentKeys {
|
||
if _, _, ok := s.fallbackProviderForAgent(key); !ok {
|
||
return false
|
||
}
|
||
}
|
||
return len(agentKeys) > 0
|
||
}
|
||
|
||
func (s *Server) invalidateTaskAgents() {
|
||
s.taskAgentMu.Lock()
|
||
s.taskAgents = map[string]*taskAgentBundle{}
|
||
s.taskAgentMu.Unlock()
|
||
}
|
||
|
||
func (s *Server) agentsForTask(t *Task) *taskAgentBundle {
|
||
s.taskAgentMu.Lock()
|
||
defer s.taskAgentMu.Unlock()
|
||
if bundle := s.taskAgents[t.ID]; bundle != nil {
|
||
return bundle
|
||
}
|
||
goalRuntime := &taskLLMRuntime{s: s, taskID: t.ID, agentKey: "goals"}
|
||
plannerRuntime := &taskLLMRuntime{s: s, taskID: t.ID, agentKey: "planner"}
|
||
workerRuntime := &taskLLMRuntime{s: s, taskID: t.ID, agentKey: "worker"}
|
||
mainRuntime := &taskLLMRuntime{s: s, taskID: t.ID, agentKey: "mainagent"}
|
||
tx := transcript.NewStore(filepath.Join(s.m.dir, "transcripts"))
|
||
window := workerRuntime.CompactionWindow()
|
||
wk := agent.NewWorker(workerRuntime, "task-router", s.m.dir, tx, window, s.agentMaxTurns("worker"))
|
||
wk.SetCompactionWindowResolver(workerRuntime.CompactionWindow)
|
||
wk.SetRunTimeout(time.Duration(s.agentRunSeconds("worker")) * time.Second)
|
||
wk.SetProxy(s.m.ProxyAddr(), s.m.ProxyCACert())
|
||
wk.SetMemory(memory.NewStore(filepath.Join(s.m.dir, "memory")))
|
||
wk.SetWebSearch(s.webSearchFor("worker"))
|
||
pl := agent.NewPlanner(plannerRuntime, "task-router", s.m.dir, tx, plannerRuntime.CompactionWindow(), s.agentMaxTurns("planner"))
|
||
pl.SetCompactionWindowResolver(plannerRuntime.CompactionWindow)
|
||
pl.SetKillWork(s.engine.KillWork)
|
||
pl.SetSteerWork(s.engine.SteerWork)
|
||
pl.SetProxy(s.m.ProxyAddr(), s.m.ProxyCACert())
|
||
pl.SetWebSearch(s.webSearchFor("planner"))
|
||
main := agent.NewMainAgent(mainRuntime, "task-router", s.m.dir, tx, mainRuntime.CompactionWindow(), s.agentMaxTurns("mainagent"))
|
||
main.SetCompactionWindowResolver(mainRuntime.CompactionWindow)
|
||
main.SetProxy(s.m.ProxyAddr(), s.m.ProxyCACert())
|
||
main.SetWebSearch(s.webSearchFor("mainagent"))
|
||
bundle := &taskAgentBundle{
|
||
runtime: goalRuntime, plannerRuntime: plannerRuntime, workerRuntime: workerRuntime,
|
||
mainRuntime: mainRuntime, pl: pl, wk: wk, main: main,
|
||
}
|
||
s.taskAgents[t.ID] = bundle
|
||
return bundle
|
||
}
|
||
|
||
func (s *Server) syncTaskLLMState(pt *db.Task) {
|
||
if pt == nil {
|
||
return
|
||
}
|
||
id := fmt.Sprintf("%d", pt.ID)
|
||
s.m.mu.Lock()
|
||
if task := s.m.tasks[id]; task != nil {
|
||
task.setLLMState(pt.LLMProfileID, pt.ActiveLLMProfileID, pt.LLMProfileIDs, pt.LLMFailoverState, pt.LLMFailoverReason)
|
||
}
|
||
s.m.mu.Unlock()
|
||
}
|
||
|
||
func (s *Server) emitTaskLLMTransition(t *Task, transition db.TaskLLMTransition, cause error) {
|
||
if t == nil {
|
||
return
|
||
}
|
||
previous := s.llmAuditProfile(transition.PreviousProfileID)
|
||
var next *llmAuditProfile
|
||
if transition.NextProfileID != nil {
|
||
next = s.llmAuditProfile(*transition.NextProfileID)
|
||
}
|
||
mode := "automatic"
|
||
kind := "llm_switch"
|
||
summary := fmt.Sprintf("%s 额度不足", llmAuditProfileLabel(previous))
|
||
if transition.NextProfileID != nil {
|
||
summary += fmt.Sprintf("，后续调用切换到 %s", llmAuditProfileLabel(next))
|
||
} else {
|
||
mode = "exhausted"
|
||
kind = "llm_failover"
|
||
summary += "，配置链已耗尽"
|
||
}
|
||
metadata, _ := json.Marshal(llmActivityMetadata{LLMTransition: llmTransitionAudit{
|
||
Mode: mode, Reason: cause.Error(), Previous: previous, Next: next,
|
||
}})
|
||
s.engine.emitActivity(t, db.Activity{Worker: "system", Kind: kind, IsError: transition.ChainExhausted, Summary: summary, Detail: cause.Error(), Metadata: metadata})
|
||
log.Printf("[llm-failover] task %s: %s", t.ID, summary)
|
||
}
|
||
|
||
func (s *Server) llmAuditProfile(id int64) *llmAuditProfile {
|
||
if id <= 0 || s.m == nil || s.m.pg == nil {
|
||
return nil
|
||
}
|
||
p, err := s.m.pg.ProfileByID(id)
|
||
if err != nil || p == nil {
|
||
return &llmAuditProfile{ID: id, Name: fmt.Sprintf("配置 #%d", id)}
|
||
}
|
||
return &llmAuditProfile{ID: p.ID, Name: p.Name, Format: p.Format, Model: p.Model}
|
||
}
|
||
|
||
func llmAuditProfileLabel(profile *llmAuditProfile) string {
|
||
if profile == nil {
|
||
return "默认配置"
|
||
}
|
||
name := profile.Name
|
||
if name == "" {
|
||
name = fmt.Sprintf("配置 #%d", profile.ID)
|
||
}
|
||
detail := []string{}
|
||
if profile.Format != "" {
|
||
detail = append(detail, profile.Format)
|
||
}
|
||
if profile.Model != "" {
|
||
detail = append(detail, profile.Model)
|
||
}
|
||
if len(detail) == 0 {
|
||
return name
|
||
}
|
||
return fmt.Sprintf("%s（%s）", name, strings.Join(detail, " / "))
|
||
}
|
||
|
||
func sameOptionalID(a, b *int64) bool {
|
||
if a == nil || b == nil {
|
||
return a == nil && b == nil
|
||
}
|
||
return *a == *b
|
||
}
|
||
|
||
func (s *Server) emitManualTaskLLMSwitch(t *Task, previousID, nextID *int64) db.Activity {
|
||
previous := (*llmAuditProfile)(nil)
|
||
next := (*llmAuditProfile)(nil)
|
||
if previousID != nil {
|
||
previous = s.llmAuditProfile(*previousID)
|
||
}
|
||
if nextID != nil {
|
||
next = s.llmAuditProfile(*nextID)
|
||
}
|
||
summary := fmt.Sprintf("用户手动将任务 LLM 从 %s 切换到 %s", llmAuditProfileLabel(previous), llmAuditProfileLabel(next))
|
||
metadata, _ := json.Marshal(llmActivityMetadata{LLMTransition: llmTransitionAudit{
|
||
Mode: "manual", Reason: "用户手动切换任务 LLM", Previous: previous, Next: next,
|
||
}})
|
||
return s.engine.emitActivity(t, db.Activity{Worker: "system", Kind: "llm_switch", Summary: summary, Detail: summary, Metadata: metadata})
|
||
}
|