463 lines
14 KiB
Go
463 lines
14 KiB
Go
package server
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"iter"
|
||
"log"
|
||
"path/filepath"
|
||
"strconv"
|
||
"strings"
|
||
"time"
|
||
|
||
"github.com/Autumn-27/artex/agent"
|
||
"github.com/Autumn-27/artex/db"
|
||
"github.com/Autumn-27/artex/llmrec"
|
||
"github.com/Autumn-27/norma/llm"
|
||
"github.com/Autumn-27/norma/memory"
|
||
"github.com/Autumn-27/norma/transcript"
|
||
)
|
||
|
||
type taskAgentBundle struct {
|
||
runtime *taskLLMRuntime // goal decomposition runtime
|
||
plannerRuntime *taskLLMRuntime
|
||
workerRuntime *taskLLMRuntime
|
||
mainRuntime *taskLLMRuntime
|
||
pl *agent.Planner
|
||
wk *agent.Worker
|
||
main *agent.MainAgent
|
||
}
|
||
|
||
type llmAuditProfile struct {
|
||
ID int64 `json:"id"`
|
||
Name string `json:"name"`
|
||
Format string `json:"format"`
|
||
Model string `json:"model"`
|
||
}
|
||
|
||
type llmTransitionAudit struct {
|
||
Mode string `json:"mode"` // automatic | manual | exhausted
|
||
Reason string `json:"reason"`
|
||
Previous *llmAuditProfile `json:"previous,omitempty"`
|
||
Next *llmAuditProfile `json:"next,omitempty"`
|
||
}
|
||
|
||
type llmActivityMetadata struct {
|
||
LLMTransition llmTransitionAudit `json:"llm_transition"`
|
||
}
|
||
|
||
type taskLLMRuntime struct {
|
||
s *Server
|
||
taskID string
|
||
agentKey string
|
||
}
|
||
|
||
type taskLLMError struct {
|
||
taskID string
|
||
chainExhausted bool
|
||
cause error
|
||
}
|
||
|
||
func (e *taskLLMError) Error() string {
|
||
if e.chainExhausted {
|
||
return fmt.Sprintf("task %s LLM profile chain exhausted: %v", e.taskID, e.cause)
|
||
}
|
||
return e.cause.Error()
|
||
}
|
||
|
||
func (e *taskLLMError) Unwrap() error { return e.cause }
|
||
|
||
func isTaskLLMRuntimeError(err error) bool {
|
||
var target *taskLLMError
|
||
return errors.As(err, &target)
|
||
}
|
||
|
||
func isTaskLLMChainExhausted(err error) bool {
|
||
var target *taskLLMError
|
||
return errors.As(err, &target) && target.chainExhausted
|
||
}
|
||
|
||
// isQuotaExhaustedError is intentionally strict. A generic 429, auth error,
|
||
// network failure, or 5xx does not rotate providers; the response must explicitly
|
||
// identify quota, credits, billing balance, or payment exhaustion.
|
||
func isQuotaExhaustedError(err error) bool {
|
||
return err != nil && agent.IsQuotaExhaustedMessage(err.Error())
|
||
}
|
||
|
||
type taskLLMSelection struct {
|
||
task *Task
|
||
profileID int64
|
||
revision int64
|
||
provider llm.Provider
|
||
}
|
||
|
||
type taskLLMStreamHooks struct {
|
||
current func() (taskLLMSelection, error)
|
||
exhaust func(taskLLMSelection, error) (db.TaskLLMTransition, error)
|
||
transition func(taskLLMSelection, db.TaskLLMTransition, error)
|
||
}
|
||
|
||
func (r *taskLLMRuntime) current() (*Task, int64, int64, llm.Provider, error) {
|
||
taskNum, err := parseTaskID(r.taskID)
|
||
if err != nil {
|
||
return nil, 0, 0, nil, err
|
||
}
|
||
pt, err := r.s.m.pg.GetTask(taskNum)
|
||
if err != nil {
|
||
return nil, 0, 0, nil, err
|
||
}
|
||
if pt == nil {
|
||
return nil, 0, 0, nil, fmt.Errorf("task %s not found", r.taskID)
|
||
}
|
||
r.s.syncTaskLLMState(pt)
|
||
t, _ := r.s.m.Task(r.taskID)
|
||
if len(pt.LLMProfileIDs) == 0 {
|
||
prov, _, ok := r.s.fallbackProviderForAgent(r.agentKey)
|
||
if !ok {
|
||
return t, 0, pt.LLMChainRevision, nil, fmt.Errorf("task %s has no available fallback LLM provider", r.taskID)
|
||
}
|
||
return t, 0, pt.LLMChainRevision, prov, nil
|
||
}
|
||
if pt.ActiveLLMProfileID == nil {
|
||
return t, 0, pt.LLMChainRevision, nil, &taskLLMError{taskID: r.taskID, chainExhausted: true, cause: errors.New("all selected profiles are quota exhausted")}
|
||
}
|
||
prov, _, ok := r.s.providerForProfile(*pt.ActiveLLMProfileID)
|
||
if !ok {
|
||
return t, *pt.ActiveLLMProfileID, pt.LLMChainRevision, nil, fmt.Errorf("LLM profile #%d is missing or invalid", *pt.ActiveLLMProfileID)
|
||
}
|
||
return t, *pt.ActiveLLMProfileID, pt.LLMChainRevision, prov, nil
|
||
}
|
||
|
||
func parseTaskID(id string) (int64, error) {
|
||
n, err := strconv.ParseInt(id, 10, 64)
|
||
if err != nil || n <= 0 {
|
||
return 0, fmt.Errorf("invalid task id %q", id)
|
||
}
|
||
return n, nil
|
||
}
|
||
|
||
func (r *taskLLMRuntime) Stream(ctx context.Context, req llm.CompletionRequest) iter.Seq2[llm.StreamEvent, error] {
|
||
ctx = llmrec.WithTaskID(ctx, r.taskID)
|
||
hooks := taskLLMStreamHooks{
|
||
current: func() (taskLLMSelection, error) {
|
||
task, profileID, revision, provider, err := r.current()
|
||
return taskLLMSelection{task: task, profileID: profileID, revision: revision, provider: provider}, err
|
||
},
|
||
exhaust: func(selection taskLLMSelection, cause error) (db.TaskLLMTransition, error) {
|
||
taskNum, _ := parseTaskID(r.taskID)
|
||
transition, err := r.s.m.pg.MarkTaskLLMProfileQuotaExhaustedAtRevision(taskNum, selection.profileID, selection.revision, cause.Error())
|
||
if err != nil {
|
||
return transition, err
|
||
}
|
||
if pt, getErr := r.s.m.pg.GetTask(taskNum); getErr == nil && pt != nil {
|
||
r.s.syncTaskLLMState(pt)
|
||
}
|
||
return transition, nil
|
||
},
|
||
transition: func(selection taskLLMSelection, transition db.TaskLLMTransition, cause error) {
|
||
r.s.emitTaskLLMTransition(selection.task, transition, cause)
|
||
},
|
||
}
|
||
return streamTaskLLM(ctx, r.taskID, req, hooks)
|
||
}
|
||
|
||
func streamTaskLLM(ctx context.Context, taskID string, req llm.CompletionRequest, hooks taskLLMStreamHooks) iter.Seq2[llm.StreamEvent, error] {
|
||
return func(yield func(llm.StreamEvent, error) bool) {
|
||
for {
|
||
selection, err := hooks.current()
|
||
if err != nil {
|
||
yield(llm.StreamEvent{}, err)
|
||
return
|
||
}
|
||
committed := false
|
||
var pending []llm.StreamEvent
|
||
var streamErr error
|
||
for event, err := range selection.provider.Stream(ctx, req) {
|
||
if err != nil {
|
||
streamErr = err
|
||
break
|
||
}
|
||
if !committed && !streamEventCommitsOutput(event) {
|
||
pending = append(pending, event)
|
||
continue
|
||
}
|
||
if !committed {
|
||
for _, buffered := range pending {
|
||
if !yield(buffered, nil) {
|
||
return
|
||
}
|
||
}
|
||
pending = nil
|
||
committed = true
|
||
}
|
||
if !yield(event, nil) {
|
||
return
|
||
}
|
||
}
|
||
if streamErr == nil {
|
||
for _, buffered := range pending {
|
||
if !yield(buffered, nil) {
|
||
return
|
||
}
|
||
}
|
||
return
|
||
}
|
||
// profileID=0 means the explicit chain was cleared while this stable task
|
||
// bundle was still in use. Agent/global fallback errors follow the legacy
|
||
// behavior and never mutate task failover state.
|
||
if selection.profileID == 0 || !isQuotaExhaustedError(streamErr) {
|
||
for _, buffered := range pending {
|
||
if !yield(buffered, nil) {
|
||
return
|
||
}
|
||
}
|
||
yield(llm.StreamEvent{}, streamErr)
|
||
return
|
||
}
|
||
transition, markErr := hooks.exhaust(selection, streamErr)
|
||
if markErr != nil {
|
||
cause := fmt.Errorf("mark profile quota exhausted after %v: %w", streamErr, markErr)
|
||
if committed {
|
||
// Output may already have driven tool execution. Report the persistence
|
||
// failure, but classify it as router-handled so the worker does not
|
||
// replay the entire intent and duplicate those side effects.
|
||
yield(llm.StreamEvent{}, &taskLLMError{taskID: taskID, cause: cause})
|
||
} else {
|
||
yield(llm.StreamEvent{}, cause)
|
||
}
|
||
return
|
||
}
|
||
if transition.Advanced && !transition.Stale && hooks.transition != nil {
|
||
hooks.transition(selection, transition, streamErr)
|
||
}
|
||
if committed || (!transition.Stale && transition.NextProfileID == nil) {
|
||
yield(llm.StreamEvent{}, &taskLLMError{taskID: taskID, chainExhausted: transition.ChainExhausted, cause: streamErr})
|
||
return
|
||
}
|
||
// No event reached the caller, so replaying the same logical request on
|
||
// the next profile cannot duplicate model output or tool execution.
|
||
}
|
||
}
|
||
}
|
||
|
||
func streamEventCommitsOutput(event llm.StreamEvent) bool {
|
||
switch event.Type {
|
||
case llm.SETextDelta, llm.SEThinkingDelta, llm.SEToolInputJSON:
|
||
return event.Text != ""
|
||
case llm.SEToolUseStart, llm.SEMessageDelta, llm.SEMessageStop:
|
||
return true
|
||
default:
|
||
return false
|
||
}
|
||
}
|
||
|
||
func (r *taskLLMRuntime) CompactionWindow() int {
|
||
taskNum, err := parseTaskID(r.taskID)
|
||
if err != nil {
|
||
return (agent.Config{}).CompactionWindow()
|
||
}
|
||
chain, err := r.s.m.pg.TaskLLMProfiles(taskNum)
|
||
if err != nil {
|
||
return (agent.Config{}).CompactionWindow()
|
||
}
|
||
if len(chain) == 0 {
|
||
if _, cfg, ok := r.s.fallbackProviderForAgent(r.agentKey); ok {
|
||
return cfg.CompactionWindow()
|
||
}
|
||
return (agent.Config{}).CompactionWindow()
|
||
}
|
||
minimum := 0
|
||
for _, entry := range chain {
|
||
if cfg, ok := r.s.loadProfileConfig(entry.ProfileID); ok {
|
||
window := cfg.CompactionWindow()
|
||
if minimum == 0 || window < minimum {
|
||
minimum = window
|
||
}
|
||
}
|
||
}
|
||
if minimum == 0 {
|
||
return (agent.Config{}).CompactionWindow()
|
||
}
|
||
return minimum
|
||
}
|
||
|
||
// fallbackProviderForAgent resolves the pre-existing precedence used by tasks
|
||
// without an explicit chain: the Agent's saved binding first, then the current
|
||
// global provider (which may come from the default profile or environment).
|
||
func (s *Server) fallbackProviderForAgent(agentKey string) (llm.Provider, agent.Config, bool) {
|
||
if id := s.effectiveProfileForAgent(agentKey, nil); id != nil {
|
||
if prov, cfg, ok := s.providerForProfile(*id); ok {
|
||
return s.poolForBinding(*id, prov, cfg), cfg, true
|
||
}
|
||
}
|
||
s.cfgMu.Lock()
|
||
defer s.cfgMu.Unlock()
|
||
if !s.llmOn || s.llmProv == nil {
|
||
return nil, agent.Config{}, false
|
||
}
|
||
return s.llmProv, s.llmCfg, true
|
||
}
|
||
|
||
func (s *Server) taskRuntimeAvailable(t *Task, agentKeys ...string) bool {
|
||
if t == nil {
|
||
return false
|
||
}
|
||
state := t.llmStateSnapshot()
|
||
if len(state.ProfileIDs) > 0 {
|
||
if state.ActiveID == nil {
|
||
return false
|
||
}
|
||
_, _, ok := s.providerForProfile(*state.ActiveID)
|
||
return ok
|
||
}
|
||
s.cfgMu.Lock()
|
||
globallyReady := s.llmOn && s.llmProv != nil
|
||
s.cfgMu.Unlock()
|
||
if globallyReady {
|
||
return true
|
||
}
|
||
for _, key := range agentKeys {
|
||
if _, _, ok := s.fallbackProviderForAgent(key); !ok {
|
||
return false
|
||
}
|
||
}
|
||
return len(agentKeys) > 0
|
||
}
|
||
|
||
func (s *Server) invalidateTaskAgents() {
|
||
s.taskAgentMu.Lock()
|
||
s.taskAgents = map[string]*taskAgentBundle{}
|
||
s.taskAgentMu.Unlock()
|
||
}
|
||
|
||
func (s *Server) agentsForTask(t *Task) *taskAgentBundle {
|
||
s.taskAgentMu.Lock()
|
||
defer s.taskAgentMu.Unlock()
|
||
if bundle := s.taskAgents[t.ID]; bundle != nil {
|
||
return bundle
|
||
}
|
||
goalRuntime := &taskLLMRuntime{s: s, taskID: t.ID, agentKey: "goals"}
|
||
plannerRuntime := &taskLLMRuntime{s: s, taskID: t.ID, agentKey: "planner"}
|
||
workerRuntime := &taskLLMRuntime{s: s, taskID: t.ID, agentKey: "worker"}
|
||
mainRuntime := &taskLLMRuntime{s: s, taskID: t.ID, agentKey: "mainagent"}
|
||
tx := transcript.NewStore(filepath.Join(s.m.dir, "transcripts"))
|
||
window := workerRuntime.CompactionWindow()
|
||
wk := agent.NewWorker(workerRuntime, "task-router", s.m.dir, tx, window, s.agentMaxTurns("worker"))
|
||
wk.SetCompactionWindowResolver(workerRuntime.CompactionWindow)
|
||
wk.SetRunTimeout(time.Duration(s.agentRunSeconds("worker")) * time.Second)
|
||
wk.SetProxy(s.m.ProxyAddr(), s.m.ProxyCACert())
|
||
wk.SetMemory(memory.NewStore(filepath.Join(s.m.dir, "memory")))
|
||
wk.SetWebSearch(s.webSearchFor("worker"))
|
||
pl := agent.NewPlanner(plannerRuntime, "task-router", s.m.dir, tx, plannerRuntime.CompactionWindow(), s.agentMaxTurns("planner"))
|
||
pl.SetCompactionWindowResolver(plannerRuntime.CompactionWindow)
|
||
pl.SetKillWork(s.engine.KillWork)
|
||
pl.SetSteerWork(s.engine.SteerWork)
|
||
pl.SetProxy(s.m.ProxyAddr(), s.m.ProxyCACert())
|
||
pl.SetWebSearch(s.webSearchFor("planner"))
|
||
main := agent.NewMainAgent(mainRuntime, "task-router", s.m.dir, tx, mainRuntime.CompactionWindow(), s.agentMaxTurns("mainagent"))
|
||
main.SetCompactionWindowResolver(mainRuntime.CompactionWindow)
|
||
main.SetProxy(s.m.ProxyAddr(), s.m.ProxyCACert())
|
||
main.SetWebSearch(s.webSearchFor("mainagent"))
|
||
bundle := &taskAgentBundle{
|
||
runtime: goalRuntime, plannerRuntime: plannerRuntime, workerRuntime: workerRuntime,
|
||
mainRuntime: mainRuntime, pl: pl, wk: wk, main: main,
|
||
}
|
||
s.taskAgents[t.ID] = bundle
|
||
return bundle
|
||
}
|
||
|
||
func (s *Server) syncTaskLLMState(pt *db.Task) {
|
||
if pt == nil {
|
||
return
|
||
}
|
||
id := fmt.Sprintf("%d", pt.ID)
|
||
s.m.mu.Lock()
|
||
if task := s.m.tasks[id]; task != nil {
|
||
task.setLLMState(pt.LLMProfileID, pt.ActiveLLMProfileID, pt.LLMProfileIDs, pt.LLMChainRevision, pt.LLMFailoverState, pt.LLMFailoverReason)
|
||
}
|
||
s.m.mu.Unlock()
|
||
}
|
||
|
||
func (s *Server) emitTaskLLMTransition(t *Task, transition db.TaskLLMTransition, cause error) {
|
||
if t == nil {
|
||
return
|
||
}
|
||
previous := s.llmAuditProfile(transition.PreviousProfileID)
|
||
var next *llmAuditProfile
|
||
if transition.NextProfileID != nil {
|
||
next = s.llmAuditProfile(*transition.NextProfileID)
|
||
}
|
||
mode := "automatic"
|
||
kind := "llm_switch"
|
||
summary := fmt.Sprintf("%s 额度不足", llmAuditProfileLabel(previous))
|
||
if transition.NextProfileID != nil {
|
||
summary += fmt.Sprintf("，后续调用切换到 %s", llmAuditProfileLabel(next))
|
||
} else {
|
||
mode = "exhausted"
|
||
kind = "llm_failover"
|
||
summary += "，配置链已耗尽"
|
||
}
|
||
metadata, _ := json.Marshal(llmActivityMetadata{LLMTransition: llmTransitionAudit{
|
||
Mode: mode, Reason: cause.Error(), Previous: previous, Next: next,
|
||
}})
|
||
s.engine.emitActivity(t, db.Activity{Worker: "system", Kind: kind, IsError: transition.ChainExhausted, Summary: summary, Detail: cause.Error(), Metadata: metadata})
|
||
log.Printf("[llm-failover] task %s: %s", t.ID, summary)
|
||
}
|
||
|
||
func (s *Server) llmAuditProfile(id int64) *llmAuditProfile {
|
||
if id <= 0 || s.m == nil || s.m.pg == nil {
|
||
return nil
|
||
}
|
||
p, err := s.m.pg.ProfileByID(id)
|
||
if err != nil || p == nil {
|
||
return &llmAuditProfile{ID: id, Name: fmt.Sprintf("配置 #%d", id)}
|
||
}
|
||
return &llmAuditProfile{ID: p.ID, Name: p.Name, Format: p.Format, Model: p.Model}
|
||
}
|
||
|
||
func llmAuditProfileLabel(profile *llmAuditProfile) string {
|
||
if profile == nil {
|
||
return "默认配置"
|
||
}
|
||
name := profile.Name
|
||
if name == "" {
|
||
name = fmt.Sprintf("配置 #%d", profile.ID)
|
||
}
|
||
detail := []string{}
|
||
if profile.Format != "" {
|
||
detail = append(detail, profile.Format)
|
||
}
|
||
if profile.Model != "" {
|
||
detail = append(detail, profile.Model)
|
||
}
|
||
if len(detail) == 0 {
|
||
return name
|
||
}
|
||
return fmt.Sprintf("%s（%s）", name, strings.Join(detail, " / "))
|
||
}
|
||
|
||
func sameOptionalID(a, b *int64) bool {
|
||
if a == nil || b == nil {
|
||
return a == nil && b == nil
|
||
}
|
||
return *a == *b
|
||
}
|
||
|
||
func (s *Server) emitManualTaskLLMSwitch(t *Task, previousID, nextID *int64) db.Activity {
|
||
previous := (*llmAuditProfile)(nil)
|
||
next := (*llmAuditProfile)(nil)
|
||
if previousID != nil {
|
||
previous = s.llmAuditProfile(*previousID)
|
||
}
|
||
if nextID != nil {
|
||
next = s.llmAuditProfile(*nextID)
|
||
}
|
||
summary := fmt.Sprintf("用户手动将任务 LLM 从 %s 切换到 %s", llmAuditProfileLabel(previous), llmAuditProfileLabel(next))
|
||
metadata, _ := json.Marshal(llmActivityMetadata{LLMTransition: llmTransitionAudit{
|
||
Mode: "manual", Reason: "用户手动切换任务 LLM", Previous: previous, Next: next,
|
||
}})
|
||
return s.engine.emitActivity(t, db.Activity{Worker: "system", Kind: "llm_switch", Summary: summary, Detail: summary, Metadata: metadata})
|
||
}
|