Files
ARTEX-chixuweihu/server/engine_workers_test.go
T
ji ruo 7d9c5699cb feat(workers): hot-reload task worker concurrency
Support live task overrides and global default/pool reconciliation.
Drain retiring workers without cancelling active intents; expose actual/target loop counts in the API and task UI.
Persist worker policy and cover resizing, API validation, settings notification, and restoration with regression tests.

Fixes #171
2026-11-19 17:26:51 +08:00

255 lines
8.7 KiB
Go

package server
import (
"context"
"fmt"
"iter"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
"time"
"github.com/Autumn-27/artex/agent"
"github.com/Autumn-27/artex/db"
"github.com/Autumn-27/norma/llm"
)
type resizingCall struct {
ctx context.Context
release chan struct{}
}
type resizingProvider struct{ calls chan resizingCall }
func (p *resizingProvider) Complete(ctx context.Context, _ llm.CompletionRequest) (llm.Message, string, llm.Usage, error) {
call := resizingCall{ctx: ctx, release: make(chan struct{})}
select {
case p.calls <- call:
case <-ctx.Done():
return llm.Message{}, "", llm.Usage{}, ctx.Err()
}
select {
case <-call.release:
return llm.Message{Role: llm.RoleAssistant, Content: []llm.ContentBlock{llm.TextBlock("fixture completed")}}, "end_turn", llm.Usage{}, nil
case <-ctx.Done():
return llm.Message{}, "", llm.Usage{}, ctx.Err()
}
}
func (*resizingProvider) Stream(context.Context, llm.CompletionRequest) iter.Seq2[llm.StreamEvent, error] {
panic("resizing fixture uses non-streaming calls")
}
func waitWorkerCondition(t *testing.T, condition func() bool) {
t.Helper()
deadline := time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) {
if condition() {
return
}
time.Sleep(10 * time.Millisecond)
}
t.Fatal("timed out waiting for worker state")
}
// Exercises the real worker claim/Execute/settlement path. Both slots remain
// blocked in provider calls while shrinking; only the retained slot may claim
// another intent, and re-expansion must reuse a name without overlapping loops.
func TestWorkerHotResizeDrainsBeforeReusingSlot(t *testing.T) {
m, err := NewManager(t.TempDir(), "")
if err != nil {
t.Skipf("postgres unavailable: %v", err)
}
defer m.Close()
task, err := m.CreateTask("resize workers", "fixture", nil, 0, 0)
if err != nil {
t.Fatal(err)
}
defer m.DeleteTask(task.ID, DeleteTaskOptions{})
count := 2
if _, err := m.UpdateTaskMetadata(task.ID, db.TaskPatch{WorkerCount: &count}); err != nil {
t.Fatal(err)
}
for range 2 {
if _, err := task.Store.AddIntent(map[string]any{"text": "fixture"}, 0, nil, "test"); err != nil {
t.Fatal(err)
}
}
e := NewEngine(m)
defer e.StopTask(task.ID)
var mu sync.Mutex
providers := make(map[string]*resizingProvider)
e.SetAuthoritativeAgentResolver(func(*Task) (*agent.Planner, *agent.Worker) { return nil, nil })
// Use a pinned pool so each slot's blocked call can be released independently.
profileID, err := m.pg.SaveProfile(&db.LLMProfile{Name: "resize-fixture", Format: "openai", Model: "fixture"})
if err != nil {
t.Fatal(err)
}
defer m.pg.DeleteProfile(profileID)
oldPools, _, _ := m.pg.GetSetting(settingWorkerPools)
oldWorkers, _, _ := m.pg.GetSetting(settingWorkers)
defer m.pg.SetSetting(settingWorkerPools, oldPools)
defer m.pg.SetSetting(settingWorkers, oldWorkers)
count = 0
if _, err := m.UpdateTaskMetadata(task.ID, db.TaskPatch{WorkerCount: &count}); err != nil {
t.Fatal(err)
}
workDir := t.TempDir()
e.SetWorkerResolver(func(_ *Task, slot WorkerSlot) (*agent.Worker, error) {
mu.Lock()
defer mu.Unlock()
p := providers[slot.Name]
if p == nil {
p = &resizingProvider{calls: make(chan resizingCall, 4)}
providers[slot.Name] = p
}
w := agent.NewWorker(p, "fixture", workDir, nil, 0, 0)
w.SetNonStreaming(func() bool { return true })
return w, nil
})
if err := m.SetWorkerPools([]WorkerPool{{ProfileID: profileID, Workers: 2}}); err != nil {
t.Fatal(err)
}
e.Run(t.Context(), task)
getCall := func(name string) resizingCall {
t.Helper()
var p *resizingProvider
waitWorkerCondition(t, func() bool {
mu.Lock()
defer mu.Unlock()
p = providers[name]
return p != nil
})
select {
case call := <-p.calls:
return call
case <-time.After(5 * time.Second):
t.Fatalf("%s did not execute an intent", name)
return resizingCall{}
}
}
first, second := getCall("work#1"), getCall("work#2")
thirdID, err := task.Store.AddIntent(map[string]any{"text": "next"}, 0, nil, "test")
if err != nil {
t.Fatal(err)
}
if err := m.SetWorkerPools([]WorkerPool{{ProfileID: profileID, Workers: 1}}); err != nil {
t.Fatal(err)
}
if actual, target := e.WorkerCounts(task); actual != 2 || target != 1 {
t.Fatalf("draining counts=%d/%d, want 2/1", actual, target)
}
if second.ctx.Err() != nil || first.ctx.Err() != nil {
t.Fatal("resize cancelled an executing intent")
}
close(second.release)
waitWorkerCondition(t, func() bool { actual, target := e.WorkerCounts(task); return actual == 1 && target == 1 })
third, err := task.Store.GetNode(thirdID)
if err != nil || third.State != "open" {
t.Fatalf("retired slot claimed queued intent: node=%+v err=%v", third, err)
}
if err := m.SetWorkerPools([]WorkerPool{{ProfileID: profileID, Workers: 2}}); err != nil {
t.Fatal(err)
}
thirdCall := getCall("work#2")
third, err = task.Store.GetNode(thirdID)
if err != nil || third.State != "running" || third.Owner != "work#2" {
t.Fatalf("expanded slot did not claim queued intent: node=%+v err=%v", third, err)
}
// A rapid shrink/grow must not start a duplicate work#2 while it drains.
if err := m.SetWorkerPools([]WorkerPool{{ProfileID: profileID, Workers: 1}}); err != nil {
t.Fatal(err)
}
if err := m.SetWorkerPools([]WorkerPool{{ProfileID: profileID, Workers: 2}}); err != nil {
t.Fatal(err)
}
if actual, target := e.WorkerCounts(task); actual != 2 || target != 2 || thirdCall.ctx.Err() != nil {
t.Fatalf("rapid resize interrupted work or duplicated slot: %d/%d err=%v", actual, target, thirdCall.ctx.Err())
}
close(thirdCall.release)
close(first.release)
waitWorkerCondition(t, func() bool {
node, _ := task.Store.GetNode(thirdID)
return node != nil && node.State != "running" && e.inflightCount(task.ID) == 0
})
}
func TestWorkerHotResizeTaskOverrideAndGlobalDefault(t *testing.T) {
m, err := NewManager(t.TempDir(), "")
if err != nil {
t.Skipf("postgres unavailable: %v", err)
}
defer m.Close()
oldPools, _, _ := m.pg.GetSetting(settingWorkerPools)
oldWorkers, _, _ := m.pg.GetSetting(settingWorkers)
defer m.pg.SetSetting(settingWorkerPools, oldPools)
defer m.pg.SetSetting(settingWorkers, oldWorkers)
if err := m.SetWorkerPools(nil, 2); err != nil {
t.Fatal(err)
}
e := NewEngine(m)
global, err := m.CreateTask("global resize fixture", "fixture", nil, 0, 0)
if err != nil {
t.Fatal(err)
}
defer m.DeleteTask(global.ID, DeleteTaskOptions{})
defer e.StopTask(global.ID)
override, err := m.CreateTask("explicit resize fixture", "fixture", nil, 0, 0)
if err != nil {
t.Fatal(err)
}
defer m.DeleteTask(override.ID, DeleteTaskOptions{})
defer e.StopTask(override.ID)
count := 3
if _, err := m.UpdateTaskMetadata(override.ID, db.TaskPatch{WorkerCount: &count}); err != nil {
t.Fatal(err)
}
e.Run(t.Context(), global)
e.Run(t.Context(), override)
if err := m.SetWorkers(4); err != nil {
t.Fatal(err)
}
if actual, target := e.WorkerCounts(global); actual != 4 || target != 4 {
t.Fatalf("global default did not expand: %d/%d", actual, target)
}
if actual, target := e.WorkerCounts(override); actual != 3 || target != 3 {
t.Fatalf("global setting changed explicit override: %d/%d", actual, target)
}
if err := m.SetWorkers(1); err != nil {
t.Fatal(err)
}
waitWorkerCondition(t, func() bool { actual, target := e.WorkerCounts(global); return actual == 1 && target == 1 })
if actual, target := e.WorkerCounts(override); actual != 3 || target != 3 {
t.Fatalf("global shrink changed explicit override: %d/%d", actual, target)
}
// Exercise the live PATCH path, then restore global inheritance through PATCH.
s := &Server{m: m, engine: e, ctx: t.Context()}
patch := func(count int) {
t.Helper()
req := httptest.NewRequest(http.MethodPatch, "/api/tasks/"+override.ID, strings.NewReader(fmt.Sprintf(`{"worker_count":%d}`, count)))
req.SetPathValue("id", override.ID)
rec := httptest.NewRecorder()
s.updateTaskMetadata(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("live patch=%d: %s", rec.Code, rec.Body.String())
}
}
patch(5)
if actual, target := e.WorkerCounts(override); actual != 5 || target != 5 {
t.Fatalf("task PATCH did not expand: %d/%d", actual, target)
}
patch(0)
waitWorkerCondition(t, func() bool { actual, target := e.WorkerCounts(override); return actual == 1 && target == 1 })
persisted, err := m.pg.GetTask(taskIDInt(t, override.ID))
if err != nil || persisted.WorkerCount != 0 {
t.Fatalf("inheritance not persisted: task=%+v err=%v", persisted, err)
}
// Reconstructing a task after restart must use the persisted override policy.
restored := taskFromPG(persisted, override.Store, m.interceptor)
if slots, err := workerSlotsForTask(restored, m.WorkerPools(), m.Workers()); err != nil || len(slots) != 1 {
t.Fatalf("restored task lost concurrency policy: slots=%v err=%v", slots, err)
}
}