Compare commits

..
Author SHA1 Message Date
ji ruo 7d9c5699cb feat(workers): hot-reload task worker concurrency
Support live task overrides and global default/pool reconciliation.
Drain retiring workers without cancelling active intents; expose actual/target loop counts in the API and task UI.
Persist worker policy and cover resizing, API validation, settings notification, and restoration with regression tests.

Fixes #171
2026-11-19 17:26:51 +08:00
21 changed files with 960 additions and 98 deletions

No files matched your search

+5
View File
@@ -4,6 +4,11 @@
## [Unreleased]
### 新增与改进
- 支持热更新运行中任务的 Worker 并发:任务详情可设置 0(跟随全局)或 1–50;全局工作并发与 Worker 池变更会同步至跟随全局的任务。增容立即补起 Worker,缩容等待当前意图完成后退出,不中断正在执行的工作;详情展示实际存活 Worker 数与目标数。配置持久化,重启恢复时沿用。
- `PATCH /api/tasks/{id}` 新增 `worker_count`;任务接口返回 `worker_actual_count`(存活池循环,含缩容等待中的 Worker)与 `worker_target_count`(有效目标数),与 `in_flight`(执行中意图数)区分。
## [0.3.30] - 2026-11-14
### 新增与改进
+38
View File
@@ -102,3 +102,41 @@ func TestDeleteConversationsReturnsExistingIDs(t *testing.T) {
t.Fatalf("deleted=%v, want existing ids only", deleted)
}
}
func TestTaskPatchWorkerCountPersistsAndValidates(t *testing.T) {
d, err := Open(testDSN(t))
if err != nil {
t.Skipf("postgres unavailable (%v) — skipping", err)
}
defer d.Close()
task, err := d.CreateTask("worker count patch", "goal", nil, 0, 0)
if err != nil {
t.Fatal(err)
}
defer func() { _ = d.DeleteTask(task.ID) }()
for _, count := range []int{1, MaxTaskWorkerCount, 0, 4} {
updated, err := d.UpdateTask(task.ID, TaskPatch{WorkerCount: &count})
if err != nil || updated == nil || updated.WorkerCount != count {
t.Fatalf("patch worker_count=%d task=%+v err=%v", count, updated, err)
}
persisted, err := d.GetTask(task.ID)
if err != nil || persisted == nil || persisted.WorkerCount != count {
t.Fatalf("read worker_count=%d task=%+v err=%v", count, persisted, err)
}
}
name := "renamed without worker override"
updated, err := d.UpdateTask(task.ID, TaskPatch{Name: &name})
if err != nil || updated == nil || updated.WorkerCount != 4 {
t.Fatalf("omitted worker_count changed override: task=%+v err=%v", updated, err)
}
for _, count := range []int{-1, MaxTaskWorkerCount + 1} {
invalidName := "must not commit"
if updated, err := d.UpdateTask(task.ID, TaskPatch{Name: &invalidName, WorkerCount: &count}); err == nil || updated != nil {
t.Fatalf("accepted invalid worker_count=%d: task=%+v err=%v", count, updated, err)
}
persisted, err := d.GetTask(task.ID)
if err != nil || persisted == nil || persisted.WorkerCount != 4 || persisted.Name != name {
t.Fatalf("invalid patch changed task=%+v err=%v", persisted, err)
}
}
}
+14 -4
View File
@@ -165,6 +165,9 @@ func normalizeWorkerCount(n int) int {
}
func validateWorkerCount(n int) error {
if n < 0 {
return fmt.Errorf("Worker 运行数量不能为负数")
}
if n > MaxTaskWorkerCount {
return fmt.Errorf("Worker 运行数量最多 %d", MaxTaskWorkerCount)
}
@@ -471,17 +474,23 @@ ORDER BY (pinned_at IS NOT NULL) DESC, pinned_at DESC NULLS LAST, id DESC`)
return out, nil
}
// TaskPatch updates editable metadata and global asset visibility.
// TaskPatch updates editable metadata, asset visibility, and worker concurrency.
type TaskPatch struct {
Name *string
Remark *string
Pinned *bool
ShareGlobalAssets *bool
WorkerCount *int
}
// UpdateTask applies a partial metadata/visibility mutation and returns the updated row.
// UpdateTask applies a partial metadata/runtime-policy mutation and returns the updated row.
// Re-pinning an already pinned task preserves its original position.
func (d *DB) UpdateTask(id int64, patch TaskPatch) (*Task, error) {
if patch.WorkerCount != nil {
if err := validateWorkerCount(*patch.WorkerCount); err != nil {
return nil, err
}
}
task, err := scanTask(d.QueryRow(`UPDATE tasks SET
name = CASE WHEN $2::boolean THEN $3 ELSE name END,
remark = CASE WHEN $5::boolean THEN $6 ELSE remark END,
@@ -490,9 +499,10 @@ func (d *DB) UpdateTask(id int64, patch TaskPatch) (*Task, error) {
WHEN $4::boolean THEN COALESCE(pinned_at, now())
ELSE NULL
END,
share_global_assets = COALESCE($7::boolean, share_global_assets)
share_global_assets = COALESCE($7::boolean, share_global_assets),
worker_count = COALESCE($8::integer, worker_count)
WHERE id=$1 AND deleted_at IS NULL
RETURNING `+taskCols, id, patch.Name != nil, patch.Name, patch.Pinned, patch.Remark != nil, patch.Remark, patch.ShareGlobalAssets))
RETURNING `+taskCols, id, patch.Name != nil, patch.Name, patch.Pinned, patch.Remark != nil, patch.Remark, patch.ShareGlobalAssets, patch.WorkerCount))
if err == sql.ErrNoRows {
return nil, nil
}
+4 -2
View File
@@ -70,7 +70,9 @@ type TaskDTO struct {
CompanyIDs []int64 `json:"company_ids"`
CoverageEnabled bool `json:"coverage_enabled"` // 资产覆盖度功能开关(创建时定)
ShareGlobalAssets bool `json:"share_global_assets"`
WorkerCount int `json:"worker_count"` // 本任务 Worker 数;0=用全局默认
WorkerCount int `json:"worker_count"` // 本任务 Worker 数;0=用全局默认
WorkerActualCount int `json:"worker_actual_count"` // live pool loops, including draining workers
WorkerTargetCount int `json:"worker_target_count"` // desired worker slots
NoResultSeconds int `json:"no_result_seconds"`
NoResultMinFindings int `json:"no_result_min_findings"`
@@ -170,7 +172,7 @@ func taskDTO(t *Task, status string) TaskDTO {
CompanyIDs: lifecycle.CompanyIDs,
CoverageEnabled: t.CoverageEnabled,
ShareGlobalAssets: lifecycle.ShareGlobalAssets,
WorkerCount: t.WorkerCount,
WorkerCount: lifecycle.WorkerCount,
NoResultSeconds: t.NoResultSeconds,
NoResultMinFindings: t.NoResultMinFindings,
+42 -47
View File
@@ -111,8 +111,9 @@ type Engine struct {
// Every long-lived task goroutine (planner, workers and deadline coordinator)
// runs under one task-scoped context. Successful deletion cancels that context,
// waits for all goroutines, then releases every task-level Engine reference.
runtimeMu sync.Mutex
runtimes map[string]*taskRuntime
runtimeMu sync.Mutex
runtimes map[string]*taskRuntime
workerPools sync.Map // taskID -> *taskWorkerPool; each pool serializes claims with resizing
// per-task execution context: each planner.Plan / worker.Execute runs under it,
// so pausing can CANCEL an in-flight run (not just skip the next one). Recreated
@@ -150,7 +151,7 @@ type Engine struct {
resolveAuthoritative bool
workerResolve func(t *Task, slot WorkerSlot) (*agent.Worker, error)
slotState sync.Map // task/slot -> slotAvailability
slotPlans sync.Map // task -> map[slot name]profile ID; immutable for a run
slotPlans sync.Map // task -> immutable snapshot of slot name/profile routing
// readiness reports whether a global LLM provider is configured — the signal behind
// Ready()/the llm_configured indicator. Wired once at startup; nil → not ready.
readiness func() bool
@@ -297,6 +298,7 @@ func (e *Engine) StopTask(taskID string) {
e.inflight.Delete(taskID)
e.coordStarted.Delete(taskID)
e.slotPlans.Delete(taskID)
e.workerPools.Delete(taskID)
e.slotState.Range(func(key, _ any) bool {
keyText, ok := key.(string)
if ok && strings.HasPrefix(keyText, taskID+"/") {
@@ -406,10 +408,14 @@ func (e *Engine) ActiveLLMCalls(taskID string) int64 {
func (e *Engine) touch(taskID string) { e.lastAct.Store(taskID, time.Now().Unix()) }
func NewEngine(m *Manager) *Engine {
return &Engine{m: m, debounce: 800 * time.Millisecond, bc: NewBroadcaster(),
e := &Engine{m: m, debounce: 800 * time.Millisecond, bc: NewBroadcaster(),
execCancel: map[string]context.CancelCauseFunc{}, execCtx: map[string]context.Context{},
work: map[int64]*workExecution{}, steerBox: map[int64][]string{},
runtimes: map[string]*taskRuntime{}}
if m != nil {
m.SetWorkerConfigChangedCallback(e.ReconcileDefaultWorkers)
}
return e
}
// registerWork records the cancel for the work currently running intentID.
@@ -796,11 +802,6 @@ func (e *Engine) ReadyFor(t *Task) bool {
// but no-op until an LLM is configured (so a task created while idle picks up
// automatically once LLM is set from the UI).
func (e *Engine) Run(ctx context.Context, t *Task) {
slots, err := workerSlotsForTask(t, e.m.WorkerPools(), e.m.Workers())
if err != nil {
log.Printf("[engine] task %s worker pool invalid, using legacy workers: %v", t.ID, err)
slots, _ = ExpandWorkerSlots(nil, workerCountForTask(t, e.m.Workers()))
}
e.deleteMu.RLock()
if e.IsDeleting(t.ID) {
e.deleteMu.RUnlock()
@@ -811,19 +812,13 @@ func (e *Engine) Run(ctx context.Context, t *Task) {
t.Notify() // already running — just nudge a planning round
return
}
rt := e.registerTaskRoutines(ctx, t.ID, 1+len(slots))
plan := make(map[string]int64, len(slots))
for _, slot := range slots {
plan[slot.Name] = slot.ProfileID
}
e.slotPlans.Store(t.ID, plan)
rt := e.registerTaskRoutines(ctx, t.ID, 1)
pool := &taskWorkerPool{runtime: rt, task: t, loops: make(map[string]*workerSlotRuntime)}
e.workerPools.Store(t.ID, pool)
e.deleteMu.RUnlock()
e.touch(t.ID)
runTaskRoutine(rt, func(loopCtx context.Context) { e.plannerLoop(loopCtx, t) })
for _, slot := range slots {
slot := slot
runTaskRoutine(rt, func(loopCtx context.Context) { e.workerLoop(loopCtx, t, slot) })
}
e.ReconcileWorkers(t)
e.startDeadlineCoordinator(ctx, t) // 任务级超时定时器(仅 timeout>0;去重)
// 仅在「完全没有活动意图(open+running)」时才 kick 首轮规划。带种子意图的任务:种子已
// 是 open,或已被上面刚起的 worker 抢先 claim 成 running——两种都算「有活干」,一律跳过
@@ -838,20 +833,20 @@ func (e *Engine) Run(ctx context.Context, t *Task) {
// workerSlotsForTask preserves an explicit task concurrency override. Tasks
// using the global default inherit its model pools and their pinned profiles.
func workerSlotsForTask(t *Task, pools []WorkerPool, global int) ([]WorkerSlot, error) {
if t != nil && t.WorkerCount > 0 {
if t != nil && t.lifecycleSnapshot().WorkerCount > 0 {
return ExpandWorkerSlots(nil, workerCountForTask(t, global))
}
return ExpandWorkerSlots(pools, global)
}
func workerCountForTask(t *Task, global int) int {
if t != nil && t.WorkerCount > 0 {
// Direct database edits can bypass validation; cap those values so the
// engine never starts more than the supported per-task worker count.
if t.WorkerCount > db.MaxTaskWorkerCount {
return db.MaxTaskWorkerCount
if t != nil {
if count := t.lifecycleSnapshot().WorkerCount; count > 0 {
if count > db.MaxTaskWorkerCount {
return db.MaxTaskWorkerCount
}
return count
}
return t.WorkerCount
}
return global
}
@@ -1004,11 +999,12 @@ func (e *Engine) plannerLoop(ctx context.Context, t *Task) {
}
}
func (e *Engine) workerLoop(ctx context.Context, t *Task, slot WorkerSlot) {
func (e *Engine) workerLoop(ctx context.Context, t *Task, slot WorkerSlot, control *workerSlotRuntime) {
name := slot.Name
idleCtx := control.idleCtx
for {
select {
case <-ctx.Done():
case <-idleCtx.Done():
return
default:
}
@@ -1031,13 +1027,13 @@ func (e *Engine) workerLoop(ctx context.Context, t *Task, slot WorkerSlot) {
_, worker = e.snapshotFor(t)
}
if worker == nil {
if sleepCtx(ctx, 1500*time.Millisecond) {
if sleepCtx(idleCtx, 1500*time.Millisecond) {
return
}
continue
}
if e.IsPaused(t.ID) {
if sleepCtx(ctx, 1000*time.Millisecond) {
if sleepCtx(idleCtx, 1000*time.Millisecond) {
return
}
continue // user-paused: don't claim/execute intents
@@ -1046,13 +1042,13 @@ func (e *Engine) workerLoop(ctx context.Context, t *Task, slot WorkerSlot) {
return
}
if e.isSettling(t.ID) {
if sleepCtx(ctx, 1000*time.Millisecond) {
if sleepCtx(idleCtx, 1000*time.Millisecond) {
return
}
continue // 任务超时收尾中:不再领新意图(在跑的自行收尾,协调器等其 drain)
}
if isTerminalStatus(e.m.TaskStatus(t.ID)) {
if sleepCtx(ctx, 1000*time.Millisecond) {
if sleepCtx(idleCtx, 1000*time.Millisecond) {
return
}
continue // 任务已终态(done/failed/timeout):停止领取遗留意图,别在完成后空跑 frontier
@@ -1060,28 +1056,27 @@ func (e *Engine) workerLoop(ctx context.Context, t *Task, slot WorkerSlot) {
if !e.beginTaskOperation(t.ID) {
return
}
claimed := e.runWorkerStep(ctx, t, name, worker)
// A resize and the next claim share the pool lock. Once retirement is
// requested this loop cannot claim again; execution keeps the task context.
control.pool.mu.Lock()
var intent *db.Node
if idleCtx.Err() == nil {
intent = e.claimNext(t, name)
}
control.pool.mu.Unlock()
if intent != nil {
log.Printf("[worker %s] task %s 领取意图 #%d", name, t.ID, intent.ID)
e.runIntent(ctx, t, name, worker, intent, "", "")
}
e.decInflight(t.ID)
if !claimed && sleepCtx(ctx, 800*time.Millisecond) {
if intent == nil && sleepCtx(idleCtx, 800*time.Millisecond) {
return
}
}
}
// runWorkerStep claims one intent from the frontier and fully settles it via
// runIntent. Returns false when nothing was claimable. The pool worker loop is its
// only caller.
func (e *Engine) runWorkerStep(ctx context.Context, t *Task, name string, worker *agent.Worker) bool {
intent := e.claimNext(t, name)
if intent == nil {
return false
}
log.Printf("[worker %s] task %s 领取意图 #%d", name, t.ID, intent.ID)
return e.runIntent(ctx, t, name, worker, intent, "", "")
}
// runIntent executes and fully settles one already-claimed (state=running) intent.
// Both the pool worker loop (via runWorkerStep) and the human-message handler (via
// Both the pool worker loop and the human-message handler (via
// runDetachedIntent, a dedicated goroutine outside the worker pool) call it, so the
// execute/retry/state-write logic lives in exactly one place. A non-empty message
// is injected as this turn's input through ExecuteWithMessage; requestID keys the
+131
View File
@@ -0,0 +1,131 @@
package server
import (
"context"
"log"
"sync"
)
// taskWorkerPool owns the desired plan and all live loops, including retirees.
// A slot name is never reused until its previous loop has fully settled its work.
type taskWorkerPool struct {
mu sync.Mutex
runtime *taskRuntime
task *Task
loops map[string]*workerSlotRuntime
desired map[string]WorkerSlot
}
type workerSlotRuntime struct {
pool *taskWorkerPool
slot WorkerSlot
idleCtx context.Context
retire context.CancelFunc // stops claiming/sleeping, never cancels Execute
}
func (e *Engine) desiredWorkerSlots(t *Task) []WorkerSlot {
slots, err := workerSlotsForTask(t, e.m.WorkerPools(), e.m.Workers())
if err != nil {
log.Printf("[engine] task %s worker pool invalid, using legacy workers: %v", t.ID, err)
slots, _ = ExpandWorkerSlots(nil, workerCountForTask(t, e.m.Workers()))
}
return slots
}
// ReconcileWorkers applies the latest persisted settings without restarting a task.
// Claims and retirement share the pool lock, so no retired slot takes new work.
func (e *Engine) ReconcileWorkers(t *Task) {
e.deleteMu.RLock()
defer e.deleteMu.RUnlock()
if e.IsDeleting(t.ID) {
return
}
value, ok := e.workerPools.Load(t.ID)
if !ok {
return // queued/not started: Run will read the latest configuration
}
pool := value.(*taskWorkerPool)
pool.mu.Lock()
defer pool.mu.Unlock()
if pool.runtime.ctx.Err() != nil {
return
}
slots := e.desiredWorkerSlots(t)
pool.desired = make(map[string]WorkerSlot, len(slots))
for _, slot := range slots {
pool.desired[slot.Name] = slot
}
for name, loop := range pool.loops {
if slot, keep := pool.desired[name]; !keep || slot != loop.slot {
loop.retire()
}
}
for _, slot := range slots {
if pool.loops[slot.Name] == nil {
e.startWorkerSlotLocked(pool, slot)
}
}
}
// ReconcileDefaultWorkers is called after a global workers/pool save. Explicit
// task overrides keep both their concurrency and legacy task-router profiles.
func (e *Engine) ReconcileDefaultWorkers() {
e.workerPools.Range(func(_, value any) bool {
t := value.(*taskWorkerPool).task
if t.lifecycleSnapshot().WorkerCount == 0 {
e.ReconcileWorkers(t)
}
return true
})
}
// WorkerCounts distinguishes live pool loops (including draining workers) from
// the effective configured target. Neither count is the number of busy intents.
func (e *Engine) WorkerCounts(t *Task) (actual, target int) {
if value, ok := e.workerPools.Load(t.ID); ok {
pool := value.(*taskWorkerPool)
pool.mu.Lock()
defer pool.mu.Unlock()
return len(pool.loops), len(pool.desired)
}
return 0, len(e.desiredWorkerSlots(t))
}
// Caller holds deleteMu.RLock and pool.mu. The delete barrier protects wg.Add
// from StopTask.Wait, including replacements started as a retiree exits.
func (e *Engine) startWorkerSlotLocked(pool *taskWorkerPool, slot WorkerSlot) {
idleCtx, retire := context.WithCancel(pool.runtime.ctx)
loop := &workerSlotRuntime{pool: pool, slot: slot, idleCtx: idleCtx, retire: retire}
pool.loops[slot.Name] = loop
// Publish a fresh immutable routing snapshot. Retired slot routes remain
// available for human continuation of paused intents until the name is reused.
plan := make(map[string]int64)
if old, ok := e.slotPlans.Load(pool.task.ID); ok {
for name, profile := range old.(map[string]int64) {
plan[name] = profile
}
}
plan[slot.Name] = slot.ProfileID
e.slotPlans.Store(pool.task.ID, plan)
pool.runtime.wg.Add(1)
runTaskRoutine(pool.runtime, func(ctx context.Context) {
defer e.finishWorkerSlot(loop)
e.workerLoop(ctx, pool.task, slot, loop)
})
}
func (e *Engine) finishWorkerSlot(loop *workerSlotRuntime) {
loop.retire()
e.deleteMu.RLock()
defer e.deleteMu.RUnlock()
pool := loop.pool
pool.mu.Lock()
defer pool.mu.Unlock()
delete(pool.loops, loop.slot.Name)
e.slotState.Delete(pool.task.ID + "/" + loop.slot.Name)
if !e.IsDeleting(pool.task.ID) && pool.runtime.ctx.Err() == nil {
if slot, needed := pool.desired[loop.slot.Name]; needed {
e.startWorkerSlotLocked(pool, slot)
}
}
}
+254
View File
@@ -0,0 +1,254 @@
package server
import (
"context"
"fmt"
"iter"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
"time"
"github.com/Autumn-27/artex/agent"
"github.com/Autumn-27/artex/db"
"github.com/Autumn-27/norma/llm"
)
type resizingCall struct {
ctx context.Context
release chan struct{}
}
type resizingProvider struct{ calls chan resizingCall }
func (p *resizingProvider) Complete(ctx context.Context, _ llm.CompletionRequest) (llm.Message, string, llm.Usage, error) {
call := resizingCall{ctx: ctx, release: make(chan struct{})}
select {
case p.calls <- call:
case <-ctx.Done():
return llm.Message{}, "", llm.Usage{}, ctx.Err()
}
select {
case <-call.release:
return llm.Message{Role: llm.RoleAssistant, Content: []llm.ContentBlock{llm.TextBlock("fixture completed")}}, "end_turn", llm.Usage{}, nil
case <-ctx.Done():
return llm.Message{}, "", llm.Usage{}, ctx.Err()
}
}
func (*resizingProvider) Stream(context.Context, llm.CompletionRequest) iter.Seq2[llm.StreamEvent, error] {
panic("resizing fixture uses non-streaming calls")
}
func waitWorkerCondition(t *testing.T, condition func() bool) {
t.Helper()
deadline := time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) {
if condition() {
return
}
time.Sleep(10 * time.Millisecond)
}
t.Fatal("timed out waiting for worker state")
}
// Exercises the real worker claim/Execute/settlement path. Both slots remain
// blocked in provider calls while shrinking; only the retained slot may claim
// another intent, and re-expansion must reuse a name without overlapping loops.
func TestWorkerHotResizeDrainsBeforeReusingSlot(t *testing.T) {
m, err := NewManager(t.TempDir(), "")
if err != nil {
t.Skipf("postgres unavailable: %v", err)
}
defer m.Close()
task, err := m.CreateTask("resize workers", "fixture", nil, 0, 0)
if err != nil {
t.Fatal(err)
}
defer m.DeleteTask(task.ID, DeleteTaskOptions{})
count := 2
if _, err := m.UpdateTaskMetadata(task.ID, db.TaskPatch{WorkerCount: &count}); err != nil {
t.Fatal(err)
}
for range 2 {
if _, err := task.Store.AddIntent(map[string]any{"text": "fixture"}, 0, nil, "test"); err != nil {
t.Fatal(err)
}
}
e := NewEngine(m)
defer e.StopTask(task.ID)
var mu sync.Mutex
providers := make(map[string]*resizingProvider)
e.SetAuthoritativeAgentResolver(func(*Task) (*agent.Planner, *agent.Worker) { return nil, nil })
// Use a pinned pool so each slot's blocked call can be released independently.
profileID, err := m.pg.SaveProfile(&db.LLMProfile{Name: "resize-fixture", Format: "openai", Model: "fixture"})
if err != nil {
t.Fatal(err)
}
defer m.pg.DeleteProfile(profileID)
oldPools, _, _ := m.pg.GetSetting(settingWorkerPools)
oldWorkers, _, _ := m.pg.GetSetting(settingWorkers)
defer m.pg.SetSetting(settingWorkerPools, oldPools)
defer m.pg.SetSetting(settingWorkers, oldWorkers)
count = 0
if _, err := m.UpdateTaskMetadata(task.ID, db.TaskPatch{WorkerCount: &count}); err != nil {
t.Fatal(err)
}
workDir := t.TempDir()
e.SetWorkerResolver(func(_ *Task, slot WorkerSlot) (*agent.Worker, error) {
mu.Lock()
defer mu.Unlock()
p := providers[slot.Name]
if p == nil {
p = &resizingProvider{calls: make(chan resizingCall, 4)}
providers[slot.Name] = p
}
w := agent.NewWorker(p, "fixture", workDir, nil, 0, 0)
w.SetNonStreaming(func() bool { return true })
return w, nil
})
if err := m.SetWorkerPools([]WorkerPool{{ProfileID: profileID, Workers: 2}}); err != nil {
t.Fatal(err)
}
e.Run(t.Context(), task)
getCall := func(name string) resizingCall {
t.Helper()
var p *resizingProvider
waitWorkerCondition(t, func() bool {
mu.Lock()
defer mu.Unlock()
p = providers[name]
return p != nil
})
select {
case call := <-p.calls:
return call
case <-time.After(5 * time.Second):
t.Fatalf("%s did not execute an intent", name)
return resizingCall{}
}
}
first, second := getCall("work#1"), getCall("work#2")
thirdID, err := task.Store.AddIntent(map[string]any{"text": "next"}, 0, nil, "test")
if err != nil {
t.Fatal(err)
}
if err := m.SetWorkerPools([]WorkerPool{{ProfileID: profileID, Workers: 1}}); err != nil {
t.Fatal(err)
}
if actual, target := e.WorkerCounts(task); actual != 2 || target != 1 {
t.Fatalf("draining counts=%d/%d, want 2/1", actual, target)
}
if second.ctx.Err() != nil || first.ctx.Err() != nil {
t.Fatal("resize cancelled an executing intent")
}
close(second.release)
waitWorkerCondition(t, func() bool { actual, target := e.WorkerCounts(task); return actual == 1 && target == 1 })
third, err := task.Store.GetNode(thirdID)
if err != nil || third.State != "open" {
t.Fatalf("retired slot claimed queued intent: node=%+v err=%v", third, err)
}
if err := m.SetWorkerPools([]WorkerPool{{ProfileID: profileID, Workers: 2}}); err != nil {
t.Fatal(err)
}
thirdCall := getCall("work#2")
third, err = task.Store.GetNode(thirdID)
if err != nil || third.State != "running" || third.Owner != "work#2" {
t.Fatalf("expanded slot did not claim queued intent: node=%+v err=%v", third, err)
}
// A rapid shrink/grow must not start a duplicate work#2 while it drains.
if err := m.SetWorkerPools([]WorkerPool{{ProfileID: profileID, Workers: 1}}); err != nil {
t.Fatal(err)
}
if err := m.SetWorkerPools([]WorkerPool{{ProfileID: profileID, Workers: 2}}); err != nil {
t.Fatal(err)
}
if actual, target := e.WorkerCounts(task); actual != 2 || target != 2 || thirdCall.ctx.Err() != nil {
t.Fatalf("rapid resize interrupted work or duplicated slot: %d/%d err=%v", actual, target, thirdCall.ctx.Err())
}
close(thirdCall.release)
close(first.release)
waitWorkerCondition(t, func() bool {
node, _ := task.Store.GetNode(thirdID)
return node != nil && node.State != "running" && e.inflightCount(task.ID) == 0
})
}
func TestWorkerHotResizeTaskOverrideAndGlobalDefault(t *testing.T) {
m, err := NewManager(t.TempDir(), "")
if err != nil {
t.Skipf("postgres unavailable: %v", err)
}
defer m.Close()
oldPools, _, _ := m.pg.GetSetting(settingWorkerPools)
oldWorkers, _, _ := m.pg.GetSetting(settingWorkers)
defer m.pg.SetSetting(settingWorkerPools, oldPools)
defer m.pg.SetSetting(settingWorkers, oldWorkers)
if err := m.SetWorkerPools(nil, 2); err != nil {
t.Fatal(err)
}
e := NewEngine(m)
global, err := m.CreateTask("global resize fixture", "fixture", nil, 0, 0)
if err != nil {
t.Fatal(err)
}
defer m.DeleteTask(global.ID, DeleteTaskOptions{})
defer e.StopTask(global.ID)
override, err := m.CreateTask("explicit resize fixture", "fixture", nil, 0, 0)
if err != nil {
t.Fatal(err)
}
defer m.DeleteTask(override.ID, DeleteTaskOptions{})
defer e.StopTask(override.ID)
count := 3
if _, err := m.UpdateTaskMetadata(override.ID, db.TaskPatch{WorkerCount: &count}); err != nil {
t.Fatal(err)
}
e.Run(t.Context(), global)
e.Run(t.Context(), override)
if err := m.SetWorkers(4); err != nil {
t.Fatal(err)
}
if actual, target := e.WorkerCounts(global); actual != 4 || target != 4 {
t.Fatalf("global default did not expand: %d/%d", actual, target)
}
if actual, target := e.WorkerCounts(override); actual != 3 || target != 3 {
t.Fatalf("global setting changed explicit override: %d/%d", actual, target)
}
if err := m.SetWorkers(1); err != nil {
t.Fatal(err)
}
waitWorkerCondition(t, func() bool { actual, target := e.WorkerCounts(global); return actual == 1 && target == 1 })
if actual, target := e.WorkerCounts(override); actual != 3 || target != 3 {
t.Fatalf("global shrink changed explicit override: %d/%d", actual, target)
}
// Exercise the live PATCH path, then restore global inheritance through PATCH.
s := &Server{m: m, engine: e, ctx: t.Context()}
patch := func(count int) {
t.Helper()
req := httptest.NewRequest(http.MethodPatch, "/api/tasks/"+override.ID, strings.NewReader(fmt.Sprintf(`{"worker_count":%d}`, count)))
req.SetPathValue("id", override.ID)
rec := httptest.NewRecorder()
s.updateTaskMetadata(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("live patch=%d: %s", rec.Code, rec.Body.String())
}
}
patch(5)
if actual, target := e.WorkerCounts(override); actual != 5 || target != 5 {
t.Fatalf("task PATCH did not expand: %d/%d", actual, target)
}
patch(0)
waitWorkerCondition(t, func() bool { actual, target := e.WorkerCounts(override); return actual == 1 && target == 1 })
persisted, err := m.pg.GetTask(taskIDInt(t, override.ID))
if err != nil || persisted.WorkerCount != 0 {
t.Fatalf("inheritance not persisted: task=%+v err=%v", persisted, err)
}
// Reconstructing a task after restart must use the persisted override policy.
restored := taskFromPG(persisted, override.Store, m.interceptor)
if slots, err := workerSlotsForTask(restored, m.WorkerPools(), m.Workers()); err != nil || len(slots) != 1 {
t.Fatalf("restored task lost concurrency policy: slots=%v err=%v", slots, err)
}
}
+55 -19
View File
@@ -99,6 +99,7 @@ type Task struct {
// directly after the task has been published by Manager.
type taskLifecycleState struct {
ShareGlobalAssets bool
WorkerCount int
Name string
Remark string
PinnedAt int64
@@ -140,6 +141,7 @@ func (t *Task) lifecycleSnapshot() taskLifecycleState {
func (t *Task) lifecycleSnapshotLocked() taskLifecycleState {
return taskLifecycleState{
ShareGlobalAssets: t.ShareGlobalAssets,
WorkerCount: t.WorkerCount,
Name: t.Name,
Remark: t.Remark,
PinnedAt: t.PinnedAt,
@@ -171,6 +173,7 @@ func (t *Task) updateLifecycle(update func(*taskLifecycleState)) {
state := t.lifecycleSnapshotLocked()
update(&state)
t.ShareGlobalAssets = state.ShareGlobalAssets
t.WorkerCount = state.WorkerCount
t.Name = state.Name
t.Remark = state.Remark
t.PinnedAt = state.PinnedAt
@@ -282,13 +285,14 @@ type Manager struct {
// taskStateMu preserves commit order between PostgreSQL lifecycle writes and
// their in-memory mirrors. lifecycleMu makes snapshots race-free, but without
// this outer write lock an older request could commit first and publish last.
taskStateMu sync.Mutex
controlMu *sync.Mutex // Server's admission/task-control mutex, when attached.
mu sync.RWMutex
tasks map[string]*Task
active string
trafficOn bool // 流量捕获开关(默认关;settings.traffic_capture)
llmRecOn bool // LLM 录制开关(默认关;settings.llm_record)
taskStateMu sync.Mutex
controlMu *sync.Mutex // Server's admission/task-control mutex, when attached.
mu sync.RWMutex
tasks map[string]*Task
workerConfigChanged func()
active string
trafficOn bool // 流量捕获开关(默认关;settings.traffic_capture)
llmRecOn bool // LLM 录制开关(默认关;settings.llm_record)
// 联网搜索开关与来源(默认关;settings.web_search_*)。brave-free 需要 braveKey;tavily 需要 tavilyKey。
// webSearchProxy 是独立出口代理(http/https/socks5),与记录流量的 MITM 代理无关。
webSearchOn bool
@@ -310,6 +314,23 @@ type Manager struct {
// participate in the Server's admission/task-control critical section.
func (m *Manager) SetTaskControlMutex(mu *sync.Mutex) { m.controlMu = mu }
// SetWorkerConfigChangedCallback installs the engine's live worker reconciler.
// Successful worker-setting writes invoke it after releasing Manager locks.
func (m *Manager) SetWorkerConfigChangedCallback(callback func()) {
m.mu.Lock()
m.workerConfigChanged = callback
m.mu.Unlock()
}
func (m *Manager) notifyWorkerConfigChanged() {
m.mu.RLock()
callback := m.workerConfigChanged
m.mu.RUnlock()
if callback != nil {
callback()
}
}
// Settings keys the UI toggles at runtime.
const (
settingTrafficCapture = "traffic_capture"
@@ -433,6 +454,9 @@ func ExpandWorkerSlots(pools []WorkerPool, legacy int) ([]WorkerSlot, error) {
if legacy <= 0 {
legacy = defaultWorkers
}
if legacy > pgdb.MaxTaskWorkerCount {
legacy = pgdb.MaxTaskWorkerCount
}
slots := make([]WorkerSlot, legacy)
for i := range slots {
slots[i] = WorkerSlot{Name: fmt.Sprintf("work#%d", i+1)}
@@ -478,8 +502,8 @@ func (m *Manager) SetConcurrency(enabled bool, limit int) error {
return m.pg.SetSetting(settingConcurrencyOn, strconv.FormatBool(enabled))
}
// Workers returns the configured concurrent work-agent count (default 3). Read
// per-task at engine.Run, so a change applies to tasks started afterwards.
// Workers returns the configured concurrent work-agent count (default 3),
// capped at the supported maximum even for settings saved by older versions.
func (m *Manager) Workers() int {
v, ok, err := m.pg.GetSetting(settingWorkers)
if err != nil || !ok {
@@ -489,6 +513,9 @@ func (m *Manager) Workers() int {
if err != nil || n <= 0 {
return defaultWorkers
}
if n > pgdb.MaxTaskWorkerCount {
return pgdb.MaxTaskWorkerCount
}
return n
}
@@ -542,23 +569,31 @@ func (m *Manager) SetWorkerPools(pools []WorkerPool, legacyWorkers ...int) error
workerCount += pool.Workers
}
} else if len(legacyWorkers) > 0 {
if legacyWorkers[0] <= 0 {
return fmt.Errorf("workers 必须 >0")
if legacyWorkers[0] <= 0 || legacyWorkers[0] > pgdb.MaxTaskWorkerCount {
return fmt.Errorf("workers 必须在 1 到 %d 之间", pgdb.MaxTaskWorkerCount)
}
workerCount = legacyWorkers[0]
}
return m.pg.SetSettingsAtomic(context.Background(), map[string]string{
if err := m.pg.SetSettingsAtomic(context.Background(), map[string]string{
settingWorkerPools: string(b),
settingWorkers: strconv.Itoa(workerCount),
})
}); err != nil {
return err
}
m.notifyWorkerConfigChanged()
return nil
}
// SetWorkers persists the concurrent work-agent count. Values <=0 are rejected.
// SetWorkers persists the concurrent work-agent count and reconciles live tasks.
func (m *Manager) SetWorkers(n int) error {
if n <= 0 {
return fmt.Errorf("workers 必须 >0")
if n <= 0 || n > pgdb.MaxTaskWorkerCount {
return fmt.Errorf("workers 必须在 1 到 %d 之间", pgdb.MaxTaskWorkerCount)
}
return m.pg.SetSetting(settingWorkers, strconv.Itoa(n))
if err := m.pg.SetSetting(settingWorkers, strconv.Itoa(n)); err != nil {
return err
}
m.notifyWorkerConfigChanged()
return nil
}
// Enrich returns the asset auto-completion engine (may be nil if init failed).
@@ -1179,8 +1214,8 @@ func taskFromPG(pt *pgdb.Task, store *pgdb.ExplorationStore, ic *intercept.Inter
}
}
// UpdateTaskMetadata changes task metadata and asset visibility without interrupting any
// planner, main-agent, or worker call.
// UpdateTaskMetadata changes task metadata, asset visibility, and worker policy
// without interrupting any planner, main-agent, or worker call.
func (m *Manager) UpdateTaskMetadata(taskID string, patch pgdb.TaskPatch) (*Task, error) {
m.taskStateMu.Lock()
defer m.taskStateMu.Unlock()
@@ -1200,6 +1235,7 @@ func (m *Manager) UpdateTaskMetadata(taskID string, patch pgdb.TaskPatch) (*Task
}
task.updateLifecycle(func(state *taskLifecycleState) {
state.ShareGlobalAssets = updated.ShareGlobalAssets
state.WorkerCount = updated.WorkerCount
state.Name = updated.Name
state.Remark = updated.Remark
state.PinnedAt = unixOrZero(updated.PinnedAt)
+18 -1
View File
@@ -55,6 +55,7 @@ func TestTaskLifecycleSnapshotConcurrentConsistency(t *testing.T) {
state.CompletedAt = generation
state.FirstRunAt = generation
state.DeadlineAt = generation
state.WorkerCount = int(generation % 51)
state.SourceTaskIDs = []int64{generation, generation + 1}
state.CompanyIDs = []int64{generation, generation + 1}
})
@@ -93,7 +94,7 @@ func TestTaskLifecycleSnapshotConcurrentConsistency(t *testing.T) {
}
generation := state.QueuedAt
if state.CompletedAt != generation || state.FirstRunAt != generation || state.DeadlineAt != generation ||
len(state.SourceTaskIDs) != 2 || len(state.CompanyIDs) != 2 ||
state.WorkerCount != int(generation%51) || len(state.SourceTaskIDs) != 2 || len(state.CompanyIDs) != 2 ||
state.SourceTaskIDs[0] != generation || state.CompanyIDs[0] != generation {
errCh <- fmt.Errorf("mixed lifecycle snapshot: %+v", state)
return
@@ -107,3 +108,19 @@ func TestTaskLifecycleSnapshotConcurrentConsistency(t *testing.T) {
t.Fatal(err)
}
}
func TestTaskLifecycleWorkerCountSnapshot(t *testing.T) {
task := &Task{WorkerCount: 2}
if got := taskDTO(task, "created").WorkerCount; got != 2 {
t.Fatalf("DTO worker_count=%d, want 2", got)
}
task.updateLifecycle(func(state *taskLifecycleState) {
state.WorkerCount = 7
})
if got := task.lifecycleSnapshot().WorkerCount; got != 7 {
t.Fatalf("snapshot worker_count=%d, want 7", got)
}
if got := taskDTO(task, "created").WorkerCount; got != 7 {
t.Fatalf("DTO worker_count=%d, want 7", got)
}
}
+4 -6
View File
@@ -1285,6 +1285,7 @@ func (s *Server) listTasks(w http.ResponseWriter, r *http.Request) {
dtos := make([]TaskDTO, 0, len(list))
for _, t := range list {
dto := taskDTO(t, s.resolvedTaskStatus(t))
dto.WorkerActualCount, dto.WorkerTargetCount = s.engine.WorkerCounts(t)
applyTaskArchiveBlocker(&dto, archiveBlockers)
metric := metrics[t.ExpID]
dto.Tokens = tokenTotalDTO(metric.Tokens)
@@ -2068,6 +2069,7 @@ func (s *Server) getTask(w http.ResponseWriter, r *http.Request) {
return
}
dto := taskDTO(t, s.resolvedTaskStatus(t))
dto.WorkerActualCount, dto.WorkerTargetCount = s.engine.WorkerCounts(t)
archiveBlockers, _ := s.m.PG().TaskArchiveBlockers()
applyTaskArchiveBlocker(&dto, archiveBlockers)
writeJSON(w, 200, dto)
@@ -3740,7 +3742,7 @@ func (s *Server) settingsPayload() map[string]any {
"show_egress_ip": s.m.pg.GetBool(settingShowEgressIP, false),
"egress_probe_url": s.m.EgressProbeURL(),
"python_interpreter": strings.TrimSpace(pyStored), // 用户/自动设的值(空=用运行时检测)
"workers": s.m.Workers(), // 并发工作 agent 数(默认3);对之后启动的任务生效
"workers": s.m.Workers(), // 并发工作 agent 数(默认3);实时更新使用全局默认的任务
"worker_pools": s.m.WorkerPools(), // 非空时按池顺序绑定 worker 会话
"task_concurrency_enabled": concOn, // 任务并发上限开关(默认关)
"task_concurrency_limit": concLimit, // 同时运行任务上限(开启后默认5)
@@ -3833,7 +3835,7 @@ func (s *Server) putSettings(w http.ResponseWriter, r *http.Request) {
ShowEgressIP *bool `json:"show_egress_ip"` // 在顶栏显示直连/全局代理出口 IP;默认关
EgressProbeURL *string `json:"egress_probe_url"` // null=不改;空串=默认公共探测服务
PythonInterp *string `json:"python_interpreter"` // 自定义脚本工具的 python 解释器路径
Workers *int `json:"workers"` // 并发工作 agent 数(>0);对之后启动的任务生效
Workers *int `json:"workers"` // 并发工作 agent 数(1..50);实时更新使用全局默认的任务
WorkerPools *[]WorkerPool `json:"worker_pools"`
// 任务并发上限:同时「运行中」的任务数上限。关闭=不限;开启后新建任务超限则排队,有空位自动启动。
ConcurrencyEnabled *bool `json:"task_concurrency_enabled"`
@@ -3923,10 +3925,6 @@ func (s *Server) putSettings(w http.ResponseWriter, r *http.Request) {
// synchronizes workers to its total. An empty list clears pools; when workers
// is also present it sets the legacy count for the newly restored legacy mode.
if req.WorkerPools != nil {
if len(*req.WorkerPools) == 0 && req.Workers != nil && *req.Workers <= 0 {
writeErr(w, 400, "workers 必须 >0")
return
}
legacyWorkers := []int(nil)
if req.Workers != nil {
legacyWorkers = []int{*req.Workers}
+15 -5
View File
@@ -15,7 +15,7 @@ const (
maxTaskMetadataRequestSize = 16 << 10
)
// updateTaskMetadata updates editable metadata and the live asset-sharing policy.
// updateTaskMetadata updates editable metadata and live task policies.
func (s *Server) updateTaskMetadata(w http.ResponseWriter, r *http.Request) {
taskID := r.PathValue("id")
if _, ok := s.m.Task(taskID); !ok {
@@ -28,6 +28,7 @@ func (s *Server) updateTaskMetadata(w http.ResponseWriter, r *http.Request) {
Remark *string `json:"remark"`
Pinned *bool `json:"pinned"`
ShareGlobalAssets *bool `json:"share_global_assets"`
WorkerCount *int `json:"worker_count"`
}
if err := decode(r, &request); err != nil {
var tooLarge *http.MaxBytesError
@@ -38,8 +39,12 @@ func (s *Server) updateTaskMetadata(w http.ResponseWriter, r *http.Request) {
}
return
}
if request.Name == nil && request.Pinned == nil && request.Remark == nil && request.ShareGlobalAssets == nil {
writeErr(w, http.StatusBadRequest, "至少需要提供 name、pinned、remark 或 share_global_assets")
if request.Name == nil && request.Pinned == nil && request.Remark == nil && request.ShareGlobalAssets == nil && request.WorkerCount == nil {
writeErr(w, http.StatusBadRequest, "至少需要提供 name、pinned、remark、share_global_assets 或 worker_count")
return
}
if request.WorkerCount != nil && (*request.WorkerCount < 0 || *request.WorkerCount > db.MaxTaskWorkerCount) {
writeErr(w, http.StatusBadRequest, fmt.Sprintf("worker_count 必须在 0 到 %d 之间", db.MaxTaskWorkerCount))
return
}
if request.Name != nil {
@@ -63,7 +68,7 @@ func (s *Server) updateTaskMetadata(w http.ResponseWriter, r *http.Request) {
}
request.Remark = &remark
}
task, err := s.m.UpdateTaskMetadata(taskID, db.TaskPatch{Name: request.Name, Pinned: request.Pinned, Remark: request.Remark, ShareGlobalAssets: request.ShareGlobalAssets})
task, err := s.m.UpdateTaskMetadata(taskID, db.TaskPatch{Name: request.Name, Pinned: request.Pinned, Remark: request.Remark, ShareGlobalAssets: request.ShareGlobalAssets, WorkerCount: request.WorkerCount})
if err != nil {
writeErr(w, http.StatusInternalServerError, err.Error())
return
@@ -72,5 +77,10 @@ func (s *Server) updateTaskMetadata(w http.ResponseWriter, r *http.Request) {
writeErr(w, http.StatusNotFound, "task not found")
return
}
writeJSON(w, http.StatusOK, taskDTO(task, s.resolvedTaskStatus(task)))
if request.WorkerCount != nil {
s.engine.ReconcileWorkers(task)
}
dto := taskDTO(task, s.resolvedTaskStatus(task))
dto.WorkerActualCount, dto.WorkerTargetCount = s.engine.WorkerCounts(task)
writeJSON(w, http.StatusOK, dto)
}
+108
View File
@@ -159,3 +159,111 @@ func TestConversationBatchDeleteReportsMissing(t *testing.T) {
t.Fatalf("missing conversation %s must include an error", fmt.Sprint(missing))
}
}
func TestTaskMetadataPatchWorkerCount(t *testing.T) {
m, err := NewManager(t.TempDir(), "")
if err != nil {
t.Skipf("postgres unavailable (%v) — skipping", err)
}
defer m.Close()
task, err := m.CreateTask("worker count patch", "goal", nil, 0, 0)
if err != nil {
t.Fatal(err)
}
taskID, _ := strconv.ParseInt(task.ID, 10, 64)
defer func() { _ = m.pg.DeleteTask(taskID) }()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
s := &Server{m: m, engine: NewEngine(m), ctx: ctx}
mux := http.NewServeMux()
mux.HandleFunc("PATCH /api/tasks/{id}", s.updateTaskMetadata)
mux.HandleFunc("GET /api/tasks/{id}", s.getTask)
mux.HandleFunc("GET /api/tasks", s.listTasks)
request := func(method, path, body string) *httptest.ResponseRecorder {
t.Helper()
req := httptest.NewRequest(method, path, strings.NewReader(body))
rec := httptest.NewRecorder()
mux.ServeHTTP(rec, req)
return rec
}
assertCounts := func(body []byte, want int) {
t.Helper()
var dto struct {
WorkerCount int `json:"worker_count"`
WorkerActualCount *int `json:"worker_actual_count"`
WorkerTargetCount *int `json:"worker_target_count"`
}
if err := json.Unmarshal(body, &dto); err != nil {
t.Fatal(err)
}
actual, target := s.engine.WorkerCounts(task)
if dto.WorkerCount != want || dto.WorkerActualCount == nil || dto.WorkerTargetCount == nil {
t.Fatalf("missing or incorrect worker counts: %s", body)
}
if *dto.WorkerActualCount != actual || *dto.WorkerTargetCount != target {
t.Fatalf("worker counts=%d/%d, engine=%d/%d", *dto.WorkerActualCount, *dto.WorkerTargetCount, actual, target)
}
if actual != 0 || (want > 0 && target != want) {
t.Fatalf("inactive task worker counts=%d/%d, want actual=0 and configured target=%d", actual, target, want)
}
}
for _, count := range []int{1, 50, 0} {
rec := request(http.MethodPatch, "/api/tasks/"+task.ID, fmt.Sprintf(`{"worker_count":%d}`, count))
if rec.Code != http.StatusOK {
t.Fatalf("patch worker_count=%d status=%d body=%s", count, rec.Code, rec.Body.String())
}
assertCounts(rec.Body.Bytes(), count)
if got := task.lifecycleSnapshot().WorkerCount; got != count {
t.Fatalf("in-memory worker count=%d, want %d", got, count)
}
persisted, err := m.pg.GetTask(taskID)
if err != nil || persisted == nil || persisted.WorkerCount != count {
t.Fatalf("persisted task=%+v err=%v, want worker_count=%d", persisted, err, count)
}
rec = request(http.MethodGet, "/api/tasks/"+task.ID, "")
if rec.Code != http.StatusOK {
t.Fatalf("get task status=%d body=%s", rec.Code, rec.Body.String())
}
assertCounts(rec.Body.Bytes(), count)
}
for _, body := range []string{
`{"worker_count":-1,"name":"must not commit"}`,
`{"worker_count":51,"name":"must not commit"}`,
`{"worker_count":1.5}`,
`{"worker_count":"2"}`,
`{"worker_count":null}`,
} {
rec := request(http.MethodPatch, "/api/tasks/"+task.ID, body)
if rec.Code != http.StatusBadRequest {
t.Fatalf("invalid patch %s status=%d body=%s", body, rec.Code, rec.Body.String())
}
}
state := task.lifecycleSnapshot()
persisted, err := m.pg.GetTask(taskID)
if err != nil || persisted == nil || persisted.WorkerCount != 0 || persisted.Name != state.Name || state.Name == "must not commit" || state.WorkerCount != 0 {
t.Fatalf("invalid patch changed state: memory=%+v persisted=%+v err=%v", state, persisted, err)
}
rec := request(http.MethodGet, "/api/tasks", "")
if rec.Code != http.StatusOK {
t.Fatalf("list tasks status=%d body=%s", rec.Code, rec.Body.String())
}
var list struct {
Tasks []json.RawMessage `json:"tasks"`
}
if err := json.Unmarshal(rec.Body.Bytes(), &list); err != nil {
t.Fatal(err)
}
for _, raw := range list.Tasks {
var dto struct {
ID string `json:"id"`
}
if err := json.Unmarshal(raw, &dto); err != nil {
t.Fatal(err)
}
if dto.ID == task.ID {
assertCounts(raw, 0)
return
}
}
t.Fatal("updated task missing from list")
}
+98 -1
View File
@@ -26,6 +26,7 @@ func TestExpandWorkerSlotsLegacy(t *testing.T) {
{"configured", 2, 2},
{"zero defaults", 0, 3},
{"negative defaults", -1, 3},
{"oversized caps", db.MaxTaskWorkerCount + 1, db.MaxTaskWorkerCount},
} {
t.Run(tc.name, func(t *testing.T) {
slots, err := ExpandWorkerSlots(nil, tc.count)
@@ -170,6 +171,13 @@ func TestWorkerPoolSettingsRoundTrip(t *testing.T) {
if clearResult.Code != http.StatusOK || m.Workers() != 4 || len(m.WorkerPools()) != 0 {
t.Fatalf("PUT legacy clear status=%d body=%s workers=%d pools=%+v", clearResult.Code, clearResult.Body.String(), m.Workers(), m.WorkerPools())
}
for _, body := range []string{`{"workers":51}`, `{"workers":51,"worker_pools":[]}`, `{"workers":0}`} {
invalid := httptest.NewRecorder()
s.putSettings(invalid, httptest.NewRequest(http.MethodPut, "/api/settings", strings.NewReader(body)))
if invalid.Code != http.StatusBadRequest || m.Workers() != 4 || len(m.WorkerPools()) != 0 {
t.Fatalf("invalid legacy workers %s status=%d body=%s workers=%d", body, invalid.Code, invalid.Body.String(), m.Workers())
}
}
res = httptest.NewRecorder()
s.getSettings(res, httptest.NewRequest(http.MethodGet, "/api/settings", nil))
var cleared map[string]any
@@ -280,7 +288,7 @@ func TestPinnedWorkerLoopDoesNotConsultTaskRouterAndReportsFailureOnce(t *testin
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
go func() {
e.workerLoop(ctx, task, WorkerSlot{Name: "work#4", ProfileID: 12})
e.workerLoop(ctx, task, WorkerSlot{Name: "work#4", ProfileID: 12}, &workerSlotRuntime{idleCtx: ctx})
close(done)
}()
time.Sleep(100 * time.Millisecond)
@@ -391,3 +399,92 @@ func TestDetachedIntentPinnedOwnerUsesPinnedResolver(t *testing.T) {
t.Fatalf("intent changed after unavailable resolver: node=%+v err=%v", node, err)
}
}
func TestWorkerSettingsDirectSettersNotifyAfterPersistence(t *testing.T) {
m, err := NewManager(t.TempDir(), "")
if err != nil {
t.Skipf("postgres unavailable: %v", err)
}
defer m.Close()
oldPools, poolsSet, _ := m.pg.GetSetting(settingWorkerPools)
oldWorkers, workersSet, _ := m.pg.GetSetting(settingWorkers)
defer func() {
for key, saved := range map[string]struct {
value string
set bool
}{
settingWorkerPools: {oldPools, poolsSet},
settingWorkers: {oldWorkers, workersSet},
} {
if saved.set {
_ = m.pg.SetSetting(key, saved.value)
} else {
_, _ = m.pg.Exec("DELETE FROM settings WHERE key=$1", key)
}
}
}()
profileID, err := m.pg.SaveProfile(&db.LLMProfile{Name: "worker-notify-test", Format: "openai", Model: "worker-notify-test", APIKey: "worker-notify-test-key"})
if err != nil {
t.Fatal(err)
}
defer m.pg.DeleteProfile(profileID)
var notifications int
wantWorkers, wantPools := 0, 0
callback := func() {
notifications++
// Re-enter Manager APIs: callbacks must run after the write and outside mu.
if got := m.Workers(); got != wantWorkers {
t.Errorf("callback observed workers=%d, want %d", got, wantWorkers)
}
if got := len(m.WorkerPools()); got != wantPools {
t.Errorf("callback observed %d pools, want %d", got, wantPools)
}
m.SetWorkerConfigChangedCallback(nil)
}
m.SetWorkerConfigChangedCallback(callback)
check := func(want int) {
t.Helper()
if notifications != want {
t.Fatalf("notifications=%d, want %d", notifications, want)
}
}
wantWorkers, wantPools = 50, len(m.WorkerPools())
if err := m.SetWorkers(50); err != nil {
t.Fatal(err)
}
check(1)
m.SetWorkerConfigChangedCallback(callback)
for _, count := range []int{-1, 0, 51} {
if err := m.SetWorkers(count); err == nil {
t.Fatalf("SetWorkers(%d) accepted", count)
}
if err := m.SetWorkerPools(nil, count); err == nil {
t.Fatalf("SetWorkerPools(nil, %d) accepted", count)
}
check(1)
}
if err := m.SetWorkerPools([]WorkerPool{{ProfileID: 9223372036854775807, Workers: 1}}); err == nil {
t.Fatal("missing profile accepted")
}
check(1)
if got := m.Workers(); got != 50 {
t.Fatalf("rejected worker settings changed persisted workers=%d, want 50", got)
}
wantWorkers, wantPools = 2, 1
if err := m.SetWorkerPools([]WorkerPool{{ProfileID: profileID, Workers: 2}}); err != nil {
t.Fatal(err)
}
check(2)
m.SetWorkerConfigChangedCallback(callback)
wantWorkers, wantPools = 3, 0
if err := m.SetWorkerPools(nil, 3); err != nil {
t.Fatal(err)
}
check(3)
if err := m.pg.SetSetting(settingWorkers, "1000"); err != nil {
t.Fatal(err)
}
if got := m.Workers(); got != db.MaxTaskWorkerCount {
t.Fatalf("old oversized workers=%d, want %d", got, db.MaxTaskWorkerCount)
}
}
@@ -24,6 +24,8 @@ import {
AlertDialogTrigger,
} from "@/components/ui/alert-dialog";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
import { Label } from "@/components/ui/label";
import {
Popover,
PopoverContent,
@@ -40,6 +42,7 @@ import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip
import { type UITranslate, useUITranslation } from "@/i18n/use-ui-translation";
import { api } from "@/lib/api";
import type { LLMProfile, Task } from "@/lib/types";
import { MAX_TASK_WORKER_COUNT, parseWorkerCountInput } from "@/lib/types";
import { AssetsTab } from "./_tabs/assets-tab";
import { BroadcastTab } from "./_tabs/broadcast-tab";
@@ -212,6 +215,87 @@ function TaskLLMControl({ task, profiles, onUpdated }: { task: Task; profiles: L
);
}
function TaskWorkerControl({ task, onUpdated }: { task: Task; onUpdated: (task: Task) => void }) {
const { ui } = useUITranslation();
const [open, setOpen] = React.useState(false);
const [draft, setDraft] = React.useState(String(task.worker_count ?? 0));
const [saving, setSaving] = React.useState(false);
const actual = task.worker_actual_count;
const target = task.worker_target_count;
const draining = actual !== undefined && target !== undefined && actual > target;
const save = async (event: React.FormEvent<HTMLFormElement>) => {
event.preventDefault();
const count = parseWorkerCountInput(draft);
if (count === null) {
toast.error(ui("taskUI.invalidWorkers", { max: MAX_TASK_WORKER_COUNT }));
return;
}
setSaving(true);
try {
const updated = await api.updateTask(task.id, { worker_count: count });
onUpdated(updated);
setOpen(false);
toast.success(ui("taskWorkers.saved"));
} catch (error) {
toast.error(ui("taskWorkers.saveFailed", { message: (error as Error).message }));
} finally {
setSaving(false);
}
};
return (
<div className="flex flex-wrap items-center gap-x-3 gap-y-1 text-xs text-muted-foreground">
<span className="tabular-nums">{ui("taskWorkers.counts", { actual: actual ?? "—", target: target ?? "—" })}</span>
<Popover
open={open}
onOpenChange={(next) => {
setOpen(next);
if (next) setDraft(String(task.worker_count ?? 0));
}}
>
<PopoverTrigger asChild>
<Button size="sm" variant="outline" className="h-7 text-xs">
{ui("taskWorkers.edit")}
</Button>
</PopoverTrigger>
<PopoverContent align="start" className="w-[min(25rem,calc(100vw-2rem))] gap-4 p-4">
<PopoverHeader>
<PopoverTitle>{ui("taskWorkers.edit")}</PopoverTitle>
<PopoverDescription>{ui("taskWorkers.help")}</PopoverDescription>
</PopoverHeader>
<form onSubmit={save} className="space-y-3">
<div className="space-y-2">
<Label htmlFor="task-worker-count">{ui("taskWorkers.label")}</Label>
<Input
id="task-worker-count"
type="number"
min={0}
max={MAX_TASK_WORKER_COUNT}
step={1}
value={draft}
disabled={saving}
onChange={(event) => setDraft(event.target.value)}
/>
</div>
<p className="text-xs text-muted-foreground">{ui("taskWorkers.transitionHelp")}</p>
<div className="flex justify-end gap-2">
<Button type="button" variant="outline" size="sm" onClick={() => setOpen(false)} disabled={saving}>
{ui("screen.3fd47edce45b36")}
</Button>
<Button type="submit" size="sm" disabled={saving}>
{saving && <Spinner data-icon="inline-start" />}
{ui(saving ? "screen.a1a1720d3eb58a" : "screen.a3030bf8f16dc6")}
</Button>
</div>
</form>
</PopoverContent>
</Popover>
{draining && <span className="text-amber-600">{ui("taskWorkers.draining")}</span>}
</div>
);
}
function TaskDetailInner() {
const { ui, liveUI } = useUITranslation();
@@ -437,6 +521,14 @@ function TaskDetailInner() {
</Button>
</div>
<p className="truncate text-xs text-muted-foreground">{task.goal}</p>
<TaskWorkerControl
key={task.id}
task={task}
onUpdated={(updated) => {
setTask(updated);
load();
}}
/>
{task.paused && task.pause_reason && (
<div className="flex flex-wrap items-center gap-2 text-xs text-amber-600">
<span>
+2 -2
View File
@@ -1731,8 +1731,8 @@ export default function SystemSettingsPage() {
</CardTitle>
<CardDescription>
{t("settings.workersHelpStart")}
<b>{t("settings.futureTasksOnly")}</b>
{t("settings.runningTasksUnchanged")}
<b>{t("settings.workersLiveUpdate")}</b>
{t("settings.workersDrainHelp")}
</CardDescription>
</CardHeader>
<CardContent className="flex flex-col gap-3">
+12 -6
View File
@@ -12,14 +12,14 @@ export const settingsMessages = {
"settings.workerLimit": ["Worker 总数不能超过 50", "The total number of Workers cannot exceed 50"],
"settings.duplicatePoolProfile": ["不能重复选择同一个 LLM 配置", "Each LLM profile can only be selected once"],
"settings.workerPoolsSaved": [
"已保存多模型 Worker 池(对之后启动的任务生效)",
"Model pools saved; applies to tasks started afterward",
"已保存多模型 Worker 池,跟随全局配置的任务已热更新",
"Model pools saved; tasks following the global configuration have been updated live",
],
"settings.saveFailed": ["保存失败:{{message}}", "Could not save: {{message}}"],
"settings.positiveConcurrency": ["并发数必须是大于 0 的整数", "Concurrency must be a positive whole number"],
"settings.workersSaved": [
"已保存并发工作 agent 数(对之后启动的任务生效)",
"Worker count saved; applies to tasks started afterward",
"已保存工作并发,跟随全局配置的任务已热更新",
"Worker count saved; tasks following the global configuration have been updated live",
],
"settings.saveFailedPrefix": ["保存失败:", "Could not save: "],
"settings.pythonSaved": ["已保存 Python 解释器配置", "Python interpreter settings saved"],
@@ -427,8 +427,14 @@ export const settingsMessages = {
"每个任务并发运行的工作 agent 数量(默认 3)。数值越大并发探测越多、消耗也越高。修改后",
"The number of Workers running concurrently per task (default: 3). More Workers mean more parallel probes and higher usage. Changes ",
],
"settings.futureTasksOnly": ["对之后启动的任务生效", "apply to tasks started afterward"],
"settings.runningTasksUnchanged": [",正在运行的任务不受影响。", "; running tasks are unaffected."],
"settings.workersLiveUpdate": [
"即时更新跟随全局配置(worker_count = 0)的任务",
"apply immediately to tasks following the global configuration (worker_count = 0)",
],
"settings.workersDrainHelp": [
";单任务覆盖值不受影响。增容启动新 Worker loops;缩容等待在跑意图完成后退出,不会强杀,实际数量可能暂时高于目标。",
"; per-task overrides are unchanged. Scaling up starts new Worker loops; scaling down lets running intents finish before loops exit, without force-cancelling them. Actual counts may temporarily exceed target.",
],
"settings.workerPools": ["多模型 Worker 池", "Model pools"],
"settings.addPool": ["添加模型池", "Add model pool"],
"settings.unavailableProfile": ["配置 #{{id}} 已删除/不可用", "Profile #{{id}} deleted or unavailable"],
+17
View File
@@ -1,4 +1,21 @@
export const taskMessages = {
"taskWorkers.edit": ["调整 Worker 并发", "Adjust Worker concurrency"],
"taskWorkers.label": ["Worker 数量(0–50)", "Worker count (0–50)"],
"taskWorkers.counts": ["Worker 实际 / 目标:{{actual}} / {{target}}", "Workers actual / target: {{actual}} / {{target}}"],
"taskWorkers.help": [
"0 表示跟随全局 Worker 配置;1–50 为本任务覆盖值,保存后热更新,无需重启任务。",
"Use 0 to follow the global Worker configuration, or 1–50 to override it for this task. Saving applies changes live without restarting the task.",
],
"taskWorkers.transitionHelp": [
"实际数量是存活的 Worker loops(含缩容退出中的 loops),不是正在执行的意图数。增容会启动新 loops;缩容等待正在执行的意图完成后退出,不会强杀意图,因此实际数量可能暂时高于目标。",
"Actual counts live Worker loops, including draining loops, not in-flight intents. Scaling up starts new loops. Scaling down lets running intents finish before loops exit, without force-cancelling them, so actual may temporarily exceed target.",
],
"taskWorkers.draining": [
"缩容过渡中:等待在跑意图完成,不会强杀。",
"Scaling down: waiting for running intents to finish; they will not be force-cancelled.",
],
"taskWorkers.saved": ["Worker 并发已更新", "Worker concurrency updated"],
"taskWorkers.saveFailed": ["Worker 并发更新失败:{{message}}", "Could not update Worker concurrency: {{message}}"],
"taskFiles.tab": ["文件", "Files"],
"taskFiles.tree": ["目录树", "Directory tree"],
"taskFiles.recent": ["最近文件", "Recent files"],
+1 -1
View File
@@ -483,7 +483,7 @@ export const api = {
taskCategories: () => get<{ categories: TaskCategory[] }>("/task-categories").then((r) => arr(r.categories)),
updateTask: (
id: string,
input: { name?: string; pinned?: boolean; remark?: string; share_global_assets?: boolean },
input: { name?: string; pinned?: boolean; remark?: string; share_global_assets?: boolean; worker_count?: number },
) => patch<Task>(`/tasks/${id}`, input),
renameTask: (id: string, name: string) => patch<Task>(`/tasks/${id}`, { name }),
pinTask: (id: string, pinned: boolean) => patch<Task>(`/tasks/${id}`, { pinned }),
+12
View File
@@ -58,6 +58,9 @@ export const tasks: Task[] = [
description: "Acme 官网与后台外部渗透(acme.com)",
goal: "拿到 acme.com 后台管理权限,确认可读取用户敏感数据。",
status: "running",
worker_count: 0,
worker_actual_count: 3,
worker_target_count: 3,
created_at: T("2026-07-24T09:12:00Z"),
created_unix: 1785489120,
active_run_seconds: 28800,
@@ -87,6 +90,9 @@ export const tasks: Task[] = [
description: "api.acme.com 越权与注入测试",
goal: "评估 api.acme.com 订单/用户接口的越权(IDOR)与注入风险。",
status: "running",
worker_count: 0,
worker_actual_count: 3,
worker_target_count: 3,
created_at: T("2026-07-25T14:05:00Z"),
created_unix: 1785592500,
active_run_seconds: 14400,
@@ -116,6 +122,9 @@ export const tasks: Task[] = [
description: "shop.acme.com 支付与订单链路",
goal: "评估支付与订单接口的越权、金额篡改与竞态风险。",
status: "paused",
worker_count: 0,
worker_actual_count: 3,
worker_target_count: 3,
created_at: T("2026-07-22T08:30:00Z"),
created_unix: 1785313800,
active_run_seconds: 21600,
@@ -144,6 +153,9 @@ export const tasks: Task[] = [
description: "对外暴露面侦察(VPN / 边界服务)",
goal: "识别 acme.com 对外暴露的可利用边界服务。",
status: "done",
worker_count: 0,
worker_actual_count: 0,
worker_target_count: 3,
created_at: T("2026-07-18T08:00:00Z"),
created_unix: 1784966400,
active_run_seconds: 86400,
+35 -3
View File
@@ -48,6 +48,7 @@ const mockInterceptHistory = structuredClone(D.interceptHistory);
const mockInterceptPending = structuredClone(D.interceptPending);
const mockInterceptDetails = structuredClone(D.interceptDetails);
const mockTasks = structuredClone(D.tasks);
const mockWorkerDrainUntil = new Map<string, number>();
const mockFindings = structuredClone(D.findings);
const mockLLMRecords = structuredClone(D.llmRecords);
const mockTaskTemplates = structuredClone(D.taskTemplates);
@@ -256,7 +257,33 @@ function mockTaskArchiveBlocker(taskID: string): string | undefined {
)?.id;
}
function reconcileMockWorkers(task: Task): void {
const target = task.worker_count || D.settings.workers || 3;
const previousTarget = task.worker_target_count;
task.worker_target_count = target;
if (task.status !== "running" && task.status !== "paused") {
task.worker_actual_count = 0;
mockWorkerDrainUntil.delete(task.id);
return;
}
const actual = task.worker_actual_count ?? target;
if (actual <= target) {
task.worker_actual_count = target;
mockWorkerDrainUntil.delete(task.id);
} else {
// Demo loops drain independently of in_flight; retain them across polls briefly.
if (!mockWorkerDrainUntil.has(task.id) || previousTarget !== target) {
mockWorkerDrainUntil.set(task.id, Date.now() + 10_000);
}
if (Date.now() >= (mockWorkerDrainUntil.get(task.id) ?? 0)) {
task.worker_actual_count = target;
mockWorkerDrainUntil.delete(task.id);
}
}
}
function publicMockTask(task: Task): Task {
reconcileMockWorkers(task);
const item = structuredClone(task);
item.numeric_id = mockTaskAssetIDs.get(task.id);
item.share_global_assets = task.share_global_assets ?? false;
@@ -1323,6 +1350,7 @@ function route(m: string, path: string, seg: string[], q: URLSearchParams, b: Re
company_ids: companyIDs,
share_global_assets: b.share_global_assets === true,
};
reconcileMockWorkers(created);
// Validate the attached preview schedule before mutating task state.
const numericTaskID = nextMockTaskAssetID;
if (b.schedule && typeof b.schedule === "object") {
@@ -1527,8 +1555,10 @@ function route(m: string, path: string, seg: string[], q: URLSearchParams, b: Re
const namePresent = has("name") && b.name !== null;
const remarkPresent = has("remark") && b.remark !== null;
const pinnedPresent = has("pinned") && b.pinned !== null;
if (!namePresent && !remarkPresent && !pinnedPresent && !sharingPresent)
throw new Error("至少需要提供 name、pinned、remark 或 share_global_assets");
const workerCountPresent = has("worker_count") && b.worker_count !== null;
const nextWorkerCount = workerCountPresent ? mockWorkerCount(b.worker_count) : task.worker_count;
if (!namePresent && !remarkPresent && !pinnedPresent && !sharingPresent && !workerCountPresent)
throw new Error("至少需要提供 name、pinned、remark、share_global_assets 或 worker_count");
let nextName = task.name;
let nextRemark = task.remark;
let nextPinned = task.pinned;
@@ -1550,8 +1580,9 @@ function route(m: string, path: string, seg: string[], q: URLSearchParams, b: Re
task.remark = nextRemark;
task.pinned = nextPinned;
task.pinned_at = nextPinnedAt;
task.worker_count = nextWorkerCount;
sortMockTasks();
return structuredClone(task);
return publicMockTask(task);
}
if (seg[0] === "tasks" && seg.length === 2 && m === "DELETE") {
const id = seg[1];
@@ -2463,6 +2494,7 @@ function route(m: string, path: string, seg: string[], q: URLSearchParams, b: Re
next.workers = next.worker_pools.reduce((sum, pool) => sum + Number(pool.workers), 0);
}
Object.assign(D.settings, next);
for (const task of mockTasks) reconcileMockWorkers(task);
return next;
}
if (path === "/settings/basic-auth" && m === "GET") return { enabled: false, username: "", password_set: false };
+3 -1
View File
@@ -26,6 +26,8 @@ export interface Task {
id: string;
remark?: string;
worker_count?: number; // 本任务 Worker 数;0/缺省=用全局默认
worker_actual_count?: number; // 当前存活的 Worker pool loops,含缩容退出中的 loops
worker_target_count?: number; // 当前期望的 Worker slots 数量
name?: string; // 可选任务名称;空/缺省=未命名,展示时回退到描述
category_id?: number;
category_name?: string;
@@ -1028,7 +1030,7 @@ export interface Settings {
show_egress_ip?: boolean; // 在顶栏显示直连 / 全局代理出口 IP;默认关闭
egress_probe_url?: string; // 自定义 IP 探测服务;空=默认公共回显服务
python_interpreter?: string; // 自定义脚本工具的 python 解释器路径(空=运行时检测)
workers?: number; // 并发工作 agent 数(默认3);对之后启动的任务生效
workers?: number; // 并发工作 agent 数(默认3);热更新跟随全局配置的任务
worker_pools: WorkerPool[];
// 任务并发上限:同时「运行中」的任务数上限。关闭=不限;开启后新建任务超限则排队,有空位自动启动。
task_concurrency_enabled?: boolean; // 默认 false