Merge pull request #60 from RuoJi6/fix/task-runtime-exclude-pause

fix: 运行时长排除暂停时间
This commit is contained in:
RuoJi6 authored and GitHub committed 2026-10-27 00:44:03 -07:00
commit cb0a69ef60
16 files changed
+587 -88

No files matched your search

+8 -1
View File
@@ -44,10 +44,13 @@ type TaskDTO struct {
Goal string `json:"goal"`
Status string `json:"status"` // created | running | paused | done | failed
CreatedAt string `json:"created_at"`
CreatedUnix int64 `json:"created_unix"` // created_at as unix seconds (for run-duration calc)
CreatedUnix int64 `json:"created_unix"`
CompletedAt string `json:"completed_at"` // RFC3339 finish time (done/failed); "" if unfinished
CompletedUnix int64 `json:"completed_unix"` // completed_at as unix seconds (0 if unfinished)
LastActivity int64 `json:"last_activity_unix"` // unix seconds of the last activity (0 if none)
RunSeconds int64 `json:"run_seconds"`
ActiveRunSeconds int64 `json:"active_run_seconds"`
RunSegmentStarted int64 `json:"run_segment_started_unix"`
Paused bool `json:"paused"`
Queued bool `json:"queued"`
Tokens TokenTotalDTO `json:"tokens"` // whole-task token consumption
@@ -107,6 +110,7 @@ func tokenTotalDTO(u db.TokenUsage) TokenTotalDTO {
func taskDTO(t *Task, status string) TaskDTO {
lifecycle := t.lifecycleSnapshot()
runSeconds := taskRunSeconds(lifecycle.ActiveRunSeconds, lifecycle.RunSegmentStartedAt, time.Now().Unix(), status, lifecycle.Paused, lifecycle.Queued)
llmState := t.llmStateSnapshot()
sourceIDs := make([]string, 0, len(lifecycle.SourceTaskIDs))
for _, id := range lifecycle.SourceTaskIDs {
@@ -127,6 +131,9 @@ func taskDTO(t *Task, status string) TaskDTO {
Status: status,
CreatedAt: rfc3339(time.Unix(t.CreatedAt, 0)),
CreatedUnix: t.CreatedAt,
RunSeconds: runSeconds,
ActiveRunSeconds: lifecycle.ActiveRunSeconds,
RunSegmentStarted: lifecycle.RunSegmentStartedAt,
CompletedAt: completedRFC(lifecycle.CompletedAt),
CompletedUnix: lifecycle.CompletedAt,
Paused: lifecycle.Paused,
+6
View File
@@ -937,6 +937,9 @@ func (e *Engine) plannerLoop(ctx context.Context, t *Task) {
return
}
defer e.decInflight(t.ID)
if err := e.m.StartTaskRunSegment(t.ID); err != nil {
log.Printf("[runtime] task %s start segment failed: %v", t.ID, err)
}
e.stampFirstRun(t) // 首次真正规划 → 盖 first_run_at + 算 deadline(仅带 timeout 的任务)
e.touch(t.ID)
emit := func(r db.Activity) { e.emitActivity(t, r) }
@@ -1086,6 +1089,9 @@ func (e *Engine) runIntent(ctx context.Context, t *Task, name string, worker *ag
}
return true
}
if err := e.m.StartTaskRunSegment(t.ID); err != nil {
log.Printf("[runtime] task %s start segment failed: %v", t.ID, err)
}
// per-work child context so the planner's kill_work can stop just this work.
workCtx, workCancel := context.WithCancelCause(ectx)
e.registerWork(intent.ID, workCancel)
+5
View File
@@ -252,6 +252,11 @@ func (s *Server) hasReadyQueuedTask(excludeID string) bool {
}
func (s *Server) startAdmittedTask(t *Task, mode string) {
if s.engine.ReadyFor(t) {
if err := s.m.StartTaskRunSegment(t.ID); err != nil {
log.Printf("[runtime] task %s start segment failed: %v", t.ID, err)
}
}
if mode == "bootstrap" {
if !s.engine.beginTaskOperation(t.ID) {
return
+174 -43
View File
@@ -63,6 +63,8 @@ type Task struct {
WorkerCount int `json:"worker_count"` // 本任务 Worker 数;0=用全局默认
FirstRunAt int64 `json:"first_run_at,omitempty"`
DeadlineAt int64 `json:"deadline_at,omitempty"`
ActiveRunSeconds int64 `json:"active_run_seconds"`
RunSegmentStartedAt int64 `json:"run_segment_started_at,omitempty"`
Store *pgdb.ExplorationStore `json:"-"`
Guard *guard.Guard `json:"-"`
notify chan struct{}
@@ -81,21 +83,23 @@ type Task struct {
// updateLifecycle instead of reading or writing the corresponding Task fields
// directly after the task has been published by Manager.
type taskLifecycleState struct {
Name string
Remark string
PinnedAt int64
Status string
Paused bool
Queued bool
QueuedAt int64
QueueMode string
CompletedAt int64
FirstRunAt int64
DeadlineAt int64
SourceTaskIDs []int64
CompanyIDs []int64
CategoryID *int64
CategoryName string
Name string
Remark string
PinnedAt int64
Status string
Paused bool
Queued bool
QueuedAt int64
QueueMode string
CompletedAt int64
FirstRunAt int64
DeadlineAt int64
ActiveRunSeconds int64
RunSegmentStartedAt int64
SourceTaskIDs []int64
CompanyIDs []int64
CategoryID *int64
CategoryName string
}
func (t *Task) lifecycleSnapshot() taskLifecycleState {
@@ -109,21 +113,23 @@ func (t *Task) lifecycleSnapshot() taskLifecycleState {
func (t *Task) lifecycleSnapshotLocked() taskLifecycleState {
return taskLifecycleState{
Name: t.Name,
Remark: t.Remark,
PinnedAt: t.PinnedAt,
Status: t.Status,
Paused: t.Paused,
Queued: t.Queued,
QueuedAt: t.QueuedAt,
QueueMode: t.QueueMode,
CompletedAt: t.CompletedAt,
FirstRunAt: t.FirstRunAt,
DeadlineAt: t.DeadlineAt,
SourceTaskIDs: append([]int64(nil), t.SourceTaskIDs...),
CompanyIDs: append([]int64(nil), t.CompanyIDs...),
CategoryID: cloneInt64Ptr(t.CategoryID),
CategoryName: t.CategoryName,
Name: t.Name,
Remark: t.Remark,
PinnedAt: t.PinnedAt,
Status: t.Status,
Paused: t.Paused,
Queued: t.Queued,
QueuedAt: t.QueuedAt,
QueueMode: t.QueueMode,
CompletedAt: t.CompletedAt,
FirstRunAt: t.FirstRunAt,
DeadlineAt: t.DeadlineAt,
ActiveRunSeconds: t.ActiveRunSeconds,
RunSegmentStartedAt: t.RunSegmentStartedAt,
SourceTaskIDs: append([]int64(nil), t.SourceTaskIDs...),
CompanyIDs: append([]int64(nil), t.CompanyIDs...),
CategoryID: cloneInt64Ptr(t.CategoryID),
CategoryName: t.CategoryName,
}
}
@@ -145,6 +151,8 @@ func (t *Task) updateLifecycle(update func(*taskLifecycleState)) {
t.CompletedAt = state.CompletedAt
t.FirstRunAt = state.FirstRunAt
t.DeadlineAt = state.DeadlineAt
t.ActiveRunSeconds = state.ActiveRunSeconds
t.RunSegmentStartedAt = state.RunSegmentStartedAt
t.SourceTaskIDs = append(t.SourceTaskIDs[:0], state.SourceTaskIDs...)
t.CompanyIDs = append(t.CompanyIDs[:0], state.CompanyIDs...)
t.CategoryID = cloneInt64Ptr(state.CategoryID)
@@ -525,6 +533,13 @@ func NewManager(dir, proxyAddr string) (*Manager, error) {
if err != nil {
return nil, err
}
// A live segment cannot be trusted across a process restart: its end time
// could be deployment/crash downtime. Clear it before tasks are restored so
// the next engine admission starts a fresh segment at actual execution time.
if _, err := pg.ClearOpenTaskRunSegments(); err != nil {
_ = pg.Close()
return nil, fmt.Errorf("clear stale task run segments: %w", err)
}
if err := pg.RecoverFindingRetests(); err != nil {
pg.Close()
return nil, fmt.Errorf("recover finding retests: %w", err)
@@ -1009,6 +1024,7 @@ func taskFromPG(pt *pgdb.Task, store *pgdb.ExplorationStore, ic *intercept.Inter
TimeoutSeconds: pt.TimeoutSeconds, PlanHeartbeatSeconds: pt.PlanHeartbeatSeconds,
CoverageEnabled: pt.CoverageEnabled, WorkerCount: pt.WorkerCount,
FirstRunAt: unixOrZero(pt.FirstRunAt), DeadlineAt: unixOrZero(pt.DeadlineAt),
ActiveRunSeconds: pt.ActiveRunSeconds, RunSegmentStartedAt: unixOrZero(pt.RunSegmentStartedAt),
Store: store, Guard: guard.NewWithInterceptor(ic), notify: make(chan struct{}, 1),
}
}
@@ -1289,13 +1305,16 @@ func (m *Manager) SetTaskPaused(id string, paused bool) error {
if err != nil {
return err
}
if err := m.pg.SetPaused(n, paused); err != nil {
active, started, err := m.pg.SetPausedRuntime(n, paused)
if err != nil {
return err
}
m.mu.Lock()
if t := m.tasks[id]; t != nil {
t.updateLifecycle(func(state *taskLifecycleState) {
state.Paused = paused
state.ActiveRunSeconds = active
state.RunSegmentStartedAt = unixOrZero(started)
})
}
m.mu.Unlock()
@@ -1331,7 +1350,8 @@ func (m *Manager) applyTaskAdmission(id, expectedStatus, status string, queued b
mode = ""
}
var queuedAt, completedAt, firstRunAt, deadlineAt sql.NullTime
var queuedAt, completedAt, firstRunAt, deadlineAt, segmentStartedAt sql.NullTime
var activeRunSeconds int64
var committedMode string
err = m.pg.QueryRow(`UPDATE tasks
SET status=$2,
@@ -1360,9 +1380,11 @@ func (m *Manager) applyTaskAdmission(id, expectedStatus, status string, queued b
WHEN $6='timeout' AND $2 NOT IN ('done','failed','timeout') THEN NULL
ELSE deadline_at
END
,active_run_seconds=active_run_seconds + CASE WHEN ($3 OR $2 IN ('done','failed','timeout')) AND run_segment_started_at IS NOT NULL THEN GREATEST(0, FLOOR(EXTRACT(EPOCH FROM (now() - run_segment_started_at)))::bigint) ELSE 0 END
,run_segment_started_at=CASE WHEN $3 OR $2 IN ('done','failed','timeout') THEN NULL ELSE run_segment_started_at END
WHERE id=$1 AND deleted_at IS NULL AND status=$6
RETURNING queued_at, queue_mode, completed_at, first_run_at, deadline_at`, n, status, queued, mode, preservePosition, expectedStatus, manualUntil).
Scan(&queuedAt, &committedMode, &completedAt, &firstRunAt, &deadlineAt)
RETURNING queued_at, queue_mode, completed_at, first_run_at, deadline_at, active_run_seconds, run_segment_started_at`, n, status, queued, mode, preservePosition, expectedStatus, manualUntil).
Scan(&queuedAt, &committedMode, &completedAt, &firstRunAt, &deadlineAt, &activeRunSeconds, &segmentStartedAt)
if errors.Is(err, sql.ErrNoRows) {
return fmt.Errorf("task %s lifecycle changed before admission (expected status %q)", id, expectedStatus)
}
@@ -1393,6 +1415,11 @@ func (m *Manager) applyTaskAdmission(id, expectedStatus, status string, queued b
if deadlineAt.Valid {
state.DeadlineAt = deadlineAt.Time.Unix()
}
state.ActiveRunSeconds = activeRunSeconds
state.RunSegmentStartedAt = 0
if segmentStartedAt.Valid {
state.RunSegmentStartedAt = segmentStartedAt.Time.Unix()
}
})
}
m.mu.Unlock()
@@ -1413,11 +1440,15 @@ func (m *Manager) ApplyTaskPauseOrigin(id string, scheduled bool) error {
return err
}
var mode string
var activeRunSeconds int64
var segmentStartedAt sql.NullTime
err = m.pg.QueryRow(`UPDATE tasks
SET paused=true, queued=false, queued_at=NULL, schedule_paused=$2, schedule_manual_until=NULL
SET paused=true, queued=false, queued_at=NULL, schedule_paused=$2, schedule_manual_until=NULL,
active_run_seconds=active_run_seconds + CASE WHEN run_segment_started_at IS NOT NULL THEN GREATEST(0, FLOOR(EXTRACT(EPOCH FROM (now() - run_segment_started_at)))::bigint) ELSE 0 END,
run_segment_started_at=NULL
WHERE id=$1 AND deleted_at IS NULL AND paused=false
AND status NOT IN ('done','failed','timeout')
RETURNING COALESCE(queue_mode,'')`, n, scheduled).Scan(&mode)
RETURNING COALESCE(queue_mode,''), active_run_seconds, run_segment_started_at`, n, scheduled).Scan(&mode, &activeRunSeconds, &segmentStartedAt)
if errors.Is(err, sql.ErrNoRows) {
return fmt.Errorf("task %s is unavailable for pause", id)
}
@@ -1431,6 +1462,11 @@ func (m *Manager) ApplyTaskPauseOrigin(id string, scheduled bool) error {
state.Queued = false
state.QueuedAt = 0
state.QueueMode = mode
state.ActiveRunSeconds = activeRunSeconds
state.RunSegmentStartedAt = 0
if segmentStartedAt.Valid {
state.RunSegmentStartedAt = segmentStartedAt.Time.Unix()
}
})
}
m.mu.Unlock()
@@ -1449,16 +1485,20 @@ func (m *Manager) EnqueueTask(id, mode string) error {
return fmt.Errorf("invalid queue mode %q", mode)
}
var queuedAt time.Time
var activeRunSeconds int64
var segmentStartedAt sql.NullTime
var committedMode string
err = m.pg.QueryRow(`UPDATE tasks
SET queued=true,
queued_at=CASE WHEN queued THEN COALESCE(queued_at, now()) ELSE now() END,
active_run_seconds=active_run_seconds + CASE WHEN run_segment_started_at IS NOT NULL THEN GREATEST(0, FLOOR(EXTRACT(EPOCH FROM (now() - run_segment_started_at)))::bigint) ELSE 0 END,
run_segment_started_at=NULL,
queue_mode=CASE
WHEN queue_mode='bootstrap' OR $2='bootstrap' THEN 'bootstrap'
ELSE 'resume'
END
WHERE id=$1 AND deleted_at IS NULL
RETURNING queued_at, queue_mode`, n, mode).Scan(&queuedAt, &committedMode)
RETURNING queued_at, queue_mode, active_run_seconds, run_segment_started_at`, n, mode).Scan(&queuedAt, &committedMode, &activeRunSeconds, &segmentStartedAt)
if errors.Is(err, sql.ErrNoRows) {
return fmt.Errorf("task %s is unavailable for enqueue", id)
}
@@ -1471,6 +1511,80 @@ func (m *Manager) EnqueueTask(id, mode string) error {
state.QueuedAt = queuedAt.UnixNano()
state.Queued = true
state.QueueMode = committedMode
state.ActiveRunSeconds = activeRunSeconds
state.RunSegmentStartedAt = 0
if segmentStartedAt.Valid {
state.RunSegmentStartedAt = segmentStartedAt.Time.Unix()
}
})
}
m.mu.Unlock()
return nil
}
// StartTaskRunSegment records the start of actual execution, once the engine
// has admitted a task and is about to run planner/worker work.
func (m *Manager) StartTaskRunSegment(id string) error {
m.taskStateMu.Lock()
defer m.taskStateMu.Unlock()
n, err := strconv.ParseInt(id, 10, 64)
if err != nil {
return err
}
m.mu.RLock()
if t := m.tasks[id]; t != nil && t.lifecycleSnapshot().RunSegmentStartedAt > 0 {
m.mu.RUnlock()
return nil
}
m.mu.RUnlock()
active, started, err := m.pg.StartTaskRunSegment(n)
if err != nil {
return err
}
if started != nil {
m.mirrorTaskRuntime(id, active, started)
}
return nil
}
// FinalizeTaskRunSegment is idempotent and is used by pause/terminal paths that
// may race with a late engine cancellation.
func (m *Manager) FinalizeTaskRunSegment(id string) error {
m.taskStateMu.Lock()
defer m.taskStateMu.Unlock()
n, err := strconv.ParseInt(id, 10, 64)
if err != nil {
return err
}
active, started, err := m.pg.FinalizeTaskRunSegment(n)
if err != nil {
return err
}
m.mirrorTaskRuntime(id, active, started)
return nil
}
func (m *Manager) mirrorTaskRuntime(id string, active int64, started *time.Time) {
m.mu.Lock()
defer m.mu.Unlock()
if t := m.tasks[id]; t != nil {
t.updateLifecycle(func(state *taskLifecycleState) {
state.ActiveRunSeconds = active
state.RunSegmentStartedAt = unixOrZero(started)
})
}
}
func (m *Manager) syncTaskRuntime(id string, n int64) error {
pt, err := m.pg.GetTask(n)
if err != nil || pt == nil {
return err
}
m.mu.Lock()
if t := m.tasks[id]; t != nil {
t.updateLifecycle(func(state *taskLifecycleState) {
state.ActiveRunSeconds = pt.ActiveRunSeconds
state.RunSegmentStartedAt = unixOrZero(pt.RunSegmentStartedAt)
})
}
m.mu.Unlock()
@@ -1559,20 +1673,30 @@ func (m *Manager) SetTaskStatusGuarded(id, status string) (won bool, err error)
if err != nil {
return false, err
}
won, err = m.pg.SetTerminalStatusGuarded(n, status)
active, started, won, err := m.pg.SetTerminalStatusGuardedRuntime(n, status)
if err != nil || !won {
return won, err
}
// Mirror the committed terminal transition before any optional hydration.
// This keeps the in-memory clock stopped even if GetTask later fails.
m.mu.Lock()
if t := m.tasks[id]; t != nil {
t.updateLifecycle(func(state *taskLifecycleState) {
state.Status = status
if state.CompletedAt == 0 {
state.CompletedAt = time.Now().Unix()
state.ActiveRunSeconds = active
state.RunSegmentStartedAt = unixOrZero(started)
if pgdb.IsTerminal(status) {
state.RunSegmentStartedAt = 0
if state.CompletedAt == 0 {
state.CompletedAt = time.Now().Unix()
}
}
})
}
m.mu.Unlock()
if err := m.syncTaskRuntime(id, n); err != nil {
return true, err
}
return true, nil
}
@@ -1585,16 +1709,20 @@ func (m *Manager) SetTaskStatus(id, status string) error {
if err != nil {
return err
}
if err := m.pg.SetStatus(n, status); err != nil {
active, started, err := m.pg.SetStatusRuntime(n, status)
if err != nil {
return err
}
// Mirror the successful UPDATE immediately. A later GetTask hydration is
// best-effort and must not reopen a terminal clock in memory.
m.mu.Lock()
if t := m.tasks[id]; t != nil {
t.updateLifecycle(func(state *taskLifecycleState) {
state.Status = status
// Mirror the DB's completed_at stamp on the live handle so the DTO shows
// the finish time without a reload (terminal -> stamp once; else clear).
state.ActiveRunSeconds = active
state.RunSegmentStartedAt = unixOrZero(started)
if pgdb.IsTerminal(status) {
state.RunSegmentStartedAt = 0
if state.CompletedAt == 0 {
state.CompletedAt = time.Now().Unix()
}
@@ -1604,6 +1732,9 @@ func (m *Manager) SetTaskStatus(id, status string) error {
})
}
m.mu.Unlock()
if err := m.syncTaskRuntime(id, n); err != nil {
return err
}
return nil
}
+3 -12
View File
@@ -175,10 +175,9 @@ func (s *Server) delegateToTask(ctx context.Context, in json.RawMessage, pick fu
func (s *Server) toolListTasks() actool.CoreTool {
return roTool("list_tasks",
"列出所有任务(id/描述/目标/状态/运行时长/父任务/LLM 配置),编排 agent 用它掌握全局、看哪些任务卡太久、各自用哪个 LLM。运行时长:运行中=创建→现在,终态=创建→最后活动(秒)。llm_profile:任务 planner/worker 用的配置名,(激活配置)=跟随全局激活。",
"列出所有任务(id/描述/目标/状态/运行时长/父任务/LLM 配置),编排 agent 用它掌握全局、看哪些任务卡太久、各自用哪个 LLM。运行时长只累计实际执行时间,暂停、排队和终态后的时间不计入。llm_profile:任务 planner/worker 用的配置名,(激活配置)=跟随全局激活。",
objSchema(map[string]any{}),
func(context.Context, json.RawMessage) (actool.Result, error) {
lastAct, _ := s.m.PG().LastActivityAll()
// id -> name to resolve each task's pinned LLM profile.
profName := map[int64]string{}
if profs, err := s.m.pg.ListProfiles(); err == nil {
@@ -189,16 +188,8 @@ func (s *Server) toolListTasks() actool.CoreTool {
out := make([]map[string]any, 0)
for _, t := range s.m.List() {
status := s.deriveTaskStatus(t)
end := lastAct[t.ExpID]
if live := s.engine.LastActivity(t.ID); live > end {
end = live
}
dur := int64(0)
if status == "running" {
dur = time.Now().Unix() - t.CreatedAt
} else if end > t.CreatedAt {
dur = end - t.CreatedAt
}
lifecycle := t.lifecycleSnapshot()
dur := taskRunSeconds(lifecycle.ActiveRunSeconds, lifecycle.RunSegmentStartedAt, time.Now().Unix(), status, lifecycle.Paused, lifecycle.Queued)
row := map[string]any{"id": t.ID, "description": t.Description, "goal": t.Goal, "status": status, "run_seconds": dur}
if t.ParentRef != "" {
row["parent_ref"] = t.ParentRef
+1 -1
View File
@@ -1138,7 +1138,7 @@ func (s *Server) listTasks(w http.ResponseWriter, r *http.Request) {
metric := metrics[t.ExpID]
dto.Tokens = tokenTotalDTO(metric.Tokens)
// prefer the live in-memory heartbeat (fresher) and fall back to the
// persisted max activity time (survives restarts) for run-duration display.
// persisted max activity time (survives restarts) for activity display.
dto.LastActivity = metric.LastActivity
if live := s.engine.LastActivity(t.ID); live > dto.LastActivity {
dto.LastActivity = live
+25
View File
@@ -0,0 +1,25 @@
package server
// taskRunSeconds computes displayed active runtime from persisted segment data.
// Segment timestamps are unix seconds; a zero start means no active segment.
func taskRunSeconds(active, segmentStarted, now int64, status string, paused, queued bool) int64 {
if active < 0 {
active = 0
}
if status == "running" && !paused && !queued && segmentStarted > 0 && now > segmentStarted {
return active + now - segmentStarted
}
return active
}
// finalizeTaskRunSeconds mirrors the database transition for deterministic unit
// tests and callers that need to reason about a segment without a live database.
func finalizeTaskRunSeconds(active, segmentStarted, now int64) (int64, int64) {
if active < 0 {
active = 0
}
if segmentStarted > 0 && now > segmentStarted {
active += now - segmentStarted
}
return active, 0
}
+35
View File
@@ -0,0 +1,35 @@
package server
import "testing"
func TestTaskRunSecondsRunningSegmentAccumulates(t *testing.T) {
if got := taskRunSeconds(12, 100, 145, "running", false, false); got != 57 {
t.Fatalf("running duration = %d, want 57", got)
}
}
func TestTaskRunSecondsPausedDoesNotGrow(t *testing.T) {
if got := taskRunSeconds(57, 100, 1000, "paused", true, false); got != 57 {
t.Fatalf("paused duration = %d, want 57", got)
}
if got := taskRunSeconds(57, 100, 1000, "running", false, true); got != 57 {
t.Fatalf("queued duration = %d, want 57", got)
}
}
func TestTaskRunSecondsResumeContinuesFromPriorTotal(t *testing.T) {
active, segment := finalizeTaskRunSeconds(12, 100, 145)
if active != 57 || segment != 0 {
t.Fatalf("finalized segment = (%d,%d), want (57,0)", active, segment)
}
if got := taskRunSeconds(active, 200, 230, "running", false, false); got != 87 {
t.Fatalf("resumed duration = %d, want 87", got)
}
}
func TestTaskRunSecondsTerminalFinalizesSegment(t *testing.T) {
active, segment := finalizeTaskRunSeconds(20, 500, 560)
if active != 80 || segment != 0 {
t.Fatalf("terminal finalization = (%d,%d), want (80,0)", active, segment)
}
}