feat(tasks): add inherited context and resilient LLM execution

Add read-only task context inheritance, ordered task-level LLM failover, live Worker controls, cascading task cleanup, FIFO task concurrency, cancellation diagnostics, UI updates, tests, build script, and changelog.
This commit is contained in:
pphua committed 2026-08-20 18:34:24 +08:00
1 parent 7090b034f8
commit 9793b522d9
82 files changed
+11094 -1712

No files matched your search

+196 -18
View File
@@ -10,6 +10,7 @@ import (
"database/sql"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io/fs"
"log"
@@ -24,9 +25,9 @@ import (
"sync/atomic"
"time"
"github.com/Autumn-27/artex/db"
"github.com/Autumn-27/norma/permission"
actool "github.com/Autumn-27/norma/tool"
"github.com/Autumn-27/artex/db"
mproxy "github.com/lqqyt2423/go-mitmproxy/proxy"
_ "modernc.org/sqlite"
)
@@ -471,28 +472,205 @@ func (t *Traffic) DeleteHost(host string) (int64, error) {
// so picking "api.example.com" never sweeps "api.example.com.cn". Returns rows
// deleted. Duplicate hosts are harmless (idempotent deletes, single tree pass).
func (t *Traffic) DeleteHostsExact(hosts []string) (int64, error) {
t.wmu.Lock()
defer t.wmu.Unlock()
var deleted int64
removed := make(map[string]bool)
for _, h := range hosts {
res, err := t.db.Exec(`DELETE FROM exchanges WHERE host=?`, h)
if err != nil {
return deleted, err
}
n, _ := res.RowsAffected()
deleted += n
if !removed[h] {
os.RemoveAll(filepath.Join(t.dir, sanitize(h)))
removed[h] = true
}
stage, err := t.StageDeleteHostsExact(hosts)
if err != nil {
return 0, err
}
if deleted > 0 {
_ = t.gcBlobs()
deleted := stage.Deleted()
if err := stage.Commit(); err != nil {
return deleted, err
}
return deleted, nil
}
type stagedTrafficPath struct {
source string
staged string
}
// HostDeleteStage keeps the SQLite delete transaction open and moves matching
// host trees to a same-filesystem staging directory. The Traffic write lock is
// held until Commit or Rollback, so no recorder can recreate a staged tree or a
// blob reference while a task deletion is being coordinated with PostgreSQL.
type HostDeleteStage struct {
traffic *Traffic
tx *sql.Tx
stageDir string
moves []stagedTrafficPath
deleted int64
done bool
}
// Deleted reports the number of exchange rows selected by this stage.
func (s *HostDeleteStage) Deleted() int64 {
if s == nil {
return 0
}
return s.deleted
}
// StageDeleteHostsExact prepares a reversible exact-host deletion. Callers must
// finish every successful stage with Commit or Rollback.
func (t *Traffic) StageDeleteHostsExact(hosts []string) (*HostDeleteStage, error) {
t.wmu.Lock()
stage := &HostDeleteStage{traffic: t}
fail := func(cause error) (*HostDeleteStage, error) {
if rollbackErr := stage.rollbackLocked(); rollbackErr != nil {
return nil, errors.Join(cause, fmt.Errorf("restore staged traffic: %w", rollbackErr))
}
return nil, cause
}
unique := make([]string, 0, len(hosts))
seen := make(map[string]struct{}, len(hosts))
for _, host := range hosts {
if _, ok := seen[host]; ok {
continue
}
seen[host] = struct{}{}
unique = append(unique, host)
}
tx, err := t.db.Begin()
if err != nil {
t.wmu.Unlock()
stage.done = true
return nil, err
}
stage.tx = tx
for _, h := range unique {
res, err := tx.Exec(`DELETE FROM exchanges WHERE host=?`, h)
if err != nil {
return fail(err)
}
n, _ := res.RowsAffected()
stage.deleted += n
}
stagedSources := make(map[string]struct{}, len(unique))
for _, host := range unique {
source := filepath.Join(t.dir, sanitize(host))
if _, duplicatePath := stagedSources[source]; duplicatePath {
continue
}
stagedSources[source] = struct{}{}
if _, err := os.Lstat(source); err != nil {
if os.IsNotExist(err) {
continue
}
return fail(fmt.Errorf("inspect traffic tree for %q: %w", host, err))
}
if stage.stageDir == "" {
parent := filepath.Join(t.dir, "_delete_staging")
if err := os.MkdirAll(parent, 0o700); err != nil {
return fail(fmt.Errorf("create traffic staging directory: %w", err))
}
stage.stageDir, err = os.MkdirTemp(parent, "hosts-")
if err != nil {
return fail(fmt.Errorf("create traffic stage: %w", err))
}
}
staged := filepath.Join(stage.stageDir, fmt.Sprintf("%d-%s", len(stage.moves), filepath.Base(source)))
if err := os.Rename(source, staged); err != nil {
return fail(fmt.Errorf("stage traffic tree for %q: %w", host, err))
}
stage.moves = append(stage.moves, stagedTrafficPath{source: source, staged: staged})
}
return stage, nil
}
// Rollback restores both the SQLite rows and every staged host tree.
func (s *HostDeleteStage) Rollback() error {
if s == nil || s.done {
return nil
}
return s.rollbackLocked()
}
func (s *HostDeleteStage) rollbackLocked() error {
if s == nil || s.done {
return nil
}
var errs []error
if s.tx != nil {
if err := s.tx.Rollback(); err != nil && !errors.Is(err, sql.ErrTxDone) {
errs = append(errs, fmt.Errorf("rollback traffic index: %w", err))
}
}
for i := len(s.moves) - 1; i >= 0; i-- {
move := s.moves[i]
if _, err := os.Lstat(move.source); err == nil {
errs = append(errs, fmt.Errorf("restore destination already exists: %s", move.source))
continue
} else if !os.IsNotExist(err) {
errs = append(errs, fmt.Errorf("inspect restore destination %s: %w", move.source, err))
continue
}
if err := os.Rename(move.staged, move.source); err != nil {
errs = append(errs, fmt.Errorf("restore %s: %w", move.source, err))
}
}
if len(errs) == 0 && s.stageDir != "" {
if err := os.RemoveAll(s.stageDir); err != nil {
errs = append(errs, fmt.Errorf("remove traffic stage: %w", err))
}
}
s.done = true
s.traffic.wmu.Unlock()
return errors.Join(errs...)
}
// Commit makes the staged deletion permanent. SQLite is committed only after
// the caller has committed its PostgreSQL task deletion.
func (s *HostDeleteStage) Commit() error {
if s == nil || s.done {
return nil
}
if err := s.tx.Commit(); err != nil {
// A failed SQLite commit normally rolls the transaction back. Restore the
// trees so the traffic store remains internally consistent and recoverable.
restoreErr := s.restoreTreesLocked()
s.done = true
s.traffic.wmu.Unlock()
return errors.Join(fmt.Errorf("commit traffic index deletion: %w", err), restoreErr)
}
var errs []error
if s.stageDir != "" {
if err := os.RemoveAll(s.stageDir); err != nil {
errs = append(errs, fmt.Errorf("remove staged traffic trees: %w", err))
}
}
if err := s.traffic.gcBlobs(); err != nil {
errs = append(errs, fmt.Errorf("garbage collect traffic blobs: %w", err))
}
s.done = true
s.traffic.wmu.Unlock()
return errors.Join(errs...)
}
func (s *HostDeleteStage) restoreTreesLocked() error {
var errs []error
for i := len(s.moves) - 1; i >= 0; i-- {
move := s.moves[i]
if _, err := os.Lstat(move.source); err == nil {
errs = append(errs, fmt.Errorf("restore destination already exists: %s", move.source))
continue
} else if !os.IsNotExist(err) {
errs = append(errs, fmt.Errorf("inspect restore destination %s: %w", move.source, err))
continue
}
if err := os.Rename(move.staged, move.source); err != nil {
errs = append(errs, fmt.Errorf("restore %s: %w", move.source, err))
}
}
if len(errs) == 0 && s.stageDir != "" {
if err := os.RemoveAll(s.stageDir); err != nil {
errs = append(errs, fmt.Errorf("remove traffic stage: %w", err))
}
}
return errors.Join(errs...)
}
// gcBlobs removes content-addressed blobs in _blobs that no remaining exchange
// tree references. References appear as "@blob sha256:<hex>" lines inside the
// tree's request.http/response.http files (see bodyOrBlob). Best-effort: walk
+146
View File
@@ -162,6 +162,152 @@ VALUES(?,?,?,?,?,?,?,?,?,?,?)`,
}
}
func TestDeleteHostsExactReportsTreeRemovalFailure(t *testing.T) {
dir := t.TempDir()
tr, err := Open(dir, "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer tr.Close()
const host = "api.example.com"
if _, err := tr.DB().Exec(`INSERT INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
VALUES(?,?,?,?,?,?,?,?,?,?,?)`,
"1-0001", 1, host, "GET", "/", "http://"+host+"/", 200, "text/html", 0, 0, host+"/GET/1-0001"); err != nil {
t.Fatal(err)
}
notDir := filepath.Join(dir, "not-a-directory")
if err := os.WriteFile(notDir, []byte("x"), 0o644); err != nil {
t.Fatal(err)
}
tr.dir = notDir
n, err := tr.DeleteHostsExact([]string{host})
if err == nil {
t.Fatal("DeleteHostsExact returned nil after traffic tree removal failed")
}
if n != 0 {
t.Fatalf("deleted=%d, want 0 after atomic rollback", n)
}
var count int
if err := tr.DB().QueryRow(`SELECT COUNT(*) FROM exchanges WHERE host=?`, host).Scan(&count); err != nil || count != 1 {
t.Fatalf("rolled-back index count=%d err=%v, want 1", count, err)
}
}
func TestDeleteHostsExactRollsBackWholeIndexBatch(t *testing.T) {
dir := t.TempDir()
tr, err := Open(dir, "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer tr.Close()
for i, host := range []string{"a.example.com", "b.example.com"} {
id := fmt.Sprintf("1-%04d", i+1)
if err := os.MkdirAll(filepath.Join(dir, host, "GET", id), 0o755); err != nil {
t.Fatal(err)
}
if _, err := tr.DB().Exec(`INSERT INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
VALUES(?,?,?,?,?,?,?,?,?,?,?)`, id, 1, host, "GET", "/", "http://"+host+"/", 200, "text/html", 0, 0, host+"/GET/"+id); err != nil {
t.Fatal(err)
}
}
if _, err := tr.DB().Exec(`CREATE TRIGGER fail_second_host BEFORE DELETE ON exchanges
WHEN OLD.host='b.example.com' BEGIN SELECT RAISE(ABORT, 'forced delete failure'); END`); err != nil {
t.Fatal(err)
}
if n, err := tr.DeleteHostsExact([]string{"a.example.com", "b.example.com"}); err == nil || n != 0 {
t.Fatalf("DeleteHostsExact failure=(%d,%v), want (0,error)", n, err)
}
var count int
if err := tr.DB().QueryRow(`SELECT COUNT(*) FROM exchanges`).Scan(&count); err != nil || count != 2 {
t.Fatalf("rolled-back index count=%d err=%v, want 2", count, err)
}
for _, host := range []string{"a.example.com", "b.example.com"} {
if _, err := os.Stat(filepath.Join(dir, host)); err != nil {
t.Fatalf("tree %s changed despite index rollback: %v", host, err)
}
}
}
func TestStageDeleteHostsExactRollbackRestoresIndexAndTree(t *testing.T) {
dir := t.TempDir()
tr, err := Open(dir, "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer tr.Close()
const host = "rollback.example.com"
const id = "1-0001"
tree := filepath.Join(dir, host, "GET", id)
if err := os.MkdirAll(tree, 0o755); err != nil {
t.Fatal(err)
}
marker := filepath.Join(tree, "request.http")
if err := os.WriteFile(marker, []byte("original"), 0o644); err != nil {
t.Fatal(err)
}
if _, err := tr.DB().Exec(`INSERT INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
VALUES(?,?,?,?,?,?,?,?,?,?,?)`, id, 1, host, "GET", "/", "http://"+host+"/", 200, "text/html", 0, 0, host+"/GET/"+id); err != nil {
t.Fatal(err)
}
stage, err := tr.StageDeleteHostsExact([]string{host})
if err != nil {
t.Fatal(err)
}
if stage.Deleted() != 1 {
t.Fatalf("staged deleted=%d, want 1", stage.Deleted())
}
if _, err := os.Stat(filepath.Join(dir, host)); !os.IsNotExist(err) {
t.Fatalf("host tree was not staged: %v", err)
}
if err := stage.Rollback(); err != nil {
t.Fatal(err)
}
if got, err := os.ReadFile(marker); err != nil || string(got) != "original" {
t.Fatalf("restored tree content=%q err=%v", got, err)
}
var count int
if err := tr.DB().QueryRow(`SELECT COUNT(*) FROM exchanges WHERE host=?`, host).Scan(&count); err != nil || count != 1 {
t.Fatalf("restored index count=%d err=%v, want 1", count, err)
}
}
func TestStageDeleteHostsExactRollbackReportsRestoreFailure(t *testing.T) {
dir := t.TempDir()
tr, err := Open(dir, "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer tr.Close()
const host = "conflict.example.com"
hostDir := filepath.Join(dir, host)
if err := os.MkdirAll(hostDir, 0o755); err != nil {
t.Fatal(err)
}
stage, err := tr.StageDeleteHostsExact([]string{host})
if err != nil {
t.Fatal(err)
}
// Simulate an out-of-band conflicting destination. Rollback must surface the
// failed rename instead of claiming the external data was restored.
if err := os.WriteFile(hostDir, []byte("conflict"), 0o644); err != nil {
t.Fatal(err)
}
if err := stage.Rollback(); err == nil || !strings.Contains(err.Error(), "restore") {
t.Fatalf("rollback err=%v, want restore failure", err)
}
if _, err := os.Stat(stage.stageDir); err != nil {
t.Fatalf("staging was removed after failed restore: %v", err)
}
}
// TestDeleteHostGCBlobs verifies blob garbage collection: after a host's trees
// are removed, blobs referenced by no remaining exchange are deleted, while
// blobs still referenced (including shared ones) survive.