From 4d2c327df723dae92fa96898aac3313f5d87b364 Mon Sep 17 00:00:00 2001 From: ji ruo <79234113+RuoJi6@users.noreply.github.com> Date: Fri, 20 Nov 2026 17:00:24 +0800 Subject: [PATCH] =?UTF-8?q?fix(backup):=20=E5=8A=A0=E5=9B=BA=E4=B8=80?= =?UTF-8?q?=E9=94=AE=E5=A4=87=E4=BB=BD=E4=B8=8E=E6=81=A2=E5=A4=8D=E5=AE=89?= =?UTF-8?q?=E5=85=A8=E6=80=A7=E5=8F=8A=E5=A4=A7=E5=8C=85=E6=B5=81=E5=BC=8F?= =?UTF-8?q?=E4=BC=A0=E8=BE=93=20(#160)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 导出前拒绝存在未暂停的运行中/排队中任务(409 backup_active_tasks + X-Artex-Error-Code),恢复事务内将未完成非暂停任务强制置为手动暂停,防止迁移后自动继续打目标,并同步更新确认文案与 README - 启动恢复时若文件替换或暂存清理失败,ApplyPendingRestore 返回错误且 main 非零退出,交由守护脚本重试,避免带着新库与半套文件启动业务引擎 - 数据库表导出改为 8 MiB 分片流式写入 tar+zstd 并刷新 HTTP 响应,消除大表落盘临时文件与反代长静默超时;README 补齐 /api/system/backup/ 的 Nginx client_max_body_size 0 与 proxy_request_buffering off 配置 - 暂存新恢复包时先将旧 pending 移走、rename 失败则回滚,避免丢弃已有待恢复包 - 恢复接口返回唯一 restore_id 并写入暂存标记,前端改为精确匹配 restore_id 判断重启恢复完成,消除时钟偏差与旧结果误判 - 补充 Windows->Linux 归档路径改写、活跃任务拦截与恢复自动暂停、多片流式 COPY、文件替换失败快速退出续跑及暂存原子替换测试 --- README.md | 24 +- cmd/artex/main.go | 7 +- db/backup.go | 32 ++ server/system_backup.go | 320 +++++++++++++++--- server/system_backup_test.go | 236 ++++++++++++- .../settings/_components/backup-card.tsx | 12 +- web/src/i18n/error-messages.ts | 4 + web/src/i18n/settings-messages.ts | 12 +- web/src/lib/api.ts | 12 +- 9 files changed, 578 insertions(+), 81 deletions(-) diff --git a/README.md b/README.md index 8d3ec18..993bb9f 100644 --- a/README.md +++ b/README.md @@ -193,13 +193,13 @@ ARTEX_TARGETS=linux/amd64,windows/amd64 ./build.sh --release 在 **系统配置 → 系统与账号** 的 **备份与迁移** 卡片中: -- **下载备份**:生成单个 `artex-backup-<时间>.tar.zst`,浏览器直接落盘。内容包括 PostgreSQL 全部业务表(同一快照事务导出,表间一致)、数据目录(任务工作区、会话、证据、任务归档包、MITM CA 证书、流量及其 SQLite 索引快照)和 skills 目录。不含系统运行日志(`server_logs`)和 `jwt.key`。 -- **从备份恢复**:在新机器上先正常安装并启动 ARTEX,登录后上传备份文件。服务端先完整解包并按当前数据库结构校验,通过后重启;重启时、在任何组件打开数据之前替换数据:数据库在一个事务内整体替换(失败则原数据不变,恢复包移到 `data/.restore-failed` 供排查),数据目录逐项替换,中途中断会在下次启动时继续。卡片会显示上次恢复的结果。 -- 恢复会**覆盖当前实例的全部数据**并中断正在运行的任务。登录密码、Basic Auth、LLM 与其他配置都随数据库迁移,恢复后用原实例的密码登录。 +- **下载备份**:生成单个 `artex-backup-<时间>.tar.zst`,浏览器直接落盘。导出前需先暂停或等待所有运行中/排队中的任务完成,避免导出期间文件写入导致不一致。内容包括 PostgreSQL 全部业务表(同一快照事务内分片流式导出,表间一致)、数据目录(任务工作区、会话、证据、任务归档包、MITM CA 证书、流量及其 SQLite 索引快照)和 skills 目录。不含系统运行日志(`server_logs`)和 `jwt.key`。 +- **从备份恢复**:在新机器上先正常安装并启动 ARTEX,登录后上传备份文件。服务端先完整解包并按当前数据库结构校验,通过后重启;重启时、在任何组件打开数据之前替换数据:数据库在一个事务内整体替换(失败则原数据不变,恢复包移到 `data/.restore-failed` 供排查),数据目录逐项替换;若文件替换失败,进程会非零退出由守护脚本重试,直到全部替换完成才启动业务引擎。卡片会显示上次恢复的结果。 +- 恢复会**覆盖当前实例的全部数据**并重启停止当前运行中的任务;备份中未结束的任务在恢复后会保持暂停,需手动恢复,不会在新机器上自动继续向目标发起请求。登录密码、Basic Auth、LLM 与其他配置都随数据库迁移,恢复后用原实例的密码登录。 - 新机器上已存在的同名 skill 保留本机版本(避免旧备份覆盖新版内置 skill),其余 skill 从备份装入。 - 只能恢复到**相同或更新版本**的 ARTEX;备份含当前版本不认识的表或列时会拒绝恢复。旧路径(如任务归档包的绝对路径)会自动改写到新数据目录。 - 恢复依赖 `start.sh` / `start.bat`(Docker 下为镜像 ENTRYPOINT)重启程序,与页面一键更新相同。 -- 备份与恢复期间均需要相当于数据量的临时磁盘空间(数据目录下)。 +- 恢复期间需在数据目录所在磁盘额外预留约一份**未压缩备份包**大小的临时空间(解包后的数据库文本与文件);若前面有 Nginx 反向代理,请按下方反代示例为 `/api/system/backup/` 放宽 `client_max_body_size` 并关闭 `proxy_request_buffering`。 ## 更新升级 @@ -465,6 +465,22 @@ server { proxy_http_version 1.1; proxy_set_header Connection ""; } + + # 一键备份 / 恢复:允许大体积备份流式上传与下载,避免触发默认 1m 限制与请求体缓冲 + location /api/system/backup/ { + proxy_pass http://127.0.0.1:8787; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + + client_max_body_size 0; + proxy_request_buffering off; + proxy_buffering off; + proxy_cache off; + proxy_read_timeout 3600s; + proxy_send_timeout 3600s; + proxy_http_version 1.1; + proxy_set_header Connection ""; + } } ``` diff --git a/cmd/artex/main.go b/cmd/artex/main.go index e2e6afb..8acec90 100644 --- a/cmd/artex/main.go +++ b/cmd/artex/main.go @@ -95,7 +95,12 @@ func run() int { if abs, err := filepath.Abs(skillDir); err == nil { skillDir = abs } - server.SetBootRestoreResult(server.ApplyPendingRestore(*dataDir, skillDir)) + restoreResult, err := server.ApplyPendingRestore(*dataDir, skillDir) + if err != nil { + log.Printf("[backup] 恢复文件替换未完成,终止本次启动等待重试:%v", err) + return 1 + } + server.SetBootRestoreResult(restoreResult) sigCtx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() diff --git a/db/backup.go b/db/backup.go index d5c79a6..c9d3dac 100644 --- a/db/backup.go +++ b/db/backup.go @@ -90,6 +90,20 @@ func queryTables(ctx context.Context, tx pgx.Tx) ([]BackupTable, error) { return tables, rows.Err() } +// CountActiveTasks returns how many non-deleted, non-terminal tasks are not +// paused (running, created/unpaused, or queued). Exporting while any exist +// would race with live disk writes and put active tasks into the backup. +func (d *DB) CountActiveTasks(ctx context.Context) (int, error) { + var count int + err := d.QueryRowContext(ctx, ` +SELECT COUNT(*) +FROM tasks +WHERE deleted_at IS NULL + AND paused = false + AND status NOT IN ('done', 'failed', 'timeout')`).Scan(&count) + return count, err +} + // DumpTables exports every application table from a single read-only // REPEATABLE READ snapshot, so rows written while the dump runs never produce a // backup whose tables disagree with each other. emit is called once per table @@ -238,6 +252,24 @@ WHERE left(archive_path, length($1)) = $1`, from, to, move.FromSeparator, move.T return fmt.Errorf("改写归档包路径: %w", err) } } + // Any non-terminal task that was not paused when backed up (for example, + // from an older backup) is forced into a manual pause so restoring onto a + // new host never resumes scanning targets on its own. + if _, err := tx.Exec(ctx, ` +UPDATE tasks +SET paused = true, + pause_reason = CASE WHEN COALESCE(pause_reason, '') = '' THEN 'paused_by_user' ELSE pause_reason END, + queued = false, + queued_at = NULL, + schedule_paused = false, + schedule_manual_until = NULL, + active_run_seconds = active_run_seconds + CASE WHEN run_segment_started_at IS NOT NULL THEN GREATEST(0, FLOOR(EXTRACT(EPOCH FROM (now() - run_segment_started_at)))::bigint) ELSE 0 END, + run_segment_started_at = NULL +WHERE deleted_at IS NULL + AND paused = false + AND status NOT IN ('done', 'failed', 'timeout')`); err != nil { + return fmt.Errorf("暂停恢复后的未完成任务: %w", err) + } for _, key := range keys { if _, err := tx.Exec(ctx, fmt.Sprintf("ALTER TABLE %s ADD CONSTRAINT %s %s", key.table, ident(key.name), key.definition)); err != nil { return fmt.Errorf("恢复外键 %s: %w", key.name, err) diff --git a/server/system_backup.go b/server/system_backup.go index 5e84328..2d01d18 100644 --- a/server/system_backup.go +++ b/server/system_backup.go @@ -3,6 +3,8 @@ package server import ( "archive/tar" "context" + "crypto/rand" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -24,10 +26,10 @@ import ( // // 备份包是 tar.zst,结构: // -// db/.copy PostgreSQL 各表的 COPY 文本数据(同一个快照事务内导出) +// db/
.copy PostgreSQL 各表的 COPY 文本数据(大表按分片追加 .000001…) // files/ 数据目录(任务工作区、会话、流量、证据、归档包、MITM CA…) // skills//… skill 目录 -// manifest.json 最后写入:表结构与行数、来源数据目录、顶层条目清单 +// manifest.json 最后写入:表结构、分片与行数、来源数据目录、顶层条目清单 // // 恢复分两步:上传时只解包到 /.restore-pending 并对照当前 schema 校验, // 然后请求重启;真正的替换在下次启动、任何组件打开数据之前由 ApplyPendingRestore @@ -35,17 +37,22 @@ import ( // 数据库在一个事务内整体替换,失败则保持原数据不动。 const ( - backupFormat = 1 - backupManifestName = "manifest.json" - backupPendingDir = ".restore-pending" - backupUploadPrefix = ".restore-upload-" - backupPreviousDir = ".restore-previous" - backupFailedDir = ".restore-failed" - backupDBDoneMarker = ".db-restored" - backupMaxFiles = 20_000_000 - backupTrafficIndex = "traffic/_index" // live SQLite; exported via SnapshotIndex + backupFormat = 1 + backupManifestName = "manifest.json" + backupPendingDir = ".restore-pending" + backupUploadPrefix = ".restore-upload-" + backupPreviousDir = ".restore-previous" + backupFailedDir = ".restore-failed" + backupDBDoneMarker = ".db-restored" + backupRestoreIDFile = ".restore-id" + backupMaxFiles = 20_000_000 + backupTrafficIndex = "traffic/_index" // live SQLite; exported via SnapshotIndex ) +// backupCopyChunkBytes bounds each streamed DB COPY tar entry so large tables +// flush compressed bytes to the client without disk spooling or proxy timeouts. +var backupCopyChunkBytes = 8 << 20 + // backupSkipped lists data-dir paths (slash form, relative) that hold transient // staging state rather than data. Top-level dot entries are skipped as well. var backupSkipped = map[string]bool{ @@ -56,13 +63,18 @@ var backupSkipped = map[string]bool{ "traffic/_index/.snapshot": true, } +type backupTableEntry struct { + pgdb.BackupTable + Parts int `json:"parts,omitempty"` +} + type backupManifest struct { Format int `json:"format"` Version string `json:"version"` CreatedAt time.Time `json:"created_at"` DataDir string `json:"data_dir"` Separator string `json:"separator"` - Tables []pgdb.BackupTable `json:"tables"` + Tables []backupTableEntry `json:"tables"` Entries []string `json:"entries"` Skills []string `json:"skills"` } @@ -75,6 +87,7 @@ var backupMu sync.Mutex // BackupRestoreResult is the outcome of the boot-time restore, reported to the // UI after the restart that applied it. type BackupRestoreResult struct { + ID string `json:"id,omitempty"` Applied bool `json:"applied"` Error string `json:"error,omitempty"` At time.Time `json:"at"` @@ -113,10 +126,22 @@ func (s *Server) backupExport(w http.ResponseWriter, r *http.Request) { return } if !backupMu.TryLock() { + w.Header().Set("X-Artex-Error-Code", "backup_busy") writeProductError(w, 409, "backup_busy", "已有备份或恢复正在进行", nil) return } defer backupMu.Unlock() + active, err := s.m.pg.CountActiveTasks(r.Context()) + if err != nil { + log.Printf("[backup] 检查运行中任务失败:%v", err) + writeProductError(w, 500, "data_unavailable", "检查运行中任务失败", nil) + return + } + if active > 0 { + w.Header().Set("X-Artex-Error-Code", "backup_active_tasks") + writeProductError(w, 409, "backup_active_tasks", "存在未暂停的运行中或排队中任务,请先暂停或等待任务完成后再下载备份", map[string]any{"count": active}) + return + } name := "artex-backup-" + time.Now().Format("20060102-150405") + ".tar.zst" w.Header().Set("Content-Type", "application/zstd") w.Header().Set("X-Content-Type-Options", "nosniff") @@ -134,39 +159,58 @@ func (s *Server) backupExport(w http.ResponseWriter, r *http.Request) { } func (s *Server) writeBackup(ctx context.Context, out io.Writer) error { - spool, err := os.MkdirTemp(s.m.dir, ".backup-") - if err != nil { - return err - } - defer os.RemoveAll(spool) encoder, err := zstd.NewWriter(out, zstd.WithEncoderLevel(zstd.SpeedDefault), zstd.WithEncoderCRC(true)) if err != nil { return err } + flusher, _ := out.(http.Flusher) tw := tar.NewWriter(encoder) manifest := backupManifest{ Format: backupFormat, Version: BuildVersion, CreatedAt: time.Now().UTC(), DataDir: s.m.dir, Separator: string(filepath.Separator), Entries: []string{}, Skills: []string{}, } + chunkMax := backupCopyChunkBytes + if chunkMax <= 0 { + chunkMax = 8 << 20 + } + var copyBuf []byte err = s.m.pg.DumpTables(ctx, func(table pgdb.BackupTable, copyTo func(io.Writer) (int64, error)) error { - path := filepath.Join(spool, "table.copy") - file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, archiveFileMode) + cw := copyChunkWriter{ + tw: tw, + encoder: encoder, + flusher: flusher, + table: table.Name, + modTime: manifest.CreatedAt, + buf: copyBuf[:0], + chunkMax: chunkMax, + } + rows, copyErr := copyTo(&cw) + if copyErr != nil { + return copyErr + } + parts, err := cw.finish() if err != nil { return err } - rows, copyErr := copyTo(file) - if err := errors.Join(copyErr, file.Close()); err != nil { - return err - } + copyBuf = cw.buf[:0] table.Rows = rows - manifest.Tables = append(manifest.Tables, table) - return addBackupFile(tw, path, "db/"+table.Name+".copy") + entry := backupTableEntry{BackupTable: table} + if parts > 1 { + entry.Parts = parts + } + manifest.Tables = append(manifest.Tables, entry) + return nil }) if err != nil { return err } snapshot := "" if tr := s.m.Traffic(); tr != nil { + spool, err := os.MkdirTemp(s.m.dir, ".backup-") + if err != nil { + return err + } + defer os.RemoveAll(spool) snapshot = filepath.Join(spool, "index.sqlite") if err := tr.SnapshotIndex(snapshot); err != nil { return fmt.Errorf("快照流量索引: %w", err) @@ -174,6 +218,12 @@ func (s *Server) writeBackup(ctx context.Context, out io.Writer) error { if err := addBackupFile(tw, snapshot, "files/"+backupTrafficIndex+"/index.sqlite"); err != nil { return err } + if err := encoder.Flush(); err != nil { + return err + } + if flusher != nil { + flusher.Flush() + } } err = walkBackupTree(s.m.dir, func(relative string, info fs.FileInfo) (bool, error) { if !strings.Contains(relative, "/") { @@ -223,6 +273,84 @@ func (s *Server) writeBackup(ctx context.Context, out io.Writer) error { return encoder.Close() } +func backupTableCopyPath(table string, part int) string { + if part == 0 { + return "db/" + table + ".copy" + } + return fmt.Sprintf("db/%s.copy.%06d", table, part) +} + +func backupTablesOnly(entries []backupTableEntry) []pgdb.BackupTable { + tables := make([]pgdb.BackupTable, len(entries)) + for i, entry := range entries { + tables[i] = entry.BackupTable + } + return tables +} + +type copyChunkWriter struct { + tw *tar.Writer + encoder *zstd.Encoder + flusher http.Flusher + table string + modTime time.Time + buf []byte + chunkMax int + parts int +} + +func (w *copyChunkWriter) Write(p []byte) (int, error) { + total := len(p) + for len(p) > 0 { + space := w.chunkMax - len(w.buf) + n := min(space, len(p)) + w.buf = append(w.buf, p[:n]...) + p = p[n:] + if len(w.buf) == w.chunkMax { + if err := w.flushChunk(); err != nil { + return total - len(p), err + } + } + } + return total, nil +} + +func (w *copyChunkWriter) finish() (int, error) { + if len(w.buf) > 0 || w.parts == 0 { + if err := w.flushChunk(); err != nil { + return w.parts, err + } + } + return w.parts, nil +} + +func (w *copyChunkWriter) flushChunk() error { + header := &tar.Header{ + Name: backupTableCopyPath(w.table, w.parts), + Mode: archiveFileMode, + Size: int64(len(w.buf)), + ModTime: w.modTime, + Typeflag: tar.TypeReg, + } + if err := w.tw.WriteHeader(header); err != nil { + return err + } + if len(w.buf) > 0 { + if _, err := w.tw.Write(w.buf); err != nil { + return err + } + } + w.buf = w.buf[:0] + w.parts++ + if err := w.encoder.Flush(); err != nil { + return err + } + if w.flusher != nil { + w.flusher.Flush() + } + return nil +} + // walkBackupTree visits root's descendants in lexical order with slash-form // relative paths. visit returns false to skip an entry (and its subtree). // Symlinks are skipped: the package format only carries files and directories. @@ -315,24 +443,32 @@ func (s *Server) backupRestore(w http.ResponseWriter, r *http.Request) { return } defer backupMu.Unlock() - manifest, err := s.stageBackupUpload(r.Context(), r.Body) + manifest, restoreID, err := s.stageBackupUpload(r.Context(), r.Body) if err != nil { log.Printf("[backup] 恢复包校验失败:%v", err) writeProductError(w, 400, "backup_invalid", "备份包无效:"+err.Error(), map[string]any{"message": err.Error()}) return } log.Printf("[backup] 已接收备份(版本 %s,创建于 %s),即将重启以完成恢复", manifest.Version, manifest.CreatedAt.Format(time.RFC3339)) - writeJSON(w, 202, map[string]any{"ok": true, "version": manifest.Version, "created_at": manifest.CreatedAt}) + writeJSON(w, 202, map[string]any{"ok": true, "restore_id": restoreID, "version": manifest.Version, "created_at": manifest.CreatedAt}) go func() { time.Sleep(500 * time.Millisecond) requestRestart() }() } -func (s *Server) stageBackupUpload(ctx context.Context, body io.Reader) (*backupManifest, error) { +func newBackupRestoreID() string { + var raw [16]byte + if _, err := rand.Read(raw[:]); err == nil { + return hex.EncodeToString(raw[:]) + } + return fmt.Sprintf("%d", time.Now().UnixNano()) +} + +func (s *Server) stageBackupUpload(ctx context.Context, body io.Reader) (*backupManifest, string, error) { upload, err := os.MkdirTemp(s.m.dir, backupUploadPrefix) if err != nil { - return nil, err + return nil, "", err } staged := false defer func() { @@ -341,24 +477,41 @@ func (s *Server) stageBackupUpload(ctx context.Context, body io.Reader) (*backup } }() if err := extractTarZst(body, upload, backupMaxFiles); err != nil { - return nil, fmt.Errorf("解包失败: %w", err) + return nil, "", fmt.Errorf("解包失败: %w", err) } manifest, err := readBackupManifest(upload) if err != nil { - return nil, err + return nil, "", err } - if err := s.m.pg.CheckRestoreTables(ctx, manifest.Tables); err != nil { - return nil, err + if err := s.m.pg.CheckRestoreTables(ctx, backupTablesOnly(manifest.Tables)); err != nil { + return nil, "", err + } + restoreID := newBackupRestoreID() + if err := os.WriteFile(filepath.Join(upload, backupRestoreIDFile), []byte(restoreID), archiveFileMode); err != nil { + return nil, "", err } pending := filepath.Join(s.m.dir, backupPendingDir) - if err := os.RemoveAll(pending); err != nil { - return nil, err + oldPending := upload + "-old" + hadPending := false + if _, err := os.Lstat(pending); err == nil { + if err := os.Rename(pending, oldPending); err != nil { + return nil, "", err + } + hadPending = true + } else if !errors.Is(err, fs.ErrNotExist) { + return nil, "", err } if err := os.Rename(upload, pending); err != nil { - return nil, err + if hadPending { + _ = os.Rename(oldPending, pending) + } + return nil, "", err } staged = true - return manifest, nil + if hadPending { + _ = os.RemoveAll(oldPending) + } + return manifest, restoreID, nil } // readBackupManifest loads and validates the manifest of an extracted package: @@ -386,8 +539,14 @@ func readBackupManifest(root string) (*backupManifest, error) { if !singleElement(table.Name) || len(table.Columns) == 0 { return nil, fmt.Errorf("备份表名无效: %q", table.Name) } - if _, err := os.Stat(filepath.Join(root, "db", table.Name+".copy")); err != nil { - return nil, fmt.Errorf("缺少表 %s 的数据", table.Name) + parts := max(table.Parts, 1) + if table.Parts < 0 || parts > backupMaxFiles { + return nil, fmt.Errorf("备份表 %s 分片数无效: %d", table.Name, table.Parts) + } + for part := range parts { + if _, err := os.Stat(filepath.Join(root, filepath.FromSlash(backupTableCopyPath(table.Name, part)))); err != nil { + return nil, fmt.Errorf("缺少表 %s 的数据", table.Name) + } } } for _, name := range manifest.Entries { @@ -416,7 +575,7 @@ func singleElement(name string) bool { // rename at a time, so a crash at any point is finished by the next boot. A // database failure leaves current data untouched and parks the package in // .restore-failed for inspection. -func ApplyPendingRestore(dataDir, skillDir string) *BackupRestoreResult { +func ApplyPendingRestore(dataDir, skillDir string) (*BackupRestoreResult, error) { if abs, err := filepath.Abs(dataDir); err == nil { dataDir = abs } @@ -427,10 +586,13 @@ func ApplyPendingRestore(dataDir, skillDir string) *BackupRestoreResult { _ = os.RemoveAll(previous) // leftover of a restore that crashed during cleanup } cleanBackupUploads(dataDir) - return nil + return nil, nil } cleanBackupUploads(dataDir) result := &BackupRestoreResult{At: time.Now()} + if rawID, err := os.ReadFile(filepath.Join(pending, backupRestoreIDFile)); err == nil { + result.ID = strings.TrimSpace(string(rawID)) + } manifest, err := readBackupManifest(pending) if err == nil { result.Source, result.Created = manifest.Version, manifest.CreatedAt @@ -446,23 +608,26 @@ func ApplyPendingRestore(dataDir, skillDir string) *BackupRestoreResult { log.Printf("[backup] 无法移走失败的恢复包,删除之:%v", renameErr) _ = os.RemoveAll(pending) } - return result + return result, nil } if err := installBackupFiles(pending, dataDir, skillDir, manifest); err != nil { - // The database is already replaced; files are retried on the next boot. + // The database is already replaced; do not start the engine on half-installed + // files. Returning an error makes main exit non-zero so the supervisor retries. result.Error = "数据库已恢复,但文件替换失败(下次启动会继续):" + err.Error() log.Printf("[backup] %s", result.Error) - return result + return result, err } if err := os.RemoveAll(previous); err != nil { log.Printf("[backup] 清理旧数据失败:%v", err) } if err := os.RemoveAll(pending); err != nil { - log.Printf("[backup] 清理恢复包失败:%v", err) + result.Error = "恢复已完成,但清理暂存目录失败:" + err.Error() + log.Printf("[backup] %s", result.Error) + return result, err } result.Applied = true log.Printf("[backup] 备份恢复完成") - return result + return result, nil } func cleanBackupUploads(dataDir string) { @@ -497,8 +662,11 @@ func restoreBackupDatabase(pending, dataDir string, manifest *backupManifest) er } tables := make([]pgdb.RestoreTable, len(manifest.Tables)) for i, table := range manifest.Tables { - path := filepath.Join(pending, "db", table.Name+".copy") - tables[i] = pgdb.RestoreTable{BackupTable: table, Open: func() (io.ReadCloser, error) { return os.Open(path) }} + name, parts := table.Name, max(table.Parts, 1) + tables[i] = pgdb.RestoreTable{ + BackupTable: table.BackupTable, + Open: func() (io.ReadCloser, error) { return openBackupTableParts(pending, name, parts) }, + } } move := pgdb.PathPrefixMove{From: manifest.DataDir, FromSeparator: manifest.Separator, To: dataDir, ToSeparator: string(filepath.Separator)} if err := pg.RestoreTables(context.Background(), tables, move); err != nil { @@ -516,6 +684,62 @@ func restoreBackupDatabase(pending, dataDir string, manifest *backupManifest) er return writeArchiveJournal(marker, map[string]any{"at": time.Now()}) } +func openBackupTableParts(pending, table string, parts int) (io.ReadCloser, error) { + if parts <= 1 { + return os.Open(filepath.Join(pending, filepath.FromSlash(backupTableCopyPath(table, 0)))) + } + paths := make([]string, parts) + for i := range parts { + paths[i] = filepath.Join(pending, filepath.FromSlash(backupTableCopyPath(table, i))) + } + return &multiFileReader{paths: paths}, nil +} + +type multiFileReader struct { + paths []string + index int + current *os.File +} + +func (r *multiFileReader) Read(p []byte) (int, error) { + for { + if r.current == nil { + if r.index >= len(r.paths) { + return 0, io.EOF + } + f, err := os.Open(r.paths[r.index]) + if err != nil { + return 0, err + } + r.current = f + r.index++ + } + n, err := r.current.Read(p) + if n > 0 || (err != nil && !errors.Is(err, io.EOF)) { + if errors.Is(err, io.EOF) { + _ = r.current.Close() + r.current = nil + return n, nil + } + return n, err + } + if closeErr := r.current.Close(); closeErr != nil { + r.current = nil + return 0, closeErr + } + r.current = nil + } +} + +func (r *multiFileReader) Close() error { + if r.current != nil { + err := r.current.Close() + r.current = nil + return err + } + return nil +} + func installBackupFiles(pending, dataDir, skillDir string, manifest *backupManifest) error { previous := filepath.Join(dataDir, backupPreviousDir) if err := os.MkdirAll(previous, archiveDirMode); err != nil { diff --git a/server/system_backup_test.go b/server/system_backup_test.go index bc0b916..faee5f6 100644 --- a/server/system_backup_test.go +++ b/server/system_backup_test.go @@ -4,7 +4,9 @@ import ( "bytes" "context" "database/sql" + "encoding/json" "fmt" + pgdb "github.com/Autumn-27/artex/db" "io" "net/http/httptest" "net/url" @@ -13,8 +15,6 @@ import ( "strings" "testing" "time" - - pgdb "github.com/Autumn-27/artex/db" ) // scratchDSN points the configured server at a fresh database that is dropped @@ -82,6 +82,24 @@ func TestBackupExportRestoreRoundTrip(t *testing.T) { writeTestFile(t, filepath.Join(sourceSkills, "shipped", "SKILL.md"), "old copy") exporter := &Server{m: source, skillDir: sourceSkills} + + // Exporting while an active (non-terminal, non-paused) task exists must be refused. + for _, method := range []string{"HEAD", "GET"} { + blocked := httptest.NewRecorder() + exporter.backupExport(blocked, httptest.NewRequest(method, "/api/system/backup/export", nil)) + if blocked.Code != 409 || blocked.Header().Get("X-Artex-Error-Code") != "backup_active_tasks" { + t.Fatalf("%s with active task: status=%d header=%q body=%s", method, blocked.Code, blocked.Header().Get("X-Artex-Error-Code"), blocked.Body.String()) + } + } + if _, err := source.pg.Exec(`UPDATE tasks SET paused=true, pause_reason='paused_by_user' WHERE id=$1`, task.ID); err != nil { + t.Fatal(err) + } + + // Force small chunk size so multi-part COPY streaming is exercised end-to-end. + prevChunk := backupCopyChunkBytes + backupCopyChunkBytes = 128 + t.Cleanup(func() { backupCopyChunkBytes = prevChunk }) + exported := httptest.NewRecorder() exporter.backupExport(exported, httptest.NewRequest("GET", "/api/system/backup/export", nil)) source.Close() @@ -104,19 +122,58 @@ func TestBackupExportRestoreRoundTrip(t *testing.T) { restorer := &Server{m: target, skillDir: targetSkills} upload := httptest.NewRecorder() restorer.backupRestore(upload, httptest.NewRequest("POST", "/api/system/backup/restore", bytes.NewReader(exported.Body.Bytes()))) - target.Close() if upload.Code != 202 { + target.Close() t.Fatalf("restore upload status=%d body=%s", upload.Code, upload.Body.String()) } - - result := ApplyPendingRestore(targetDir, targetSkills) - if result == nil || !result.Applied { - t.Fatalf("restore not applied: %+v", result) + var uploadResp struct { + RestoreID string `json:"restore_id"` } - if again := ApplyPendingRestore(targetDir, targetSkills); again != nil { - t.Fatalf("pending restore applied twice: %+v", again) + if err := json.Unmarshal(upload.Body.Bytes(), &uploadResp); err != nil || uploadResp.RestoreID == "" { + target.Close() + t.Fatalf("missing restore_id in 202 response: %s (err=%v)", upload.Body.String(), err) } + // A failed subsequent upload must leave the already-staged pending backup intact. + badUpload := httptest.NewRecorder() + restorer.backupRestore(badUpload, httptest.NewRequest("POST", "/api/system/backup/restore", strings.NewReader("corrupt"))) + if badUpload.Code != 400 { + target.Close() + t.Fatalf("corrupt upload status=%d body=%s", badUpload.Code, badUpload.Body.String()) + } + if rawID, err := os.ReadFile(filepath.Join(targetDir, backupPendingDir, backupRestoreIDFile)); err != nil || strings.TrimSpace(string(rawID)) != uploadResp.RestoreID { + target.Close() + t.Fatalf("corrupt upload clobbered pending restore_id: got=%q err=%v want=%q", string(rawID), err, uploadResp.RestoreID) + } + + // Uploading a second valid backup atomically replaces pending without leaving -old dirs. + upload2 := httptest.NewRecorder() + restorer.backupRestore(upload2, httptest.NewRequest("POST", "/api/system/backup/restore", bytes.NewReader(exported.Body.Bytes()))) + target.Close() + if upload2.Code != 202 { + t.Fatalf("second upload status=%d body=%s", upload2.Code, upload2.Body.String()) + } + var uploadResp2 struct { + RestoreID string `json:"restore_id"` + } + if err := json.Unmarshal(upload2.Body.Bytes(), &uploadResp2); err != nil || uploadResp2.RestoreID == "" || uploadResp2.RestoreID == uploadResp.RestoreID { + t.Fatalf("unexpected second restore_id: %s (first=%q err=%v)", upload2.Body.String(), uploadResp.RestoreID, err) + } + if entries, err := os.ReadDir(targetDir); err == nil { + for _, entry := range entries { + if strings.HasPrefix(entry.Name(), backupUploadPrefix) { + t.Fatalf("staging left upload/old dir behind: %s", entry.Name()) + } + } + } + + result, err := ApplyPendingRestore(targetDir, targetSkills) + if err != nil || result == nil || !result.Applied || result.ID != uploadResp2.RestoreID { + t.Fatalf("restore not applied: result=%+v err=%v wantID=%q", result, err, uploadResp2.RestoreID) + } + if again, err := ApplyPendingRestore(targetDir, targetSkills); err != nil || again != nil { + t.Fatalf("pending restore applied twice: result=%+v err=%v", again, err) + } restored, err := pgdb.Open(targetDSN) if err != nil { t.Fatal(err) @@ -209,3 +266,164 @@ func TestBackupRestoreRejectsForeignPackage(t *testing.T) { } } } + +func TestBackupRestoreWindowsPathsAndPausesUnpausedTasks(t *testing.T) { + base, _, err := pgdb.DSN() + if err != nil { + t.Skipf("no database config (%v)", err) + } + suffix := fmt.Sprintf("%d", time.Now().UnixNano()) + srcDSN := scratchDSN(t, base, "artex_backup_win_src_"+suffix) + dstDSN := scratchDSN(t, base, "artex_backup_win_dst_"+suffix) + + src, err := pgdb.Open(srcDSN) + if err != nil { + t.Skipf("postgres unavailable (%v)", err) + } + defer src.Close() + activeTask, err := src.CreateTask("active", "running goal", nil, 0, 0) + if err != nil { + t.Fatal(err) + } + if _, err := src.Exec(`UPDATE tasks SET status='running', paused=false, queued=true, schedule_paused=true WHERE id=$1`, activeTask.ID); err != nil { + t.Fatal(err) + } + doneTask, err := src.CreateTask("done", "finished goal", nil, 0, 0) + if err != nil { + t.Fatal(err) + } + if _, err := src.Exec(`UPDATE tasks SET status='done', paused=false WHERE id=$1`, doneTask.ID); err != nil { + t.Fatal(err) + } + otherTask, err := src.CreateTask("other", "linux path goal", nil, 0, 0) + if err != nil { + t.Fatal(err) + } + if _, err := src.Exec(`UPDATE tasks SET status='done', paused=false WHERE id=$1`, otherTask.ID); err != nil { + t.Fatal(err) + } + for _, row := range []struct { + id int64 + path string + }{ + {activeTask.ID, `C:\artex\data\archives\tasks\win-1.tar.zst`}, + {doneTask.ID, `C:\artex\data-old\archives\tasks\prefix-collision.tar.zst`}, + {otherTask.ID, `/app/data/archives/tasks/already-linux.tar.zst`}, + } { + if _, err := src.Exec(`INSERT INTO task_archives(task_id, state, archive_path) VALUES ($1, 'ready', $2)`, row.id, row.path); err != nil { + t.Fatal(err) + } + } + + var dumped []pgdb.RestoreTable + err = src.DumpTables(context.Background(), func(table pgdb.BackupTable, copyTo func(io.Writer) (int64, error)) error { + var buf bytes.Buffer + rows, err := copyTo(&buf) + if err != nil { + return err + } + table.Rows = rows + raw := buf.Bytes() + dumped = append(dumped, pgdb.RestoreTable{ + BackupTable: table, + Open: func() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(raw)), nil }, + }) + return nil + }) + if err != nil { + t.Fatal(err) + } + + dst, err := pgdb.Open(dstDSN) + if err != nil { + t.Fatal(err) + } + defer dst.Close() + move := pgdb.PathPrefixMove{From: `C:\artex\data`, FromSeparator: `\`, To: `/app/data`, ToSeparator: `/`} + if err := dst.RestoreTables(context.Background(), dumped, move); err != nil { + t.Fatal(err) + } + + var paused, queued, schedulePaused bool + var pauseReason string + if err := dst.QueryRow(`SELECT paused, pause_reason, queued, schedule_paused FROM tasks WHERE id=$1`, activeTask.ID). + Scan(&paused, &pauseReason, &queued, &schedulePaused); err != nil { + t.Fatal(err) + } + if !paused || pauseReason != "paused_by_user" || queued || schedulePaused { + t.Fatalf("restored active task not safely paused: paused=%v reason=%q queued=%v schedule_paused=%v", paused, pauseReason, queued, schedulePaused) + } + if err := dst.QueryRow(`SELECT paused FROM tasks WHERE id=$1`, doneTask.ID).Scan(&paused); err != nil { + t.Fatal(err) + } + if paused { + t.Fatal("terminal done task should not be flipped to paused") + } + for id, want := range map[int64]string{ + activeTask.ID: `/app/data/archives/tasks/win-1.tar.zst`, + doneTask.ID: `C:\artex\data-old\archives\tasks\prefix-collision.tar.zst`, + otherTask.ID: `/app/data/archives/tasks/already-linux.tar.zst`, + } { + var got string + if err := dst.QueryRow(`SELECT archive_path FROM task_archives WHERE task_id=$1`, id).Scan(&got); err != nil { + t.Fatal(err) + } + if got != want { + t.Fatalf("task %d archive_path = %q, want %q", id, got, want) + } + } +} + +func TestApplyPendingRestoreFailsFastOnFileInstallError(t *testing.T) { + dataDir := t.TempDir() + pending := filepath.Join(dataDir, backupPendingDir) + writeTestFile(t, filepath.Join(pending, backupDBDoneMarker), `{"at":"2026-11-20T00:00:00Z"}`) + writeTestFile(t, filepath.Join(pending, backupRestoreIDFile), "rid-file-fail") + writeTestFile(t, filepath.Join(pending, "db", "settings.copy"), "") + writeTestFile(t, filepath.Join(pending, "files", "tasks", "1", "note.txt"), "restored note") + writeTestFile(t, filepath.Join(pending, "skills", "custom-skill", "SKILL.md"), "skill body") + manifest := backupManifest{ + Format: backupFormat, + Version: "v0.3.30", + CreatedAt: time.Now().UTC(), + Tables: []backupTableEntry{{BackupTable: pgdb.BackupTable{Name: "settings", Columns: []string{"key", "value"}}}}, + Entries: []string{"tasks"}, + Skills: []string{"custom-skill"}, + } + raw, err := json.Marshal(manifest) + if err != nil { + t.Fatal(err) + } + writeTestFile(t, filepath.Join(pending, backupManifestName), string(raw)) + + // A regular file where skillDir should be forces installBackupFiles to fail after DB commit. + blockingSkillPath := filepath.Join(t.TempDir(), "not-a-dir") + writeTestFile(t, blockingSkillPath, "blocker") + + res, err := ApplyPendingRestore(dataDir, blockingSkillPath) + if err == nil { + t.Fatalf("expected error on file install failure, got result=%+v", res) + } + if res == nil || res.Applied || res.ID != "rid-file-fail" { + t.Fatalf("unexpected failure result: %+v", res) + } + if _, statErr := os.Stat(pending); statErr != nil { + t.Fatalf("pending dir must stay for retry on next boot: %v", statErr) + } + + // Once the blocker is fixed, the next boot resumes and finishes cleanly. + realSkillDir := t.TempDir() + res, err = ApplyPendingRestore(dataDir, realSkillDir) + if err != nil || res == nil || !res.Applied || res.ID != "rid-file-fail" { + t.Fatalf("resume after file install failure failed: result=%+v err=%v", res, err) + } + if got, err := os.ReadFile(filepath.Join(dataDir, "tasks", "1", "note.txt")); err != nil || string(got) != "restored note" { + t.Fatalf("tasks note=%q err=%v", got, err) + } + if got, err := os.ReadFile(filepath.Join(realSkillDir, "custom-skill", "SKILL.md")); err != nil || string(got) != "skill body" { + t.Fatalf("skill=%q err=%v", got, err) + } + if _, statErr := os.Stat(pending); !os.IsNotExist(statErr) { + t.Fatalf("pending dir should be removed after completion: %v", statErr) + } +} diff --git a/web/src/app/(main)/system/settings/_components/backup-card.tsx b/web/src/app/(main)/system/settings/_components/backup-card.tsx index bd87a1d..13970ee 100644 --- a/web/src/app/(main)/system/settings/_components/backup-card.tsx +++ b/web/src/app/(main)/system/settings/_components/backup-card.tsx @@ -49,9 +49,9 @@ export function BackupCard() { } }; - // 重启期间接口连不上是预期的;以"出现了晚于本次上传的恢复结果"为准判断新进程已就绪。 + // 重启期间接口连不上是预期的;以"返回本次上传对应的 restore_id 结果"为准判断新进程已就绪。 const waitForRestore = React.useCallback( - async (since: number) => { + async (restoreId: string) => { setRestarting(true); const deadline = Date.now() + RESTORE_TIMEOUT_MS; while (Date.now() < deadline) { @@ -60,8 +60,7 @@ export function BackupCard() { await tick.promise; try { const next = await api.backupStatus(); - const at = next.last_restore ? Date.parse(next.last_restore.at) : 0; - if (at >= since) { + if (next.last_restore?.id === restoreId) { window.location.reload(); return; } @@ -80,11 +79,10 @@ export function BackupCard() { const selected = file; setFile(null); setUploading(true); - const since = Date.now() - 60_000; // 容忍浏览器与服务器之间的时钟偏差 try { - await api.restoreBackup(selected); + const res = await api.restoreBackup(selected); setUploading(false); - await waitForRestore(since); + await waitForRestore(res.restore_id); } catch (e) { setUploading(false); toast.error(t("settings.backup.restoreFailedPrefix") + localizedError(e, t)); diff --git a/web/src/i18n/error-messages.ts b/web/src/i18n/error-messages.ts index 31b8b46..b68d2e5 100644 --- a/web/src/i18n/error-messages.ts +++ b/web/src/i18n/error-messages.ts @@ -4,6 +4,10 @@ export const errorMessages = { "The intelligence response is incomplete. Refresh to try again.", ], "error.backup_busy": ["已有备份或恢复正在进行", "A backup or restore is already in progress"], + "error.backup_active_tasks": [ + "存在未暂停的运行中或排队中任务,请先暂停或等待任务完成后再下载备份", + "Active or queued tasks are running. Pause them or wait for them to finish before downloading a backup.", + ], "error.backup_update_running": [ "更新正在进行中,无法恢复备份", "An update is in progress. The backup cannot be restored now.", diff --git a/web/src/i18n/settings-messages.ts b/web/src/i18n/settings-messages.ts index 15f8da5..aa06a5d 100644 --- a/web/src/i18n/settings-messages.ts +++ b/web/src/i18n/settings-messages.ts @@ -130,8 +130,8 @@ export const settingsMessages = { "settings.backup.restoreFailedPrefix": ["恢复失败:", "Restore failed: "], "settings.backup.confirmTitle": ["用备份替换当前全部数据?", "Replace all current data with this backup?"], "settings.backup.confirmBody": [ - "当前实例的任务、资产、漏洞、配置、文件和流量都会被「{{name}}」中的内容替换,且无法撤销。上传校验通过后程序会重启,正在运行的任务会被中断。建议先下载一份当前数据的备份。", - "Tasks, assets, findings, settings, files and traffic on this instance will be replaced by the contents of “{{name}}”. This cannot be undone. After the upload is verified, ARTEX restarts and running tasks are interrupted. Download a backup of the current data first.", + "当前实例的任务、资产、漏洞、配置、文件和流量都会被「{{name}}」中的内容替换,且无法撤销。上传校验通过后程序会重启并停止当前运行中的任务;备份中未结束的任务在恢复后会保持暂停,不会自动继续执行。建议先下载一份当前数据的备份。", + "Tasks, assets, findings, settings, files and traffic on this instance will be replaced by the contents of “{{name}}”. This cannot be undone. After the upload is verified, ARTEX restarts and stops tasks currently running here; any unfinished tasks from the backup start paused rather than resuming automatically. Download a backup of the current data first.", ], "settings.backup.cancel": ["取消", "Cancel"], "settings.backup.confirm": ["替换并重启", "Replace and restart"], @@ -140,8 +140,8 @@ export const settingsMessages = { "Backup verified. Restarting to restore data. Please wait; the page will reload automatically…", ], "settings.backup.restoreTimeout": [ - "等待服务重启超时。请检查后端日志,或确认 artex 是通过 start.sh / start.bat 启动的。", - "Timed out waiting for the service to restart. Check the backend logs and confirm ARTEX was started with start.sh / start.bat.", + "等待服务重启超时(大备份的数据库恢复可能仍在进行)。请查看后端日志、稍后刷新页面,或确认 artex 是通过 start.sh / start.bat 启动的。", + "Timed out waiting for the service to restart (a large backup may still be restoring on the server). Check the backend logs, refresh later, or confirm ARTEX was started with start.sh / start.bat.", ], "settings.backup.lastRestoreOk": [ "已于 {{time}} 从备份恢复(备份版本 {{version}})。", @@ -153,8 +153,8 @@ export const settingsMessages = { "An uploaded backup will be restored at the next restart.", ], "settings.backup.note": [ - "恢复需要相同或更新版本的 ARTEX。登录密码、Basic Auth 与 LLM 配置随数据库一起迁移,恢复后请使用原实例的密码登录;新机器上已有的同名 skill 保留本机版本。", - "Restoring requires the same or a newer ARTEX version. The sign-in password, Basic Auth and LLM settings move with the database, so sign in with the original instance's password afterwards. Skills that already exist on the new machine keep the local copy.", + "下载备份前需先暂停或等待所有运行中、排队中的任务结束。恢复需要相同或更新版本的 ARTEX;登录密码、Basic Auth 与 LLM 配置随数据库一起迁移,恢复后请使用原实例的密码登录;新机器上已有的同名 skill 保留本机版本。", + "Pause or finish active and queued tasks before downloading a backup. Restoring requires the same or a newer ARTEX version. The sign-in password, Basic Auth and LLM settings move with the database, so sign in with the original instance's password afterwards. Skills that already exist on the new machine keep the local copy.", ], "settings.keywords.basicAuth": [ "基础认证 登录 用户名 密码", diff --git a/web/src/lib/api.ts b/web/src/lib/api.ts index e487a9b..0c4040d 100644 --- a/web/src/lib/api.ts +++ b/web/src/lib/api.ts @@ -168,6 +168,7 @@ export interface BackupStatus { pending: boolean; // 本次启动时执行的恢复结果;未执行恢复时为 null。 last_restore: { + id?: string; applied: boolean; error?: string; at: string; @@ -439,11 +440,10 @@ export const api = { method: "HEAD", headers: token ? { Authorization: `Bearer ${token}` } : {}, }); - if (!response.ok) - throw new ApiError( - `HEAD /system/backup/export: ${response.status}`, - response.status === 409 ? "backup_busy" : undefined, - ); + if (!response.ok) { + const code = response.headers.get("X-Artex-Error-Code") ?? (response.status === 409 ? "backup_busy" : undefined); + throw new ApiError(`HEAD /system/backup/export: ${response.status}`, code); + } const a = document.createElement("a"); a.href = "/api/system/backup/export"; document.body.appendChild(a); @@ -451,7 +451,7 @@ export const api = { a.remove(); }, restoreBackup: (file: File) => - http<{ ok: boolean; version: string; created_at: string }>("/system/backup/restore", { + http<{ ok: boolean; restore_id: string; version: string; created_at: string }>("/system/backup/restore", { method: "POST", body: file, headers: { "Content-Type": "application/octet-stream" },