diff --git a/CHANGELOG.md b/CHANGELOG.md index e6d8a3f..089bef9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ ## [Unreleased] +### 新增与改进 + +- 新增一键备份与迁移(#160):在系统配置中下载包含数据库、任务工作区、会话、流量、证据、归档包、MITM 证书和 skills 的单个备份文件,并可在另一实例上传恢复。恢复先校验备份,再在重启时整体替换;数据库替换失败时保留原数据,中断后下次启动继续完成文件替换。 + ## [0.3.30] - 2026-11-14 ### 新增与改进 diff --git a/README.md b/README.md index 4f658af..8d3ec18 100644 --- a/README.md +++ b/README.md @@ -189,6 +189,18 @@ ARTEX_TARGETS=linux/amd64,windows/amd64 ./build.sh --release --- +## 一键备份与迁移 + +在 **系统配置 → 系统与账号** 的 **备份与迁移** 卡片中: + +- **下载备份**:生成单个 `artex-backup-<时间>.tar.zst`,浏览器直接落盘。内容包括 PostgreSQL 全部业务表(同一快照事务导出,表间一致)、数据目录(任务工作区、会话、证据、任务归档包、MITM CA 证书、流量及其 SQLite 索引快照)和 skills 目录。不含系统运行日志(`server_logs`)和 `jwt.key`。 +- **从备份恢复**:在新机器上先正常安装并启动 ARTEX,登录后上传备份文件。服务端先完整解包并按当前数据库结构校验,通过后重启;重启时、在任何组件打开数据之前替换数据:数据库在一个事务内整体替换(失败则原数据不变,恢复包移到 `data/.restore-failed` 供排查),数据目录逐项替换,中途中断会在下次启动时继续。卡片会显示上次恢复的结果。 +- 恢复会**覆盖当前实例的全部数据**并中断正在运行的任务。登录密码、Basic Auth、LLM 与其他配置都随数据库迁移,恢复后用原实例的密码登录。 +- 新机器上已存在的同名 skill 保留本机版本(避免旧备份覆盖新版内置 skill),其余 skill 从备份装入。 +- 只能恢复到**相同或更新版本**的 ARTEX;备份含当前版本不认识的表或列时会拒绝恢复。旧路径(如任务归档包的绝对路径)会自动改写到新数据目录。 +- 恢复依赖 `start.sh` / `start.bat`(Docker 下为镜像 ENTRYPOINT)重启程序,与页面一键更新相同。 +- 备份与恢复期间均需要相当于数据量的临时磁盘空间(数据目录下)。 + ## 更新升级 > 升级只换程序、不动数据:Postgres 数据卷 `pgdata`、`./data`(jwt.key / SQLite 等)、`./skills` 都会保留。**数据库迁移无需手动执行**——`artex` 每次启动会幂等重跑 `schema.sql`(含 `ADD COLUMN` / `CREATE INDEX IF NOT EXISTS`),即“重启即迁移”。升级前仍建议先备份 `./data` 与数据库。 diff --git a/cmd/artex/main.go b/cmd/artex/main.go index 29e3005..e2e6afb 100644 --- a/cmd/artex/main.go +++ b/cmd/artex/main.go @@ -90,6 +90,13 @@ func run() int { log.Printf("[config] 配置文件: %s (不存在 — 将仅尝试环境变量 ARTEX_PG_DSN)", cfgPath) } + // 页面「恢复备份」暂存的数据必须在任何组件打开数据目录与数据库之前装好。 + skillDir := config.SkillDir() + if abs, err := filepath.Abs(skillDir); err == nil { + skillDir = abs + } + server.SetBootRestoreResult(server.ApplyPendingRestore(*dataDir, skillDir)) + sigCtx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() ctx, shutdown := shutdownContext(sigCtx) @@ -107,10 +114,6 @@ func run() int { settle := time.AfterFunc(selfupdate.SettleDelay, selfupdate.Settle) defer settle.Stop() - skillDir := config.SkillDir() - if abs, err := filepath.Abs(skillDir); err == nil { - skillDir = abs - } log.Printf("[config] skill 目录: %s", skillDir) srv := server.New(ctx, mgr, skillDir, *dataDir, config.BaseDir()) defer srv.Close() diff --git a/db/backup.go b/db/backup.go new file mode 100644 index 0000000..d5c79a6 --- /dev/null +++ b/db/backup.go @@ -0,0 +1,330 @@ +package db + +import ( + "context" + "fmt" + "io" + "strings" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/stdlib" +) + +// BackupTable describes one table's columns, in the order its COPY data uses, +// and how many rows the data holds. +type BackupTable struct { + Name string `json:"name"` + Columns []string `json:"columns"` + Rows int64 `json:"rows"` +} + +// RestoreTable is one table's COPY text-format data to load during a restore. +type RestoreTable struct { + BackupTable + Open func() (io.ReadCloser, error) +} + +// PathPrefixMove rewrites absolute paths recorded under an old data directory +// (task_archives.archive_path) so they point at the restoring instance's one. +// Separators are carried separately so a Windows backup lands on Linux intact. +type PathPrefixMove struct{ From, FromSeparator, To, ToSeparator string } + +// backupSkippedTables are neither exported nor replaced: server_logs is this +// process's own runtime log ring, not data worth migrating. +var backupSkippedTables = map[string]bool{"server_logs": true} + +// tableColumnsSQL lists every ordinary table of the application schema with its +// writable columns. Generated columns cannot be COPY'd in, so they are left out. +const tableColumnsSQL = ` +SELECT c.relname::text, array_agg(a.attname::text ORDER BY a.attnum) +FROM pg_class c +JOIN pg_namespace n ON n.oid = c.relnamespace +JOIN pg_attribute a ON a.attrelid = c.oid AND a.attnum > 0 AND NOT a.attisdropped AND a.attgenerated = '' +WHERE n.nspname = current_schema() AND c.relkind = 'r' +GROUP BY c.relname +ORDER BY c.relname` + +func ident(name string) string { return pgx.Identifier{name}.Sanitize() } + +func columnList(columns []string) string { + quoted := make([]string, len(columns)) + for i, column := range columns { + quoted[i] = ident(column) + } + return strings.Join(quoted, ", ") +} + +// withPgx runs fn on one pooled connection's native pgx handle; COPY is only +// reachable through it. +func (d *DB) withPgx(ctx context.Context, fn func(*pgx.Conn) error) error { + conn, err := d.Conn(ctx) + if err != nil { + return err + } + defer conn.Close() + return conn.Raw(func(driverConn any) error { + native, ok := driverConn.(*stdlib.Conn) + if !ok { + return fmt.Errorf("unexpected driver connection %T", driverConn) + } + return fn(native.Conn()) + }) +} + +func queryTables(ctx context.Context, tx pgx.Tx) ([]BackupTable, error) { + rows, err := tx.Query(ctx, tableColumnsSQL) + if err != nil { + return nil, err + } + defer rows.Close() + var tables []BackupTable + for rows.Next() { + var table BackupTable + if err := rows.Scan(&table.Name, &table.Columns); err != nil { + return nil, err + } + if !backupSkippedTables[table.Name] { + tables = append(tables, table) + } + } + return tables, rows.Err() +} + +// DumpTables exports every application table from a single read-only +// REPEATABLE READ snapshot, so rows written while the dump runs never produce a +// backup whose tables disagree with each other. emit is called once per table +// with a function that writes the table's COPY text data and returns its row +// count; emit records the count itself. +func (d *DB) DumpTables(ctx context.Context, emit func(BackupTable, func(io.Writer) (int64, error)) error) error { + return d.withPgx(ctx, func(conn *pgx.Conn) error { + tx, err := conn.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadOnly}) + if err != nil { + return err + } + defer tx.Rollback(ctx) //nolint:errcheck // read-only; nothing to keep + tables, err := queryTables(ctx, tx) + if err != nil { + return err + } + for _, table := range tables { + statement := fmt.Sprintf("COPY %s (%s) TO STDOUT", ident(table.Name), columnList(table.Columns)) + copyTo := func(w io.Writer) (int64, error) { + tag, err := tx.Conn().PgConn().CopyTo(ctx, w, statement) + return tag.RowsAffected(), err + } + if err := emit(table, copyTo); err != nil { + return fmt.Errorf("备份表 %s: %w", table.Name, err) + } + } + return nil + }) +} + +// CheckRestoreTables reports whether every table and column of a backup exists +// in the current schema. A backup taken by a newer ARTEX may carry columns this +// build does not know; loading it would fail halfway, so it is refused upfront. +func (d *DB) CheckRestoreTables(ctx context.Context, tables []BackupTable) error { + return d.withPgx(ctx, func(conn *pgx.Conn) error { + tx, err := conn.BeginTx(ctx, pgx.TxOptions{AccessMode: pgx.ReadOnly}) + if err != nil { + return err + } + defer tx.Rollback(ctx) //nolint:errcheck // read-only + current, err := queryTables(ctx, tx) + if err != nil { + return err + } + return checkRestoreTables(current, tables) + }) +} + +func checkRestoreTables(current, tables []BackupTable) error { + known := make(map[string]map[string]bool, len(current)) + for _, table := range current { + columns := make(map[string]bool, len(table.Columns)) + for _, column := range table.Columns { + columns[column] = true + } + known[table.Name] = columns + } + seen := make(map[string]bool, len(tables)) + for _, table := range tables { + if seen[table.Name] { + return fmt.Errorf("备份中表 %s 重复", table.Name) + } + seen[table.Name] = true + columns, ok := known[table.Name] + if !ok { + return fmt.Errorf("备份包含当前版本不存在的表 %s(备份可能来自更新的 ARTEX 版本)", table.Name) + } + for _, column := range table.Columns { + if !columns[column] { + return fmt.Errorf("备份表 %s 含当前版本不存在的列 %s(备份可能来自更新的 ARTEX 版本)", table.Name, column) + } + } + } + return nil +} + +type foreignKey struct{ table, name, definition string } + +// RestoreTables replaces the contents of every application table with the given +// data in one transaction: either the whole backup lands or nothing changes. +// Tables of the current schema that the backup does not carry end up empty; +// columns it does not carry take their defaults. +// +// Foreign keys are dropped for the load and re-added afterwards (which +// re-validates them), and user triggers are disabled so derived rows such as +// task_asset_links come from the backup instead of being regenerated on insert. +// Both only need table ownership, unlike session_replication_role, which needs +// superuser. Serial sequences are moved past the restored ids. +func (d *DB) RestoreTables(ctx context.Context, tables []RestoreTable, move PathPrefixMove) error { + return d.withPgx(ctx, func(conn *pgx.Conn) error { + tx, err := conn.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(ctx) //nolint:errcheck // no-op once committed + if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock($1)`, schemaMigrationLockKey); err != nil { + return fmt.Errorf("coordinate with schema migration: %w", err) + } + current, err := queryTables(ctx, tx) + if err != nil { + return err + } + backup := make([]BackupTable, len(tables)) + for i, table := range tables { + backup[i] = table.BackupTable + } + if err := checkRestoreTables(current, backup); err != nil { + return err + } + names := make([]string, len(current)) + for i, table := range current { + names[i] = ident(table.Name) + } + + keys, err := listForeignKeys(ctx, tx) + if err != nil { + return err + } + for _, key := range keys { + if _, err := tx.Exec(ctx, fmt.Sprintf("ALTER TABLE %s DROP CONSTRAINT %s", key.table, ident(key.name))); err != nil { + return fmt.Errorf("暂时移除外键 %s: %w", key.name, err) + } + } + for _, name := range names { + if _, err := tx.Exec(ctx, "ALTER TABLE "+name+" DISABLE TRIGGER USER"); err != nil { + return fmt.Errorf("暂停触发器 %s: %w", name, err) + } + } + if len(names) > 0 { + if _, err := tx.Exec(ctx, "TRUNCATE "+strings.Join(names, ", ")); err != nil { + return fmt.Errorf("清空当前数据: %w", err) + } + } + for _, table := range tables { + if err := copyIn(ctx, tx, table); err != nil { + return fmt.Errorf("恢复表 %s: %w", table.Name, err) + } + } + if move.From != "" && (move.From != move.To || move.FromSeparator != move.ToSeparator) { + // Runs while triggers are still off so updated_at keeps the backup's value. + from := strings.TrimSuffix(move.From, move.FromSeparator) + move.FromSeparator + to := strings.TrimSuffix(move.To, move.ToSeparator) + move.ToSeparator + if _, err := tx.Exec(ctx, ` +UPDATE task_archives SET archive_path = $2 || replace(substr(archive_path, length($1) + 1), $3, $4) +WHERE left(archive_path, length($1)) = $1`, from, to, move.FromSeparator, move.ToSeparator); err != nil { + return fmt.Errorf("改写归档包路径: %w", err) + } + } + for _, key := range keys { + if _, err := tx.Exec(ctx, fmt.Sprintf("ALTER TABLE %s ADD CONSTRAINT %s %s", key.table, ident(key.name), key.definition)); err != nil { + return fmt.Errorf("恢复外键 %s: %w", key.name, err) + } + } + for _, name := range names { + if _, err := tx.Exec(ctx, "ALTER TABLE "+name+" ENABLE TRIGGER USER"); err != nil { + return fmt.Errorf("恢复触发器 %s: %w", name, err) + } + } + if err := resetSequences(ctx, tx); err != nil { + return err + } + return tx.Commit(ctx) + }) +} + +func listForeignKeys(ctx context.Context, tx pgx.Tx) ([]foreignKey, error) { + rows, err := tx.Query(ctx, ` +SELECT conrelid::regclass::text, conname::text, pg_get_constraintdef(oid) +FROM pg_constraint +WHERE contype = 'f' AND connamespace = current_schema()::regnamespace +ORDER BY 1, 2`) + if err != nil { + return nil, err + } + defer rows.Close() + var keys []foreignKey + for rows.Next() { + var key foreignKey + if err := rows.Scan(&key.table, &key.name, &key.definition); err != nil { + return nil, err + } + keys = append(keys, key) + } + return keys, rows.Err() +} + +func copyIn(ctx context.Context, tx pgx.Tx, table RestoreTable) error { + data, err := table.Open() + if err != nil { + return err + } + defer data.Close() + statement := fmt.Sprintf("COPY %s (%s) FROM STDIN", ident(table.Name), columnList(table.Columns)) + tag, err := tx.Conn().PgConn().CopyFrom(ctx, data, statement) + if err != nil { + return err + } + if tag.RowsAffected() != table.Rows { + return fmt.Errorf("写入 %d 行,备份清单记录 %d 行", tag.RowsAffected(), table.Rows) + } + return nil +} + +// resetSequences moves every column-owned sequence past the restored maximum; +// otherwise the next insert would collide with a restored primary key. +func resetSequences(ctx context.Context, tx pgx.Tx) error { + rows, err := tx.Query(ctx, ` +SELECT c.relname::text, a.attname::text, s.seq +FROM pg_class c +JOIN pg_namespace n ON n.oid = c.relnamespace +JOIN pg_attribute a ON a.attrelid = c.oid AND a.attnum > 0 AND NOT a.attisdropped +CROSS JOIN LATERAL (SELECT pg_get_serial_sequence(format('%I.%I', n.nspname, c.relname), a.attname) AS seq) s +WHERE n.nspname = current_schema() AND c.relkind = 'r' AND s.seq IS NOT NULL`) + if err != nil { + return err + } + type serial struct{ table, column, sequence string } + var serials []serial + for rows.Next() { + var item serial + if err := rows.Scan(&item.table, &item.column, &item.sequence); err != nil { + rows.Close() + return err + } + serials = append(serials, item) + } + rows.Close() + if err := rows.Err(); err != nil { + return err + } + for _, item := range serials { + statement := fmt.Sprintf("SELECT setval($1, COALESCE((SELECT max(%s) FROM %s), 0) + 1, false)", ident(item.column), ident(item.table)) + if _, err := tx.Exec(ctx, statement, item.sequence); err != nil { + return fmt.Errorf("重置序列 %s: %w", item.sequence, err) + } + } + return nil +} diff --git a/server/server.go b/server/server.go index 7da95ad..79e5088 100644 --- a/server/server.go +++ b/server/server.go @@ -825,6 +825,11 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("POST /api/update/rollback", s.updateRollback) mux.HandleFunc("GET /api/update/stream", s.updateStream) + // 一键备份 / 恢复(迁移数据)。恢复会替换全部数据并重启,同样只对已登录管理员开放。 + mux.HandleFunc("GET /api/system/backup/status", s.backupStatus) + mux.HandleFunc("GET /api/system/backup/export", s.backupExport) + mux.HandleFunc("POST /api/system/backup/restore", s.backupRestore) + mux.HandleFunc("GET /api/tasks", s.listTasks) mux.HandleFunc("POST /api/tasks", s.createTask) mux.HandleFunc("GET /api/schedules", s.listSchedules) diff --git a/server/system_backup.go b/server/system_backup.go new file mode 100644 index 0000000..5e84328 --- /dev/null +++ b/server/system_backup.go @@ -0,0 +1,631 @@ +package server + +import ( + "archive/tar" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "log" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "time" + + pgdb "github.com/Autumn-27/artex/db" + "github.com/klauspost/compress/zstd" +) + +// 一键备份 / 恢复(迁移数据)。 +// +// 备份包是 tar.zst,结构: +// +// db/.copy PostgreSQL 各表的 COPY 文本数据(同一个快照事务内导出) +// files/ 数据目录(任务工作区、会话、流量、证据、归档包、MITM CA…) +// skills//… skill 目录 +// manifest.json 最后写入:表结构与行数、来源数据目录、顶层条目清单 +// +// 恢复分两步:上传时只解包到 /.restore-pending 并对照当前 schema 校验, +// 然后请求重启;真正的替换在下次启动、任何组件打开数据之前由 ApplyPendingRestore +// 完成——运行中的 Agent、流量代理和 SQLite 索引都还没碰数据目录,替换才安全。 +// 数据库在一个事务内整体替换,失败则保持原数据不动。 + +const ( + backupFormat = 1 + backupManifestName = "manifest.json" + backupPendingDir = ".restore-pending" + backupUploadPrefix = ".restore-upload-" + backupPreviousDir = ".restore-previous" + backupFailedDir = ".restore-failed" + backupDBDoneMarker = ".db-restored" + backupMaxFiles = 20_000_000 + backupTrafficIndex = "traffic/_index" // live SQLite; exported via SnapshotIndex +) + +// backupSkipped lists data-dir paths (slash form, relative) that hold transient +// staging state rather than data. Top-level dot entries are skipped as well. +var backupSkipped = map[string]bool{ + "traffic/_delete_staging": true, + "archives/tasks/.staging": true, + "archives/tasks/.restore": true, + "evidence/.staging": true, + "traffic/_index/.snapshot": true, +} + +type backupManifest struct { + Format int `json:"format"` + Version string `json:"version"` + CreatedAt time.Time `json:"created_at"` + DataDir string `json:"data_dir"` + Separator string `json:"separator"` + Tables []pgdb.BackupTable `json:"tables"` + Entries []string `json:"entries"` + Skills []string `json:"skills"` +} + +// backupMu serializes export and restore staging: two concurrent restores +// would race on the pending directory, and an export racing a restore upload +// is never what the operator meant. +var backupMu sync.Mutex + +// BackupRestoreResult is the outcome of the boot-time restore, reported to the +// UI after the restart that applied it. +type BackupRestoreResult struct { + Applied bool `json:"applied"` + Error string `json:"error,omitempty"` + At time.Time `json:"at"` + Source string `json:"source_version,omitempty"` + Created time.Time `json:"backup_created_at,omitempty"` +} + +var ( + bootRestoreMu sync.Mutex + bootRestore *BackupRestoreResult +) + +// SetBootRestoreResult is called once by main with ApplyPendingRestore's result. +func SetBootRestoreResult(result *BackupRestoreResult) { + bootRestoreMu.Lock() + defer bootRestoreMu.Unlock() + bootRestore = result +} + +func bootRestoreResult() *BackupRestoreResult { + bootRestoreMu.Lock() + defer bootRestoreMu.Unlock() + return bootRestore +} + +// GET /api/system/backup/status +func (s *Server) backupStatus(w http.ResponseWriter, r *http.Request) { + _, err := os.Stat(filepath.Join(s.m.dir, backupPendingDir)) + writeJSON(w, 200, map[string]any{"pending": err == nil, "last_restore": bootRestoreResult()}) +} + +// GET|HEAD /api/system/backup/export streams the backup package. Errors after +// the first byte abort the connection so a partial file never looks complete. +func (s *Server) backupExport(w http.ResponseWriter, r *http.Request) { + if s.pg(w) == nil { + return + } + if !backupMu.TryLock() { + writeProductError(w, 409, "backup_busy", "已有备份或恢复正在进行", nil) + return + } + defer backupMu.Unlock() + name := "artex-backup-" + time.Now().Format("20060102-150405") + ".tar.zst" + w.Header().Set("Content-Type", "application/zstd") + w.Header().Set("X-Content-Type-Options", "nosniff") + w.Header().Set("Cache-Control", "no-store") + setFileDisposition(w, "attachment", name) + if r.Method == http.MethodHead { + return + } + started := time.Now() + if err := s.writeBackup(r.Context(), w); err != nil { + log.Printf("[backup] 导出失败:%v", err) + panic(http.ErrAbortHandler) + } + log.Printf("[backup] 已导出备份 %s(%s)", name, time.Since(started).Round(time.Millisecond)) +} + +func (s *Server) writeBackup(ctx context.Context, out io.Writer) error { + spool, err := os.MkdirTemp(s.m.dir, ".backup-") + if err != nil { + return err + } + defer os.RemoveAll(spool) + encoder, err := zstd.NewWriter(out, zstd.WithEncoderLevel(zstd.SpeedDefault), zstd.WithEncoderCRC(true)) + if err != nil { + return err + } + tw := tar.NewWriter(encoder) + manifest := backupManifest{ + Format: backupFormat, Version: BuildVersion, CreatedAt: time.Now().UTC(), + DataDir: s.m.dir, Separator: string(filepath.Separator), Entries: []string{}, Skills: []string{}, + } + err = s.m.pg.DumpTables(ctx, func(table pgdb.BackupTable, copyTo func(io.Writer) (int64, error)) error { + path := filepath.Join(spool, "table.copy") + file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, archiveFileMode) + if err != nil { + return err + } + rows, copyErr := copyTo(file) + if err := errors.Join(copyErr, file.Close()); err != nil { + return err + } + table.Rows = rows + manifest.Tables = append(manifest.Tables, table) + return addBackupFile(tw, path, "db/"+table.Name+".copy") + }) + if err != nil { + return err + } + snapshot := "" + if tr := s.m.Traffic(); tr != nil { + snapshot = filepath.Join(spool, "index.sqlite") + if err := tr.SnapshotIndex(snapshot); err != nil { + return fmt.Errorf("快照流量索引: %w", err) + } + if err := addBackupFile(tw, snapshot, "files/"+backupTrafficIndex+"/index.sqlite"); err != nil { + return err + } + } + err = walkBackupTree(s.m.dir, func(relative string, info fs.FileInfo) (bool, error) { + if !strings.Contains(relative, "/") { + if strings.HasPrefix(relative, ".") { + return false, nil + } + manifest.Entries = append(manifest.Entries, relative) + } + if backupSkipped[relative] || (snapshot != "" && relative == backupTrafficIndex) { + return false, nil + } + return true, addBackupEntry(tw, filepath.Join(s.m.dir, filepath.FromSlash(relative)), "files/"+relative, info) + }) + if err != nil { + return err + } + if s.skillDir != "" { + err = walkBackupTree(s.skillDir, func(relative string, info fs.FileInfo) (bool, error) { + if strings.HasPrefix(relative, ".") { + return false, nil + } + if !strings.Contains(relative, "/") { + if !info.IsDir() { + return false, nil + } + manifest.Skills = append(manifest.Skills, relative) + } + return true, addBackupEntry(tw, filepath.Join(s.skillDir, filepath.FromSlash(relative)), "skills/"+relative, info) + }) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return err + } + } + raw, err := json.MarshalIndent(manifest, "", " ") + if err != nil { + return err + } + if err := tw.WriteHeader(&tar.Header{Name: backupManifestName, Mode: archiveFileMode, Size: int64(len(raw)), ModTime: manifest.CreatedAt, Typeflag: tar.TypeReg}); err != nil { + return err + } + if _, err := tw.Write(raw); err != nil { + return err + } + if err := tw.Close(); err != nil { + return err + } + return encoder.Close() +} + +// walkBackupTree visits root's descendants in lexical order with slash-form +// relative paths. visit returns false to skip an entry (and its subtree). +// Symlinks are skipped: the package format only carries files and directories. +func walkBackupTree(root string, visit func(relative string, info fs.FileInfo) (bool, error)) error { + return filepath.WalkDir(root, func(current string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + if current != root && errors.Is(walkErr, fs.ErrNotExist) { + return nil // removed while walking (an agent cleaned up) + } + return walkErr + } + relative, err := filepath.Rel(root, current) + if err != nil || relative == "." { + return err + } + if entry.Type()&fs.ModeSymlink != 0 { + log.Printf("[backup] 跳过符号链接:%s", current) + return nil + } + info, err := entry.Info() + if errors.Is(err, fs.ErrNotExist) { + return nil + } + if err != nil { + return err + } + if !info.IsDir() && !info.Mode().IsRegular() { + return nil + } + keep, err := visit(filepath.ToSlash(relative), info) + if err != nil { + return err + } + if !keep && info.IsDir() { + return filepath.SkipDir + } + return nil + }) +} + +func addBackupFile(tw *tar.Writer, path, name string) error { + info, err := os.Stat(path) + if err != nil { + return err + } + return addBackupEntry(tw, path, name, info) +} + +func addBackupEntry(tw *tar.Writer, path, name string, info fs.FileInfo) error { + header, err := tar.FileInfoHeader(info, "") + if err != nil { + return err + } + header.Name = name + header.Uname, header.Gname = "", "" + if info.IsDir() { + header.Name += "/" + return tw.WriteHeader(header) + } + input, err := os.Open(path) + if errors.Is(err, fs.ErrNotExist) { + return nil // removed between listing and opening + } + if err != nil { + return err + } + defer input.Close() + if err := tw.WriteHeader(header); err != nil { + return err + } + // Exactly the stat'ed size: a file still being appended to is cut at that + // point instead of overflowing its tar entry. + if _, err := io.CopyN(tw, input, header.Size); err != nil { + return fmt.Errorf("读取 %s: %w", path, err) + } + return nil +} + +// POST /api/system/backup/restore — body is the raw .tar.zst package. +func (s *Server) backupRestore(w http.ResponseWriter, r *http.Request) { + if s.pg(w) == nil { + return + } + if _, running := updHub.snapshot(); running { + writeProductError(w, 409, "backup_update_running", "更新正在进行中,无法恢复备份", nil) + return + } + if !backupMu.TryLock() { + writeProductError(w, 409, "backup_busy", "已有备份或恢复正在进行", nil) + return + } + defer backupMu.Unlock() + manifest, err := s.stageBackupUpload(r.Context(), r.Body) + if err != nil { + log.Printf("[backup] 恢复包校验失败:%v", err) + writeProductError(w, 400, "backup_invalid", "备份包无效:"+err.Error(), map[string]any{"message": err.Error()}) + return + } + log.Printf("[backup] 已接收备份(版本 %s,创建于 %s),即将重启以完成恢复", manifest.Version, manifest.CreatedAt.Format(time.RFC3339)) + writeJSON(w, 202, map[string]any{"ok": true, "version": manifest.Version, "created_at": manifest.CreatedAt}) + go func() { + time.Sleep(500 * time.Millisecond) + requestRestart() + }() +} + +func (s *Server) stageBackupUpload(ctx context.Context, body io.Reader) (*backupManifest, error) { + upload, err := os.MkdirTemp(s.m.dir, backupUploadPrefix) + if err != nil { + return nil, err + } + staged := false + defer func() { + if !staged { + _ = os.RemoveAll(upload) + } + }() + if err := extractTarZst(body, upload, backupMaxFiles); err != nil { + return nil, fmt.Errorf("解包失败: %w", err) + } + manifest, err := readBackupManifest(upload) + if err != nil { + return nil, err + } + if err := s.m.pg.CheckRestoreTables(ctx, manifest.Tables); err != nil { + return nil, err + } + pending := filepath.Join(s.m.dir, backupPendingDir) + if err := os.RemoveAll(pending); err != nil { + return nil, err + } + if err := os.Rename(upload, pending); err != nil { + return nil, err + } + staged = true + return manifest, nil +} + +// readBackupManifest loads and validates the manifest of an extracted package: +// every name it lists must be a single local path element that actually exists, +// so a crafted manifest cannot point the restore outside the package. +func readBackupManifest(root string) (*backupManifest, error) { + raw, err := os.ReadFile(filepath.Join(root, backupManifestName)) + if errors.Is(err, fs.ErrNotExist) { + return nil, errors.New("缺少 manifest.json(不是 ARTEX 备份包,或文件不完整)") + } + if err != nil { + return nil, err + } + var manifest backupManifest + if err := json.Unmarshal(raw, &manifest); err != nil { + return nil, fmt.Errorf("manifest.json 格式错误: %w", err) + } + if manifest.Format < 1 || manifest.Format > backupFormat { + return nil, fmt.Errorf("不支持的备份格式版本 %d(备份可能来自更新的 ARTEX 版本)", manifest.Format) + } + if len(manifest.Tables) == 0 { + return nil, errors.New("备份中没有数据库表") + } + for _, table := range manifest.Tables { + if !singleElement(table.Name) || len(table.Columns) == 0 { + return nil, fmt.Errorf("备份表名无效: %q", table.Name) + } + if _, err := os.Stat(filepath.Join(root, "db", table.Name+".copy")); err != nil { + return nil, fmt.Errorf("缺少表 %s 的数据", table.Name) + } + } + for _, name := range manifest.Entries { + if !singleElement(name) || strings.HasPrefix(name, ".") { + return nil, fmt.Errorf("备份条目名无效: %q", name) + } + } + for _, name := range manifest.Skills { + if !singleElement(name) || strings.HasPrefix(name, ".") { + return nil, fmt.Errorf("备份 skill 名无效: %q", name) + } + } + return &manifest, nil +} + +func singleElement(name string) bool { + return name != "" && filepath.IsLocal(name) && !strings.ContainsAny(name, `/\`) +} + +// ApplyPendingRestore installs a backup staged by the restore endpoint. main +// calls it before anything opens the data directory or the database. It returns +// nil when nothing was pending. +// +// Steps are resumable: the database transaction is marked done inside the +// pending directory, and each top-level entry is moved aside and replaced one +// rename at a time, so a crash at any point is finished by the next boot. A +// database failure leaves current data untouched and parks the package in +// .restore-failed for inspection. +func ApplyPendingRestore(dataDir, skillDir string) *BackupRestoreResult { + if abs, err := filepath.Abs(dataDir); err == nil { + dataDir = abs + } + pending := filepath.Join(dataDir, backupPendingDir) + previous := filepath.Join(dataDir, backupPreviousDir) + if _, err := os.Stat(pending); err != nil { + if errors.Is(err, fs.ErrNotExist) { + _ = os.RemoveAll(previous) // leftover of a restore that crashed during cleanup + } + cleanBackupUploads(dataDir) + return nil + } + cleanBackupUploads(dataDir) + result := &BackupRestoreResult{At: time.Now()} + manifest, err := readBackupManifest(pending) + if err == nil { + result.Source, result.Created = manifest.Version, manifest.CreatedAt + log.Printf("[backup] 开始恢复备份(版本 %s,创建于 %s)", manifest.Version, manifest.CreatedAt.Format(time.RFC3339)) + err = restoreBackupDatabase(pending, dataDir, manifest) + } + if err != nil { + result.Error = err.Error() + log.Printf("[backup] 恢复失败,当前数据保持不变:%v", err) + failed := filepath.Join(dataDir, backupFailedDir) + _ = os.RemoveAll(failed) + if renameErr := os.Rename(pending, failed); renameErr != nil { + log.Printf("[backup] 无法移走失败的恢复包,删除之:%v", renameErr) + _ = os.RemoveAll(pending) + } + return result + } + if err := installBackupFiles(pending, dataDir, skillDir, manifest); err != nil { + // The database is already replaced; files are retried on the next boot. + result.Error = "数据库已恢复,但文件替换失败(下次启动会继续):" + err.Error() + log.Printf("[backup] %s", result.Error) + return result + } + if err := os.RemoveAll(previous); err != nil { + log.Printf("[backup] 清理旧数据失败:%v", err) + } + if err := os.RemoveAll(pending); err != nil { + log.Printf("[backup] 清理恢复包失败:%v", err) + } + result.Applied = true + log.Printf("[backup] 备份恢复完成") + return result +} + +func cleanBackupUploads(dataDir string) { + entries, _ := os.ReadDir(dataDir) + for _, entry := range entries { + if strings.HasPrefix(entry.Name(), backupUploadPrefix) || strings.HasPrefix(entry.Name(), ".backup-") { + _ = os.RemoveAll(filepath.Join(dataDir, entry.Name())) + } + } +} + +func restoreBackupDatabase(pending, dataDir string, manifest *backupManifest) error { + marker := filepath.Join(pending, backupDBDoneMarker) + if _, err := os.Stat(marker); err == nil { + return nil + } + dsn, _, err := pgdb.DSN() + if err != nil { + return err + } + pg, err := pgdb.Open(dsn) + if err != nil { + return err + } + defer pg.Close() + // Created outside schema.sql; a fresh target database does not have them yet. + if err := pg.EnsureLLMRecordsTable(); err != nil { + return err + } + if err := pg.EnsureLLMUsageTable(); err != nil { + return err + } + tables := make([]pgdb.RestoreTable, len(manifest.Tables)) + for i, table := range manifest.Tables { + path := filepath.Join(pending, "db", table.Name+".copy") + tables[i] = pgdb.RestoreTable{BackupTable: table, Open: func() (io.ReadCloser, error) { return os.Open(path) }} + } + move := pgdb.PathPrefixMove{From: manifest.DataDir, FromSeparator: manifest.Separator, To: dataDir, ToSeparator: string(filepath.Separator)} + if err := pg.RestoreTables(context.Background(), tables, move); err != nil { + return err + } + // A machine-specific interpreter path from the old host would break custom + // tools; dropping it lets startup re-detect the local one. + if value, ok, _ := pg.GetSetting(settingPythonInterp); ok && strings.TrimSpace(value) != "" { + if _, err := os.Stat(strings.TrimSpace(value)); err != nil { + if _, err := pg.Exec(`DELETE FROM settings WHERE key=$1`, settingPythonInterp); err != nil { + log.Printf("[backup] 清除失效的 python 解释器配置失败:%v", err) + } + } + } + return writeArchiveJournal(marker, map[string]any{"at": time.Now()}) +} + +func installBackupFiles(pending, dataDir, skillDir string, manifest *backupManifest) error { + previous := filepath.Join(dataDir, backupPreviousDir) + if err := os.MkdirAll(previous, archiveDirMode); err != nil { + return err + } + keep := make(map[string]bool, len(manifest.Entries)) + for _, name := range manifest.Entries { + keep[name] = true + } + entries, err := os.ReadDir(dataDir) + if err != nil { + return err + } + for _, entry := range entries { + name := entry.Name() + if strings.HasPrefix(name, ".") || keep[name] { + continue + } + if err := moveAside(filepath.Join(dataDir, name), filepath.Join(previous, name)); err != nil { + return err + } + } + for _, name := range manifest.Entries { + source := filepath.Join(pending, "files", name) + if _, err := os.Lstat(source); errors.Is(err, fs.ErrNotExist) { + continue // already installed by an interrupted earlier boot + } else if err != nil { + return err + } + destination := filepath.Join(dataDir, name) + if _, err := os.Lstat(destination); err == nil { + if err := moveAside(destination, filepath.Join(previous, name)); err != nil { + return err + } + } + if err := os.Rename(source, destination); err != nil { + return err + } + } + if skillDir == "" { + return nil + } + if err := os.MkdirAll(skillDir, 0o755); err != nil { + return err + } + // Skills already present here are kept: built-in skills ship with this + // build and must not be downgraded by an older backup's copy. + for _, name := range manifest.Skills { + source := filepath.Join(pending, "skills", name) + destination := filepath.Join(skillDir, name) + if _, err := os.Lstat(source); err != nil { + continue + } + if _, err := os.Lstat(destination); err == nil { + continue + } + if err := os.Rename(source, destination); err != nil { + if err := copyBackupTree(source, destination); err != nil { + return fmt.Errorf("安装 skill %s: %w", name, err) + } + } + } + return nil +} + +func moveAside(source, destination string) error { + if _, err := os.Lstat(destination); err == nil { + if err := os.RemoveAll(destination); err != nil { + return err + } + } + return os.Rename(source, destination) +} + +// copyBackupTree is the fallback when the skill directory sits on another +// filesystem than the data directory (a Docker bind mount), where rename fails. +func copyBackupTree(source, destination string) error { + temporary := destination + ".restore-tmp" + _ = os.RemoveAll(temporary) + err := filepath.WalkDir(source, func(current string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + relative, err := filepath.Rel(source, current) + if err != nil { + return err + } + target := filepath.Join(temporary, relative) + if entry.IsDir() { + return os.MkdirAll(target, 0o755) + } + info, err := entry.Info() + if err != nil { + return err + } + input, err := os.Open(current) + if err != nil { + return err + } + defer input.Close() + output, err := os.OpenFile(target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, info.Mode().Perm()) + if err != nil { + return err + } + _, copyErr := io.Copy(output, input) + return errors.Join(copyErr, output.Close()) + }) + if err != nil { + _ = os.RemoveAll(temporary) + return err + } + return os.Rename(temporary, destination) +} diff --git a/server/system_backup_test.go b/server/system_backup_test.go new file mode 100644 index 0000000..bc0b916 --- /dev/null +++ b/server/system_backup_test.go @@ -0,0 +1,211 @@ +package server + +import ( + "bytes" + "context" + "database/sql" + "fmt" + "io" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + "time" + + pgdb "github.com/Autumn-27/artex/db" +) + +// scratchDSN points the configured server at a fresh database that is dropped +// when the test ends, so restores never truncate the shared development DB. +func scratchDSN(t *testing.T, base, name string) string { + t.Helper() + u, err := url.Parse(base) + if err != nil { + t.Skipf("unparseable DSN: %v", err) + } + u.Path = "/" + name + admin := *u + admin.Path = "/postgres" + t.Cleanup(func() { + conn, err := sql.Open("pgx", admin.String()) + if err != nil { + return + } + defer conn.Close() + _, _ = conn.Exec(`DROP DATABASE IF EXISTS "` + name + `" WITH (FORCE)`) + }) + return u.String() +} + +func writeTestFile(t *testing.T, path, body string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } +} + +func TestBackupExportRestoreRoundTrip(t *testing.T) { + base, _, err := pgdb.DSN() + if err != nil { + t.Skipf("no database config (%v)", err) + } + suffix := fmt.Sprintf("%d", time.Now().UnixNano()) + sourceDSN := scratchDSN(t, base, "artex_backup_src_"+suffix) + targetDSN := scratchDSN(t, base, "artex_backup_dst_"+suffix) + + sourceDir, sourceSkills := t.TempDir(), t.TempDir() + t.Setenv("ARTEX_PG_DSN", sourceDSN) + source, err := NewManager(sourceDir, "127.0.0.1:0") + if err != nil { + t.Skipf("postgres unavailable (%v)", err) + } + task, err := source.pg.CreateTask("backup source", "migrate me", nil, 0, 0) + if err != nil { + t.Fatal(err) + } + if err := source.pg.SetSetting("backup.marker", "from-source"); err != nil { + t.Fatal(err) + } + archivePath := filepath.Join(sourceDir, "archives", "tasks", "task-1.tar.zst") + if _, err := source.pg.Exec(`INSERT INTO task_archives(task_id, state, archive_path) VALUES ($1, 'ready', $2)`, task.ID, archivePath); err != nil { + t.Fatal(err) + } + writeTestFile(t, filepath.Join(sourceDir, "tasks", fmt.Sprint(task.ID), "note.txt"), "task workspace") + writeTestFile(t, archivePath, "archive package") + writeTestFile(t, filepath.Join(sourceDir, ".restore-failed", "junk"), "never exported") + writeTestFile(t, filepath.Join(sourceSkills, "custom-skill", "SKILL.md"), "---\nname: custom-skill\n---\n") + writeTestFile(t, filepath.Join(sourceSkills, "shipped", "SKILL.md"), "old copy") + + exporter := &Server{m: source, skillDir: sourceSkills} + exported := httptest.NewRecorder() + exporter.backupExport(exported, httptest.NewRequest("GET", "/api/system/backup/export", nil)) + source.Close() + if exported.Code != 200 || exported.Body.Len() == 0 { + t.Fatalf("export status=%d size=%d", exported.Code, exported.Body.Len()) + } + + targetDir, targetSkills := t.TempDir(), t.TempDir() + t.Setenv("ARTEX_PG_DSN", targetDSN) + target, err := NewManager(targetDir, "") + if err != nil { + t.Fatal(err) + } + if err := target.pg.SetSetting("backup.stale", "replaced"); err != nil { + t.Fatal(err) + } + writeTestFile(t, filepath.Join(targetDir, "tasks", "999", "old.txt"), "stale workspace") + writeTestFile(t, filepath.Join(targetSkills, "shipped", "SKILL.md"), "new copy") + + restorer := &Server{m: target, skillDir: targetSkills} + upload := httptest.NewRecorder() + restorer.backupRestore(upload, httptest.NewRequest("POST", "/api/system/backup/restore", bytes.NewReader(exported.Body.Bytes()))) + target.Close() + if upload.Code != 202 { + t.Fatalf("restore upload status=%d body=%s", upload.Code, upload.Body.String()) + } + + result := ApplyPendingRestore(targetDir, targetSkills) + if result == nil || !result.Applied { + t.Fatalf("restore not applied: %+v", result) + } + if again := ApplyPendingRestore(targetDir, targetSkills); again != nil { + t.Fatalf("pending restore applied twice: %+v", again) + } + + restored, err := pgdb.Open(targetDSN) + if err != nil { + t.Fatal(err) + } + defer restored.Close() + if got, err := restored.GetTask(task.ID); err != nil || got == nil || got.Goal != "migrate me" { + t.Fatalf("restored task=%+v err=%v", got, err) + } + if value, ok, _ := restored.GetSetting("backup.marker"); !ok || value != "from-source" { + t.Fatalf("restored setting=%q ok=%v", value, ok) + } + if _, ok, _ := restored.GetSetting("backup.stale"); ok { + t.Fatal("target-only setting survived the restore") + } + var restoredArchivePath string + if err := restored.QueryRow(`SELECT archive_path FROM task_archives WHERE task_id=$1`, task.ID).Scan(&restoredArchivePath); err != nil { + t.Fatal(err) + } + if want := filepath.Join(targetDir, "archives", "tasks", "task-1.tar.zst"); restoredArchivePath != want { + t.Fatalf("archive path=%q want %q", restoredArchivePath, want) + } + next, err := restored.CreateTask("after restore", "fresh id", nil, 0, 0) + if err != nil { + t.Fatalf("insert after restore (sequence not reset?): %v", err) + } + if next.ID <= task.ID { + t.Fatalf("new task id %d does not follow restored id %d", next.ID, task.ID) + } + + for path, want := range map[string]string{ + filepath.Join(targetDir, "tasks", fmt.Sprint(task.ID), "note.txt"): "task workspace", + filepath.Join(targetDir, "archives", "tasks", "task-1.tar.zst"): "archive package", + filepath.Join(targetSkills, "custom-skill", "SKILL.md"): "---\nname: custom-skill\n---\n", + filepath.Join(targetSkills, "shipped", "SKILL.md"): "new copy", + } { + if got, err := os.ReadFile(path); err != nil || string(got) != want { + t.Fatalf("%s = %q, %v; want %q", path, got, err, want) + } + } + if _, err := os.Stat(filepath.Join(targetDir, "traffic", "_index", "index.sqlite")); err != nil { + t.Fatalf("traffic index snapshot not restored: %v", err) + } + for _, gone := range []string{ + filepath.Join(targetDir, "tasks", "999"), + filepath.Join(targetDir, ".restore-failed"), + filepath.Join(targetDir, backupPendingDir), + filepath.Join(targetDir, backupPreviousDir), + } { + if _, err := os.Stat(gone); !os.IsNotExist(err) { + t.Fatalf("%s should not exist after restore: %v", gone, err) + } + } + + // A load that fails midway must leave the database exactly as it was. + broken := []pgdb.RestoreTable{{ + BackupTable: pgdb.BackupTable{Name: "settings", Columns: []string{"key", "value"}, Rows: 2}, + Open: func() (io.ReadCloser, error) { return io.NopCloser(strings.NewReader("only\tone\n")), nil }, + }} + if err := restored.RestoreTables(context.Background(), broken, pgdb.PathPrefixMove{}); err == nil { + t.Fatal("row-count mismatch was accepted") + } + if got, err := restored.GetTask(task.ID); err != nil || got == nil { + t.Fatalf("failed restore lost data: task=%+v err=%v", got, err) + } +} + +func TestBackupRestoreRejectsForeignPackage(t *testing.T) { + if _, _, err := pgdb.DSN(); err != nil { + t.Skipf("no database config (%v)", err) + } + dataDir := t.TempDir() + m, err := NewManager(dataDir, "") + if err != nil { + t.Skipf("postgres unavailable (%v)", err) + } + defer m.Close() + s := &Server{m: m} + w := httptest.NewRecorder() + s.backupRestore(w, httptest.NewRequest("POST", "/api/system/backup/restore", strings.NewReader("not a backup"))) + if w.Code != 400 { + t.Fatalf("status=%d body=%s", w.Code, w.Body.String()) + } + if _, err := os.Stat(filepath.Join(dataDir, backupPendingDir)); !os.IsNotExist(err) { + t.Fatalf("invalid upload left a pending restore: %v", err) + } + entries, _ := os.ReadDir(dataDir) + for _, entry := range entries { + if strings.HasPrefix(entry.Name(), backupUploadPrefix) { + t.Fatalf("invalid upload left staging dir %s", entry.Name()) + } + } +} diff --git a/server/task_archive_package.go b/server/task_archive_package.go index bfbee15..6c6be34 100644 --- a/server/task_archive_package.go +++ b/server/task_archive_package.go @@ -656,7 +656,13 @@ func extractTaskArchivePackage(path, expectedSHA, destination string) error { return err } } - decoder, err := zstd.NewReader(file) + return extractTarZst(file, destination, maxArchiveFiles) +} + +// extractTarZst unpacks a zstd-compressed tar stream of plain files and +// directories into destination, refusing entries that would escape it. +func extractTarZst(input io.Reader, destination string, maxFiles int) error { + decoder, err := zstd.NewReader(input) if err != nil { return err } @@ -676,17 +682,17 @@ func extractTaskArchivePackage(path, expectedSHA, destination string) error { return err } entries++ - if entries > maxArchiveFiles || header.Size < 0 || total+header.Size > maxArchiveBytes { - return errors.New("task archive exceeds extraction safety limits") + if entries > maxFiles || header.Size < 0 || total+header.Size > maxArchiveBytes { + return errors.New("archive exceeds extraction safety limits") } total += header.Size clean := filepath.Clean(filepath.FromSlash(header.Name)) if clean == "." || filepath.IsAbs(clean) || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) { - return fmt.Errorf("unsafe task archive path %q", header.Name) + return fmt.Errorf("unsafe archive path %q", header.Name) } target := filepath.Join(destination, clean) if relative, err := filepath.Rel(destination, target); err != nil || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { - return fmt.Errorf("unsafe task archive target %q", header.Name) + return fmt.Errorf("unsafe archive target %q", header.Name) } switch header.Typeflag { case tar.TypeDir: @@ -707,7 +713,7 @@ func extractTaskArchivePackage(path, expectedSHA, destination string) error { return err } default: - return fmt.Errorf("unsupported task archive entry type %d", header.Typeflag) + return fmt.Errorf("unsupported archive entry type %d", header.Typeflag) } } return nil diff --git a/traffic/traffic.go b/traffic/traffic.go index d9baf42..7e4fbcc 100644 --- a/traffic/traffic.go +++ b/traffic/traffic.go @@ -378,6 +378,14 @@ func (t *Traffic) CACertPath() string { return filepath.Join(t.dir, "_ca", "mitmproxy-ca-cert.pem") } +// SnapshotIndex writes a transactionally consistent copy of the live SQLite +// index to dest (which must not exist). Copying index.sqlite directly would miss +// pages still in the WAL and could capture a half-applied write. +func (t *Traffic) SnapshotIndex(dest string) error { + _, err := t.db.Exec(`VACUUM INTO ?`, dest) + return err +} + // Start runs the proxy (blocking); run in a goroutine. func (t *Traffic) Start() error { return t.proxy.Start() } diff --git a/web/src/app/(main)/system/settings/_components/backup-card.tsx b/web/src/app/(main)/system/settings/_components/backup-card.tsx new file mode 100644 index 0000000..bd87a1d --- /dev/null +++ b/web/src/app/(main)/system/settings/_components/backup-card.tsx @@ -0,0 +1,175 @@ +"use client"; + +import * as React from "react"; + +import { ArchiveRestoreIcon, DatabaseBackupIcon, DownloadIcon } from "lucide-react"; +import { toast } from "sonner"; + +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from "@/components/ui/alert-dialog"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; +import { Progress } from "@/components/ui/progress"; +import { localizedError } from "@/i18n/errors"; +import { useLiveTranslation } from "@/i18n/use-live-translation"; +import { api, type BackupStatus } from "@/lib/api"; + +/** 恢复要先完成数据库整体替换和文件换装,再走一次正常启动;数据量大时比升级慢得多。 */ +const RESTORE_TIMEOUT_MS = 15 * 60_000; + +export function BackupCard() { + const { t, liveT } = useLiveTranslation(); + const [status, setStatus] = React.useState(null); + const [file, setFile] = React.useState(null); + const [uploading, setUploading] = React.useState(false); + const [restarting, setRestarting] = React.useState(false); + const inputRef = React.useRef(null); + + React.useEffect(() => { + api + .backupStatus() + .then(setStatus) + .catch(() => setStatus(null)); + }, []); + + const download = async () => { + try { + await api.downloadBackup(); + toast.success(t("settings.backup.downloadStarted")); + } catch (e) { + toast.error(t("settings.backup.downloadFailedPrefix") + localizedError(e, t)); + } + }; + + // 重启期间接口连不上是预期的;以"出现了晚于本次上传的恢复结果"为准判断新进程已就绪。 + const waitForRestore = React.useCallback( + async (since: number) => { + setRestarting(true); + const deadline = Date.now() + RESTORE_TIMEOUT_MS; + while (Date.now() < deadline) { + const tick = Promise.withResolvers(); + setTimeout(tick.resolve, 3000); + await tick.promise; + try { + const next = await api.backupStatus(); + const at = next.last_restore ? Date.parse(next.last_restore.at) : 0; + if (at >= since) { + window.location.reload(); + return; + } + } catch { + // 继续轮询。 + } + } + setRestarting(false); + toast.error(liveT("settings.backup.restoreTimeout")); + }, + [liveT], + ); + + const restore = async () => { + if (!file) return; + const selected = file; + setFile(null); + setUploading(true); + const since = Date.now() - 60_000; // 容忍浏览器与服务器之间的时钟偏差 + try { + await api.restoreBackup(selected); + setUploading(false); + await waitForRestore(since); + } catch (e) { + setUploading(false); + toast.error(t("settings.backup.restoreFailedPrefix") + localizedError(e, t)); + } + }; + + const busy = uploading || restarting; + const last = status?.last_restore; + + return ( + + + + + {t("settings.backup.title")} + + {t("settings.backup.help")} + + + {last?.applied ? ( +

+ {t("settings.backup.lastRestoreOk", { + time: new Date(last.at).toLocaleString(), + version: last.source_version || "-", + })} +

+ ) : null} + {last && !last.applied && last.error ? ( +

+ {t("settings.backup.lastRestoreFailed", { message: last.error })} +

+ ) : null} + {status?.pending && !busy ? ( +

{t("settings.backup.pending")}

+ ) : null} + + {busy ? ( +
+ +

+ {restarting ? t("settings.backup.restarting") : t("settings.backup.uploading")} +

+
+ ) : null} + +
+ + + { + setFile(e.target.files?.[0] ?? null); + e.target.value = ""; + }} + /> +
+ +

{t("settings.backup.note")}

+
+ + (open ? null : setFile(null))}> + + + {t("settings.backup.confirmTitle")} + + {t("settings.backup.confirmBody", { name: file?.name ?? "" })} + + + + {t("settings.backup.cancel")} + + {t("settings.backup.confirm")} + + + + +
+ ); +} diff --git a/web/src/app/(main)/system/settings/page.tsx b/web/src/app/(main)/system/settings/page.tsx index 2eb10de..73fdc0c 100644 --- a/web/src/app/(main)/system/settings/page.tsx +++ b/web/src/app/(main)/system/settings/page.tsx @@ -30,6 +30,7 @@ import { CHAT_SEND_MODE_OPTIONS, type ChatSendMode, setChatSendMode, useChatSend import type { EgressReport, LLMProfile, ProxyPoolEntry, Settings, WorkerPool } from "@/lib/types"; import { useEgressIp } from "../../_components/egress-ip-banner"; +import { BackupCard } from "./_components/backup-card"; import { BasicAuthCard } from "./_components/basic-auth-card"; import { McpServerCard } from "./_components/mcp-server-card"; import { UpdateCard } from "./_components/update-card"; @@ -766,6 +767,7 @@ export default function SystemSettingsPage() {
{isSettingVisible("system", "settings.keywords.update") ? : null} + {isSettingVisible("system", "settings.keywords.backup") ? : null} {isSettingVisible("system", "settings.keywords.basicAuth") ? : null} {isSettingVisible("system", "settings.keywords.mcpServer") ? : null} diff --git a/web/src/i18n/error-messages.ts b/web/src/i18n/error-messages.ts index 6078f14..31b8b46 100644 --- a/web/src/i18n/error-messages.ts +++ b/web/src/i18n/error-messages.ts @@ -3,6 +3,16 @@ export const errorMessages = { "情报数据响应不完整,请刷新后重试", "The intelligence response is incomplete. Refresh to try again.", ], + "error.backup_busy": ["已有备份或恢复正在进行", "A backup or restore is already in progress"], + "error.backup_update_running": [ + "更新正在进行中,无法恢复备份", + "An update is in progress. The backup cannot be restored now.", + ], + "error.backup_invalid": ["备份包无效:{{message}}", "Invalid backup package: {{message}}"], + "error.demo_backup_disabled": [ + "演示模式不支持备份与恢复,请在实际部署中使用", + "Backup and restore are not available in the demo. Use them in a deployed instance.", + ], "error.basic_auth_required": ["请先完成 HTTP Basic Auth 验证", "Complete HTTP Basic Auth verification first"], "error.auth_session_failed": ["认证会话生成失败", "Unable to create an authentication session"], "error.basic_auth_username_invalid": [ diff --git a/web/src/i18n/i18n.test.cjs b/web/src/i18n/i18n.test.cjs index e7b5cd4..f707bed 100644 --- a/web/src/i18n/i18n.test.cjs +++ b/web/src/i18n/i18n.test.cjs @@ -138,6 +138,7 @@ test("migrated pages reference existing keys and contain no hardcoded Chinese di "components/approval-execution-focus.tsx", "hooks/use-side-questions.ts", "app/(main)/dashboard/page.tsx", + "app/(main)/system/settings/_components/backup-card.tsx", "app/(main)/system/settings/_components/basic-auth-card.tsx", "app/(main)/system/settings/_components/update-card.tsx", "app/(main)/_components/egress-ip-banner.tsx", diff --git a/web/src/i18n/settings-messages.ts b/web/src/i18n/settings-messages.ts index 5a5d95d..15f8da5 100644 --- a/web/src/i18n/settings-messages.ts +++ b/web/src/i18n/settings-messages.ts @@ -110,6 +110,52 @@ export const settingsMessages = { "settings.matchingSettings": ["匹配“{{query}}”的设置项", "Settings matching “{{query}}”"], "settings.clearSearch": ["清除搜索", "Clear search"], "settings.keywords.update": ["更新 版本 升级", "update version upgrade"], + "settings.keywords.backup": [ + "备份 恢复 迁移 导出 导入 数据 一键备份", + "backup restore migrate migration export import data", + ], + "settings.backup.title": ["备份与迁移", "Backup and migration"], + "settings.backup.help": [ + "一键导出全部数据(数据库、任务工作区、会话、流量、证据、归档包、MITM 证书和 skills),在新机器上恢复即可完成迁移。", + "Export all data (database, task workspaces, sessions, traffic, evidence, archive packages, MITM certificate and skills) in one file, then restore it on another machine to migrate.", + ], + "settings.backup.download": ["下载备份", "Download backup"], + "settings.backup.downloadStarted": [ + "备份已开始生成,浏览器会直接下载,数据较多时需要一些时间", + "The backup is being generated and will download directly. Large installations take a while.", + ], + "settings.backup.downloadFailedPrefix": ["下载备份失败:", "Could not download backup: "], + "settings.backup.restore": ["从备份恢复", "Restore from backup"], + "settings.backup.uploading": ["正在上传并校验备份…", "Uploading and verifying the backup…"], + "settings.backup.restoreFailedPrefix": ["恢复失败:", "Restore failed: "], + "settings.backup.confirmTitle": ["用备份替换当前全部数据?", "Replace all current data with this backup?"], + "settings.backup.confirmBody": [ + "当前实例的任务、资产、漏洞、配置、文件和流量都会被「{{name}}」中的内容替换,且无法撤销。上传校验通过后程序会重启,正在运行的任务会被中断。建议先下载一份当前数据的备份。", + "Tasks, assets, findings, settings, files and traffic on this instance will be replaced by the contents of “{{name}}”. This cannot be undone. After the upload is verified, ARTEX restarts and running tasks are interrupted. Download a backup of the current data first.", + ], + "settings.backup.cancel": ["取消", "Cancel"], + "settings.backup.confirm": ["替换并重启", "Replace and restart"], + "settings.backup.restarting": [ + "备份已校验,正在重启并恢复数据,请稍候(页面会自动刷新)…", + "Backup verified. Restarting to restore data. Please wait; the page will reload automatically…", + ], + "settings.backup.restoreTimeout": [ + "等待服务重启超时。请检查后端日志,或确认 artex 是通过 start.sh / start.bat 启动的。", + "Timed out waiting for the service to restart. Check the backend logs and confirm ARTEX was started with start.sh / start.bat.", + ], + "settings.backup.lastRestoreOk": [ + "已于 {{time}} 从备份恢复(备份版本 {{version}})。", + "Restored from a backup at {{time}} (backup version {{version}}).", + ], + "settings.backup.lastRestoreFailed": ["上次恢复失败:{{message}}", "The last restore failed: {{message}}"], + "settings.backup.pending": [ + "有一份已上传的备份等待重启后恢复。", + "An uploaded backup will be restored at the next restart.", + ], + "settings.backup.note": [ + "恢复需要相同或更新版本的 ARTEX。登录密码、Basic Auth 与 LLM 配置随数据库一起迁移,恢复后请使用原实例的密码登录;新机器上已有的同名 skill 保留本机版本。", + "Restoring requires the same or a newer ARTEX version. The sign-in password, Basic Auth and LLM settings move with the database, so sign in with the original instance's password afterwards. Skills that already exist on the new machine keep the local copy.", + ], "settings.keywords.basicAuth": [ "基础认证 登录 用户名 密码", "basic auth authentication sign-in login username password", diff --git a/web/src/lib/api.ts b/web/src/lib/api.ts index b37b0f7..e487a9b 100644 --- a/web/src/lib/api.ts +++ b/web/src/lib/api.ts @@ -163,6 +163,19 @@ export async function apiFetch(input: string, init?: RequestInit): Promise http<{ version: string }>("/version", { cache: "no-store" }), systemMetrics: () => get("/system/metrics"), + // ---- 一键备份 / 恢复 ---- + backupStatus: () => get("/system/backup/status"), + // 备份包可能有数 GB:与任务文件下载一样先 HEAD 校验登录,再交给浏览器直接落盘(cookie 鉴权),不读进内存。 + downloadBackup: async () => { + if (MOCK) throw new ApiError("演示模式不支持备份与恢复", "demo_backup_disabled"); + const token = getToken(); + const response = await apiFetch("/api/system/backup/export", { + method: "HEAD", + headers: token ? { Authorization: `Bearer ${token}` } : {}, + }); + if (!response.ok) + throw new ApiError( + `HEAD /system/backup/export: ${response.status}`, + response.status === 409 ? "backup_busy" : undefined, + ); + const a = document.createElement("a"); + a.href = "/api/system/backup/export"; + document.body.appendChild(a); + a.click(); + a.remove(); + }, + restoreBackup: (file: File) => + http<{ ok: boolean; version: string; created_at: string }>("/system/backup/restore", { + method: "POST", + body: file, + headers: { "Content-Type": "application/octet-stream" }, + }), + // ---- auth ---- authStatus: () => get<{ initialized: boolean }>("/auth/status"), login: (username: string, password: string) => post<{ token: string }>("/auth/login", { username, password }), diff --git a/web/src/lib/mock/handler.ts b/web/src/lib/mock/handler.ts index 08467dc..55060f9 100644 --- a/web/src/lib/mock/handler.ts +++ b/web/src/lib/mock/handler.ts @@ -2470,6 +2470,9 @@ function route(m: string, path: string, seg: string[], q: URLSearchParams, b: Re throw Object.assign(new Error("演示模式不支持修改 HTTP Basic Auth,请在实际部署中配置"), { code: "demo_basic_auth_disabled", }); + if (path === "/system/backup/status") return { pending: false, last_restore: null }; + if (path === "/system/backup/restore") + throw Object.assign(new Error("演示模式不支持备份与恢复,请在实际部署中使用"), { code: "demo_backup_disabled" }); if (path === "/settings/web-search/test") return { ok: true, count: 5, backend: D.settings.web_search_backend }; if (path === "/settings/proxy/test") return {