Files
ARTEX-chixuweihu/server/system_backup.go
T
ji ruo c6f7c29b71 feat: 一键备份与迁移 (#160)
- 系统配置新增「备份与迁移」:下载单个 tar.zst 备份(全部业务表的一致快照、数据目录、流量索引快照、skills)
- 上传恢复:先解包并按当前 schema 校验,通过后重启;启动时数据库单事务整体替换,文件逐项替换且可续做
- 归档包绝对路径改写到新数据目录;本机已有同名 skill 保留本机版本
- 抽出 extractTarZst 供流式解包复用
2026-11-19 23:34:02 +08:00

632 lines
20 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package server
import (
"archive/tar"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"log"
"net/http"
"os"
"path/filepath"
"strings"
"sync"
"time"
pgdb "github.com/Autumn-27/artex/db"
"github.com/klauspost/compress/zstd"
)
// 一键备份 / 恢复(迁移数据)。
//
// 备份包是 tar.zst,结构:
//
// db/<table>.copy PostgreSQL 各表的 COPY 文本数据(同一个快照事务内导出)
// files/<rel> 数据目录(任务工作区、会话、流量、证据、归档包、MITM CA…)
// skills/<name>/… skill 目录
// manifest.json 最后写入:表结构与行数、来源数据目录、顶层条目清单
//
// 恢复分两步:上传时只解包到 <data>/.restore-pending 并对照当前 schema 校验,
// 然后请求重启;真正的替换在下次启动、任何组件打开数据之前由 ApplyPendingRestore
// 完成——运行中的 Agent、流量代理和 SQLite 索引都还没碰数据目录,替换才安全。
// 数据库在一个事务内整体替换,失败则保持原数据不动。
const (
backupFormat = 1
backupManifestName = "manifest.json"
backupPendingDir = ".restore-pending"
backupUploadPrefix = ".restore-upload-"
backupPreviousDir = ".restore-previous"
backupFailedDir = ".restore-failed"
backupDBDoneMarker = ".db-restored"
backupMaxFiles = 20_000_000
backupTrafficIndex = "traffic/_index" // live SQLite; exported via SnapshotIndex
)
// backupSkipped lists data-dir paths (slash form, relative) that hold transient
// staging state rather than data. Top-level dot entries are skipped as well.
var backupSkipped = map[string]bool{
"traffic/_delete_staging": true,
"archives/tasks/.staging": true,
"archives/tasks/.restore": true,
"evidence/.staging": true,
"traffic/_index/.snapshot": true,
}
type backupManifest struct {
Format int `json:"format"`
Version string `json:"version"`
CreatedAt time.Time `json:"created_at"`
DataDir string `json:"data_dir"`
Separator string `json:"separator"`
Tables []pgdb.BackupTable `json:"tables"`
Entries []string `json:"entries"`
Skills []string `json:"skills"`
}
// backupMu serializes export and restore staging: two concurrent restores
// would race on the pending directory, and an export racing a restore upload
// is never what the operator meant.
var backupMu sync.Mutex
// BackupRestoreResult is the outcome of the boot-time restore, reported to the
// UI after the restart that applied it.
type BackupRestoreResult struct {
Applied bool `json:"applied"`
Error string `json:"error,omitempty"`
At time.Time `json:"at"`
Source string `json:"source_version,omitempty"`
Created time.Time `json:"backup_created_at,omitempty"`
}
var (
bootRestoreMu sync.Mutex
bootRestore *BackupRestoreResult
)
// SetBootRestoreResult is called once by main with ApplyPendingRestore's result.
func SetBootRestoreResult(result *BackupRestoreResult) {
bootRestoreMu.Lock()
defer bootRestoreMu.Unlock()
bootRestore = result
}
func bootRestoreResult() *BackupRestoreResult {
bootRestoreMu.Lock()
defer bootRestoreMu.Unlock()
return bootRestore
}
// GET /api/system/backup/status
func (s *Server) backupStatus(w http.ResponseWriter, r *http.Request) {
_, err := os.Stat(filepath.Join(s.m.dir, backupPendingDir))
writeJSON(w, 200, map[string]any{"pending": err == nil, "last_restore": bootRestoreResult()})
}
// GET|HEAD /api/system/backup/export streams the backup package. Errors after
// the first byte abort the connection so a partial file never looks complete.
func (s *Server) backupExport(w http.ResponseWriter, r *http.Request) {
if s.pg(w) == nil {
return
}
if !backupMu.TryLock() {
writeProductError(w, 409, "backup_busy", "已有备份或恢复正在进行", nil)
return
}
defer backupMu.Unlock()
name := "artex-backup-" + time.Now().Format("20060102-150405") + ".tar.zst"
w.Header().Set("Content-Type", "application/zstd")
w.Header().Set("X-Content-Type-Options", "nosniff")
w.Header().Set("Cache-Control", "no-store")
setFileDisposition(w, "attachment", name)
if r.Method == http.MethodHead {
return
}
started := time.Now()
if err := s.writeBackup(r.Context(), w); err != nil {
log.Printf("[backup] 导出失败:%v", err)
panic(http.ErrAbortHandler)
}
log.Printf("[backup] 已导出备份 %s(%s)", name, time.Since(started).Round(time.Millisecond))
}
func (s *Server) writeBackup(ctx context.Context, out io.Writer) error {
spool, err := os.MkdirTemp(s.m.dir, ".backup-")
if err != nil {
return err
}
defer os.RemoveAll(spool)
encoder, err := zstd.NewWriter(out, zstd.WithEncoderLevel(zstd.SpeedDefault), zstd.WithEncoderCRC(true))
if err != nil {
return err
}
tw := tar.NewWriter(encoder)
manifest := backupManifest{
Format: backupFormat, Version: BuildVersion, CreatedAt: time.Now().UTC(),
DataDir: s.m.dir, Separator: string(filepath.Separator), Entries: []string{}, Skills: []string{},
}
err = s.m.pg.DumpTables(ctx, func(table pgdb.BackupTable, copyTo func(io.Writer) (int64, error)) error {
path := filepath.Join(spool, "table.copy")
file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, archiveFileMode)
if err != nil {
return err
}
rows, copyErr := copyTo(file)
if err := errors.Join(copyErr, file.Close()); err != nil {
return err
}
table.Rows = rows
manifest.Tables = append(manifest.Tables, table)
return addBackupFile(tw, path, "db/"+table.Name+".copy")
})
if err != nil {
return err
}
snapshot := ""
if tr := s.m.Traffic(); tr != nil {
snapshot = filepath.Join(spool, "index.sqlite")
if err := tr.SnapshotIndex(snapshot); err != nil {
return fmt.Errorf("快照流量索引: %w", err)
}
if err := addBackupFile(tw, snapshot, "files/"+backupTrafficIndex+"/index.sqlite"); err != nil {
return err
}
}
err = walkBackupTree(s.m.dir, func(relative string, info fs.FileInfo) (bool, error) {
if !strings.Contains(relative, "/") {
if strings.HasPrefix(relative, ".") {
return false, nil
}
manifest.Entries = append(manifest.Entries, relative)
}
if backupSkipped[relative] || (snapshot != "" && relative == backupTrafficIndex) {
return false, nil
}
return true, addBackupEntry(tw, filepath.Join(s.m.dir, filepath.FromSlash(relative)), "files/"+relative, info)
})
if err != nil {
return err
}
if s.skillDir != "" {
err = walkBackupTree(s.skillDir, func(relative string, info fs.FileInfo) (bool, error) {
if strings.HasPrefix(relative, ".") {
return false, nil
}
if !strings.Contains(relative, "/") {
if !info.IsDir() {
return false, nil
}
manifest.Skills = append(manifest.Skills, relative)
}
return true, addBackupEntry(tw, filepath.Join(s.skillDir, filepath.FromSlash(relative)), "skills/"+relative, info)
})
if err != nil && !errors.Is(err, fs.ErrNotExist) {
return err
}
}
raw, err := json.MarshalIndent(manifest, "", " ")
if err != nil {
return err
}
if err := tw.WriteHeader(&tar.Header{Name: backupManifestName, Mode: archiveFileMode, Size: int64(len(raw)), ModTime: manifest.CreatedAt, Typeflag: tar.TypeReg}); err != nil {
return err
}
if _, err := tw.Write(raw); err != nil {
return err
}
if err := tw.Close(); err != nil {
return err
}
return encoder.Close()
}
// walkBackupTree visits root's descendants in lexical order with slash-form
// relative paths. visit returns false to skip an entry (and its subtree).
// Symlinks are skipped: the package format only carries files and directories.
func walkBackupTree(root string, visit func(relative string, info fs.FileInfo) (bool, error)) error {
return filepath.WalkDir(root, func(current string, entry fs.DirEntry, walkErr error) error {
if walkErr != nil {
if current != root && errors.Is(walkErr, fs.ErrNotExist) {
return nil // removed while walking (an agent cleaned up)
}
return walkErr
}
relative, err := filepath.Rel(root, current)
if err != nil || relative == "." {
return err
}
if entry.Type()&fs.ModeSymlink != 0 {
log.Printf("[backup] 跳过符号链接:%s", current)
return nil
}
info, err := entry.Info()
if errors.Is(err, fs.ErrNotExist) {
return nil
}
if err != nil {
return err
}
if !info.IsDir() && !info.Mode().IsRegular() {
return nil
}
keep, err := visit(filepath.ToSlash(relative), info)
if err != nil {
return err
}
if !keep && info.IsDir() {
return filepath.SkipDir
}
return nil
})
}
func addBackupFile(tw *tar.Writer, path, name string) error {
info, err := os.Stat(path)
if err != nil {
return err
}
return addBackupEntry(tw, path, name, info)
}
func addBackupEntry(tw *tar.Writer, path, name string, info fs.FileInfo) error {
header, err := tar.FileInfoHeader(info, "")
if err != nil {
return err
}
header.Name = name
header.Uname, header.Gname = "", ""
if info.IsDir() {
header.Name += "/"
return tw.WriteHeader(header)
}
input, err := os.Open(path)
if errors.Is(err, fs.ErrNotExist) {
return nil // removed between listing and opening
}
if err != nil {
return err
}
defer input.Close()
if err := tw.WriteHeader(header); err != nil {
return err
}
// Exactly the stat'ed size: a file still being appended to is cut at that
// point instead of overflowing its tar entry.
if _, err := io.CopyN(tw, input, header.Size); err != nil {
return fmt.Errorf("读取 %s: %w", path, err)
}
return nil
}
// POST /api/system/backup/restore — body is the raw .tar.zst package.
func (s *Server) backupRestore(w http.ResponseWriter, r *http.Request) {
if s.pg(w) == nil {
return
}
if _, running := updHub.snapshot(); running {
writeProductError(w, 409, "backup_update_running", "更新正在进行中,无法恢复备份", nil)
return
}
if !backupMu.TryLock() {
writeProductError(w, 409, "backup_busy", "已有备份或恢复正在进行", nil)
return
}
defer backupMu.Unlock()
manifest, err := s.stageBackupUpload(r.Context(), r.Body)
if err != nil {
log.Printf("[backup] 恢复包校验失败:%v", err)
writeProductError(w, 400, "backup_invalid", "备份包无效:"+err.Error(), map[string]any{"message": err.Error()})
return
}
log.Printf("[backup] 已接收备份(版本 %s,创建于 %s),即将重启以完成恢复", manifest.Version, manifest.CreatedAt.Format(time.RFC3339))
writeJSON(w, 202, map[string]any{"ok": true, "version": manifest.Version, "created_at": manifest.CreatedAt})
go func() {
time.Sleep(500 * time.Millisecond)
requestRestart()
}()
}
func (s *Server) stageBackupUpload(ctx context.Context, body io.Reader) (*backupManifest, error) {
upload, err := os.MkdirTemp(s.m.dir, backupUploadPrefix)
if err != nil {
return nil, err
}
staged := false
defer func() {
if !staged {
_ = os.RemoveAll(upload)
}
}()
if err := extractTarZst(body, upload, backupMaxFiles); err != nil {
return nil, fmt.Errorf("解包失败: %w", err)
}
manifest, err := readBackupManifest(upload)
if err != nil {
return nil, err
}
if err := s.m.pg.CheckRestoreTables(ctx, manifest.Tables); err != nil {
return nil, err
}
pending := filepath.Join(s.m.dir, backupPendingDir)
if err := os.RemoveAll(pending); err != nil {
return nil, err
}
if err := os.Rename(upload, pending); err != nil {
return nil, err
}
staged = true
return manifest, nil
}
// readBackupManifest loads and validates the manifest of an extracted package:
// every name it lists must be a single local path element that actually exists,
// so a crafted manifest cannot point the restore outside the package.
func readBackupManifest(root string) (*backupManifest, error) {
raw, err := os.ReadFile(filepath.Join(root, backupManifestName))
if errors.Is(err, fs.ErrNotExist) {
return nil, errors.New("缺少 manifest.json(不是 ARTEX 备份包,或文件不完整)")
}
if err != nil {
return nil, err
}
var manifest backupManifest
if err := json.Unmarshal(raw, &manifest); err != nil {
return nil, fmt.Errorf("manifest.json 格式错误: %w", err)
}
if manifest.Format < 1 || manifest.Format > backupFormat {
return nil, fmt.Errorf("不支持的备份格式版本 %d(备份可能来自更新的 ARTEX 版本)", manifest.Format)
}
if len(manifest.Tables) == 0 {
return nil, errors.New("备份中没有数据库表")
}
for _, table := range manifest.Tables {
if !singleElement(table.Name) || len(table.Columns) == 0 {
return nil, fmt.Errorf("备份表名无效: %q", table.Name)
}
if _, err := os.Stat(filepath.Join(root, "db", table.Name+".copy")); err != nil {
return nil, fmt.Errorf("缺少表 %s 的数据", table.Name)
}
}
for _, name := range manifest.Entries {
if !singleElement(name) || strings.HasPrefix(name, ".") {
return nil, fmt.Errorf("备份条目名无效: %q", name)
}
}
for _, name := range manifest.Skills {
if !singleElement(name) || strings.HasPrefix(name, ".") {
return nil, fmt.Errorf("备份 skill 名无效: %q", name)
}
}
return &manifest, nil
}
func singleElement(name string) bool {
return name != "" && filepath.IsLocal(name) && !strings.ContainsAny(name, `/\`)
}
// ApplyPendingRestore installs a backup staged by the restore endpoint. main
// calls it before anything opens the data directory or the database. It returns
// nil when nothing was pending.
//
// Steps are resumable: the database transaction is marked done inside the
// pending directory, and each top-level entry is moved aside and replaced one
// rename at a time, so a crash at any point is finished by the next boot. A
// database failure leaves current data untouched and parks the package in
// .restore-failed for inspection.
func ApplyPendingRestore(dataDir, skillDir string) *BackupRestoreResult {
if abs, err := filepath.Abs(dataDir); err == nil {
dataDir = abs
}
pending := filepath.Join(dataDir, backupPendingDir)
previous := filepath.Join(dataDir, backupPreviousDir)
if _, err := os.Stat(pending); err != nil {
if errors.Is(err, fs.ErrNotExist) {
_ = os.RemoveAll(previous) // leftover of a restore that crashed during cleanup
}
cleanBackupUploads(dataDir)
return nil
}
cleanBackupUploads(dataDir)
result := &BackupRestoreResult{At: time.Now()}
manifest, err := readBackupManifest(pending)
if err == nil {
result.Source, result.Created = manifest.Version, manifest.CreatedAt
log.Printf("[backup] 开始恢复备份(版本 %s,创建于 %s)", manifest.Version, manifest.CreatedAt.Format(time.RFC3339))
err = restoreBackupDatabase(pending, dataDir, manifest)
}
if err != nil {
result.Error = err.Error()
log.Printf("[backup] 恢复失败,当前数据保持不变:%v", err)
failed := filepath.Join(dataDir, backupFailedDir)
_ = os.RemoveAll(failed)
if renameErr := os.Rename(pending, failed); renameErr != nil {
log.Printf("[backup] 无法移走失败的恢复包,删除之:%v", renameErr)
_ = os.RemoveAll(pending)
}
return result
}
if err := installBackupFiles(pending, dataDir, skillDir, manifest); err != nil {
// The database is already replaced; files are retried on the next boot.
result.Error = "数据库已恢复,但文件替换失败(下次启动会继续):" + err.Error()
log.Printf("[backup] %s", result.Error)
return result
}
if err := os.RemoveAll(previous); err != nil {
log.Printf("[backup] 清理旧数据失败:%v", err)
}
if err := os.RemoveAll(pending); err != nil {
log.Printf("[backup] 清理恢复包失败:%v", err)
}
result.Applied = true
log.Printf("[backup] 备份恢复完成")
return result
}
func cleanBackupUploads(dataDir string) {
entries, _ := os.ReadDir(dataDir)
for _, entry := range entries {
if strings.HasPrefix(entry.Name(), backupUploadPrefix) || strings.HasPrefix(entry.Name(), ".backup-") {
_ = os.RemoveAll(filepath.Join(dataDir, entry.Name()))
}
}
}
func restoreBackupDatabase(pending, dataDir string, manifest *backupManifest) error {
marker := filepath.Join(pending, backupDBDoneMarker)
if _, err := os.Stat(marker); err == nil {
return nil
}
dsn, _, err := pgdb.DSN()
if err != nil {
return err
}
pg, err := pgdb.Open(dsn)
if err != nil {
return err
}
defer pg.Close()
// Created outside schema.sql; a fresh target database does not have them yet.
if err := pg.EnsureLLMRecordsTable(); err != nil {
return err
}
if err := pg.EnsureLLMUsageTable(); err != nil {
return err
}
tables := make([]pgdb.RestoreTable, len(manifest.Tables))
for i, table := range manifest.Tables {
path := filepath.Join(pending, "db", table.Name+".copy")
tables[i] = pgdb.RestoreTable{BackupTable: table, Open: func() (io.ReadCloser, error) { return os.Open(path) }}
}
move := pgdb.PathPrefixMove{From: manifest.DataDir, FromSeparator: manifest.Separator, To: dataDir, ToSeparator: string(filepath.Separator)}
if err := pg.RestoreTables(context.Background(), tables, move); err != nil {
return err
}
// A machine-specific interpreter path from the old host would break custom
// tools; dropping it lets startup re-detect the local one.
if value, ok, _ := pg.GetSetting(settingPythonInterp); ok && strings.TrimSpace(value) != "" {
if _, err := os.Stat(strings.TrimSpace(value)); err != nil {
if _, err := pg.Exec(`DELETE FROM settings WHERE key=$1`, settingPythonInterp); err != nil {
log.Printf("[backup] 清除失效的 python 解释器配置失败:%v", err)
}
}
}
return writeArchiveJournal(marker, map[string]any{"at": time.Now()})
}
func installBackupFiles(pending, dataDir, skillDir string, manifest *backupManifest) error {
previous := filepath.Join(dataDir, backupPreviousDir)
if err := os.MkdirAll(previous, archiveDirMode); err != nil {
return err
}
keep := make(map[string]bool, len(manifest.Entries))
for _, name := range manifest.Entries {
keep[name] = true
}
entries, err := os.ReadDir(dataDir)
if err != nil {
return err
}
for _, entry := range entries {
name := entry.Name()
if strings.HasPrefix(name, ".") || keep[name] {
continue
}
if err := moveAside(filepath.Join(dataDir, name), filepath.Join(previous, name)); err != nil {
return err
}
}
for _, name := range manifest.Entries {
source := filepath.Join(pending, "files", name)
if _, err := os.Lstat(source); errors.Is(err, fs.ErrNotExist) {
continue // already installed by an interrupted earlier boot
} else if err != nil {
return err
}
destination := filepath.Join(dataDir, name)
if _, err := os.Lstat(destination); err == nil {
if err := moveAside(destination, filepath.Join(previous, name)); err != nil {
return err
}
}
if err := os.Rename(source, destination); err != nil {
return err
}
}
if skillDir == "" {
return nil
}
if err := os.MkdirAll(skillDir, 0o755); err != nil {
return err
}
// Skills already present here are kept: built-in skills ship with this
// build and must not be downgraded by an older backup's copy.
for _, name := range manifest.Skills {
source := filepath.Join(pending, "skills", name)
destination := filepath.Join(skillDir, name)
if _, err := os.Lstat(source); err != nil {
continue
}
if _, err := os.Lstat(destination); err == nil {
continue
}
if err := os.Rename(source, destination); err != nil {
if err := copyBackupTree(source, destination); err != nil {
return fmt.Errorf("安装 skill %s: %w", name, err)
}
}
}
return nil
}
func moveAside(source, destination string) error {
if _, err := os.Lstat(destination); err == nil {
if err := os.RemoveAll(destination); err != nil {
return err
}
}
return os.Rename(source, destination)
}
// copyBackupTree is the fallback when the skill directory sits on another
// filesystem than the data directory (a Docker bind mount), where rename fails.
func copyBackupTree(source, destination string) error {
temporary := destination + ".restore-tmp"
_ = os.RemoveAll(temporary)
err := filepath.WalkDir(source, func(current string, entry fs.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
relative, err := filepath.Rel(source, current)
if err != nil {
return err
}
target := filepath.Join(temporary, relative)
if entry.IsDir() {
return os.MkdirAll(target, 0o755)
}
info, err := entry.Info()
if err != nil {
return err
}
input, err := os.Open(current)
if err != nil {
return err
}
defer input.Close()
output, err := os.OpenFile(target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, info.Mode().Perm())
if err != nil {
return err
}
_, copyErr := io.Copy(output, input)
return errors.Join(copyErr, output.Close())
})
if err != nil {
_ = os.RemoveAll(temporary)
return err
}
return os.Rename(temporary, destination)
}