From 2f5f7c0227d4172fcd2926ea8f9d8a6d8d3c47ed Mon Sep 17 00:00:00 2001 From: Autumn-27 <53844290+Autumn-27@users.noreply.github.com> Date: Wed, 30 Sep 2026 02:50:49 -0400 Subject: [PATCH] =?UTF-8?q?feat(traffic):=20=E6=B5=81=E9=87=8F=E5=88=97?= =?UTF-8?q?=E8=A1=A8=E6=96=B0=E5=A2=9E=E3=80=8C=E6=B8=85=E7=A9=BA=E5=85=A8?= =?UTF-8?q?=E9=83=A8=E3=80=8D=EF=BC=8C=E5=B9=B6=E5=80=9F=E7=A9=BA=E5=BA=93?= =?UTF-8?q?=E6=97=B6=E6=9C=BA=E5=8E=8B=E5=AE=9E=E7=B4=A2=E5=BC=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 一次删除全部流量记录，忽略当前筛选条件。与按 host 删除的两条路径相比多做 两件事： - 清理索引已不再记录的历史 host 目录。按 host 删除只能从索引反查目录， 行已消失的孤立目录会一直留着；「清空全部」既然语义是不留残余，就直接 扫顶层目录（跳过 _index/_blobs/_ca/_delete_staging）。 - 收尾做一次全量压实（optimize + VACUUM + wal_checkpoint(TRUNCATE)），把 索引占用的磁盘空间还给系统，并把实际释放量返回给前端提示。 VACUUM 的耗时与临时空间只和它保留的内容成正比，所以空库是做全量压实唯一 免费的时机——也正因如此，这一步顺带成为已有实例的转换途径：auto_vacuum 只能通过 VACUUM 生效，清空一次之后旧库即转为增量回收模式，此后日常按 host 删除就能自行回收空间（TestDeleteAllConvertsLegacyIndex 覆盖）。 已绑定到漏洞的流量证据保存在独立的证据库中，不受影响——绑定时已拷出独立 副本，正是为了让热流量可以随时清空而不带走证据。确认弹窗里明确写了这点。 前端把「清空全部」做成 outline 样式而非第二个 destructive 按钮：它忽略一切 筛选条件，不该看起来和「删除该目标」只差一次误点；无流量时禁用。 测试：新增 TestDeleteAllPurgesAndCompacts（清空后索引、正文、blob 引用与 孤立目录全清，空间回收到 1/8 以下，清空后仍可继续录制）与 TestDeleteAllConvertsLegacyIndex（旧库经清空后转为增量回收，随后普通删除 即可回收）。 --- CHANGELOG.md | 6 +- server/server.go | 20 ++++ traffic/reclaim_test.go | 101 +++++++++++++++++ traffic/traffic.go | 110 +++++++++++++++++++ web/src/app/(main)/function/traffic/page.tsx | 98 +++++++++++++---- web/src/lib/api.ts | 3 + 6 files changed, 314 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2a1c2d..ee007d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,11 +6,15 @@ ### 流量 +#### 新增的功能 + +- **流量列表新增「清空全部」**：一次删除全部流量记录，忽略当前筛选条件，并额外清理索引已不再记录的历史 host 目录，不留残余。清空后顺带做一次全量压实（`optimize` + `VACUUM` + `wal_checkpoint(TRUNCATE)`），把索引占用的磁盘空间还给系统，完成后提示实际释放了多少。已绑定到漏洞的流量证据保存在独立的证据库中，不受影响。空库上的 `VACUUM` 几乎没有成本，因此这同时是**已有实例把索引库转成增量回收模式的途径**——清空一次之后，日常按 host 删除就能自行回收空间了。 + #### 修复的问题 - **修复删除流量后磁盘空间不被释放**：删除只让数据不可见，空间一直留在索引文件里。SQLite 删行仅把页挂到 freelist，而索引库建库时没有启用 `auto_vacuum`，文件永不收缩；同时 `ex_fts` 是 `contentless_delete` 全文索引，`DELETE` 只写 tombstone 而不回收原 postings，不合并就永久累积——删流量反而让索引变大。由于 256KB 以下的正文全部内联在这个库里，加上 trigram 索引约为正文体积的 2 倍，抓包量大的实例会为早已删掉的流量长期占用数倍磁盘（实测抓 6MB 正文 → 索引 16MB，删光后仍是 16MB）。现在新建索引库直接启用 `auto_vacuum=incremental`，每次删除提交后在后台分块执行「全文索引增量合并 + `incremental_vacuum` + `wal_checkpoint(TRUNCATE)`」，逐步把空间还给操作系统（同一场景删除后回落到 104KB）。回收分块进行并在块间释放写锁，不阻塞流量录制；进程退出时立即让出，剩余工作在下次删除时续做。 -> 升级说明：`auto_vacuum` 只能在建库时设定，因此**已有实例的索引库仍是旧模式**，`incremental_vacuum` 在其上是空操作——启动时会打印一行提示。这类库升级后 tombstone 的持续增长已经止住（全文索引合并照常执行），但已占用的体积需要后续提供的「存储压缩」入口一次性回收。 +> 升级说明：`auto_vacuum` 只能在建库时设定，因此**已有实例的索引库仍是旧模式**，`incremental_vacuum` 在其上是空操作——启动时会打印一行提示。这类库升级后 tombstone 的持续增长已经止住（全文索引合并照常执行），而已占用的体积用流量列表的「清空全部」回收一次即可——那一步会把库转成增量回收模式，此后日常删除自行生效。 ### LLM diff --git a/server/server.go b/server/server.go index e14414c..d30204e 100644 --- a/server/server.go +++ b/server/server.go @@ -784,6 +784,7 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("GET /api/traffic/hosts", s.getTrafficHosts) mux.HandleFunc("DELETE /api/traffic", s.deleteTraffic) mux.HandleFunc("DELETE /api/traffic/hosts", s.deleteTrafficHosts) + mux.HandleFunc("DELETE /api/traffic/all", s.deleteAllTraffic) mux.HandleFunc("GET /api/traffic/exchange", s.getTrafficExchange) mux.HandleFunc("GET /api/traffic/blob", s.getTrafficBlob) mux.HandleFunc("GET /api/commands", s.pgListCommands) @@ -3185,6 +3186,25 @@ func (s *Server) deleteTrafficHosts(w http.ResponseWriter, r *http.Request) { writeJSON(w, 200, map[string]any{"deleted": n}) } +// deleteAllTraffic purges every recorded exchange, then compacts the index so +// the space is actually returned to the filesystem — an emptied index is the one +// moment a full rewrite is cheap. Evidence already bound to a finding lives in +// the evidence store and is deliberately left alone. Returns the number of +// exchanges deleted and the bytes of index reclaimed. +func (s *Server) deleteAllTraffic(w http.ResponseWriter, r *http.Request) { + tr := s.m.Traffic() + if tr == nil { + writeErr(w, 404, "traffic disabled") + return + } + n, reclaimed, err := tr.DeleteAll() + if err != nil { + writeErr(w, 500, err.Error()) + return + } + writeJSON(w, 200, map[string]any{"deleted": n, "reclaimed": reclaimed}) +} + // getTrafficExchange returns the full raw request/response of one exchange, // read on demand from the traffic tree (bodies are not in the paged list). func (s *Server) getTrafficExchange(w http.ResponseWriter, r *http.Request) { diff --git a/traffic/reclaim_test.go b/traffic/reclaim_test.go index ec22920..ee8438d 100644 --- a/traffic/reclaim_test.go +++ b/traffic/reclaim_test.go @@ -154,3 +154,104 @@ func TestReclaimOnLegacyIndexIsHarmless(t *testing.T) { t.Fatal("旧库居然回收了空闲页，说明测试没有真的构造出旧库") } } + +// TestDeleteAllPurgesAndCompacts covers the page's clear-everything action: it +// must leave nothing behind — including host directories the index no longer +// knows about — and it must hand the index space back, since an emptied index is +// the one moment a full rewrite is cheap. +func TestDeleteAllPurgesAndCompacts(t *testing.T) { + tr, dir := openTraffic(t) + bulkRecord(tr, "a.example.com", 10, 200*1024) + bulkRecord(tr, "b.example.com", 10, 200*1024) + // A text body so the full-text index has real content, and a spilled one so a + // blob exists to collect. + tr.record(newFlow("c.example.com", "GET", "/page", nil, []byte(strings.Repeat("secret-token ", 500)))) + tr.record(newFlow("c.example.com", "GET", "/big", nil, + []byte(strings.Repeat("B", maxInlineBody+1024)), withRespType("application/sql"))) + // An orphaned legacy directory: no index row points at it, so only a + // clear-everything should take it. + orphan := filepath.Join(dir, "orphan.example.com") + if err := os.MkdirAll(orphan, 0o755); err != nil { + t.Fatal(err) + } + grown := tr.indexBytes() + if grown < 5<<20 { + t.Fatalf("索引只有 %d 字节，样本不足", grown) + } + + deleted, reclaimed, err := tr.DeleteAll() + if err != nil { + t.Fatalf("DeleteAll: %v", err) + } + if deleted != 22 { + t.Fatalf("deleted=%d，应为 22", deleted) + } + tr.reaping.Wait() + + if reclaimed < grown/2 { + t.Fatalf("只回收了 %d 字节（删除前索引 %d）", reclaimed, grown) + } + if after := tr.indexBytes(); after > grown/8 { + t.Fatalf("清空后索引仍占 %d 字节（删除前 %d）", after, grown) + } + for _, q := range []string{ + `SELECT COUNT(*) FROM exchanges`, + `SELECT COUNT(*) FROM exchange_bodies`, + `SELECT COUNT(*) FROM blob_refs`, + } { + var c int + if err := tr.DB().QueryRow(q).Scan(&c); err != nil { + t.Fatal(err) + } + if c != 0 { + t.Fatalf("%s = %d，应为 0", q, c) + } + } + if _, err := os.Stat(orphan); !os.IsNotExist(err) { + t.Fatalf("孤立的历史 host 目录未被清理：%v", err) + } + // Recording must keep working against the freshly rewritten file. + tr.record(newFlow("d.example.com", "GET", "/after", nil, []byte("清空后仍可录制"))) + if n, err := tr.Count(); err != nil || n != 1 { + t.Fatalf("清空后 Count=(%d,%v)，应为 (1,nil)", n, err) + } +} + +// TestDeleteAllConvertsLegacyIndex is why the purge compacts rather than just +// deleting: auto_vacuum cannot be switched on after the fact except through a +// VACUUM, and an emptied index is the cheapest place to pay for one. After this, +// ordinary deletions reclaim space on their own. +func TestDeleteAllConvertsLegacyIndex(t *testing.T) { + dir := t.TempDir() + old := openLegacyIndex(t, dir) + if err := old.Close(); err != nil { + t.Fatal(err) + } + tr, err := Open(dir, "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { tr.Close() }) + if tr.incrementalVacuum { + t.Fatal("旧库不应报告已启用增量回收") + } + + bulkRecord(tr, "legacy.example.com", 10, 200*1024) + if _, _, err := tr.DeleteAll(); err != nil { + t.Fatalf("DeleteAll: %v", err) + } + if !tr.incrementalVacuum { + t.Fatal("清空后旧库未被转换为增量回收模式") + } + + // The converted database now reclaims on an ordinary host deletion. + bulkRecord(tr, "again.example.com", 10, 200*1024) + grown := tr.indexBytes() + if _, err := tr.DeleteHostsExact([]string{"again.example.com"}); err != nil { + t.Fatal(err) + } + tr.reaping.Wait() + if after := tr.indexBytes(); after > grown/4 { + t.Fatalf("转换后普通删除仍未回收：%d 字节（删除前 %d）", after, grown) + } +} diff --git a/traffic/traffic.go b/traffic/traffic.go index 587e5d9..ecd846b 100644 --- a/traffic/traffic.go +++ b/traffic/traffic.go @@ -1066,6 +1066,116 @@ func (t *Traffic) DeleteHost(host string) (int64, error) { return n, nil } +// DeleteAll removes every recorded exchange — the page's clear-everything +// action. It differs from the host-scoped deletions in two ways. It sweeps host +// directories without consulting the index, because "clear everything" should +// leave nothing behind and a directory whose rows are already gone would +// otherwise survive. And it ends with a full compaction: VACUUM costs what it +// keeps, so an emptied index is the one moment it is free, and it is also the +// only way to switch on auto_vacuum for a database created without it. +// +// Evidence already bound to a finding is untouched. Those bodies were copied +// into the evidence store when they were bound, precisely so that disposable +// traffic could be cleared without taking proof with it. +// +// Returns the number of exchanges deleted and how many bytes of index the +// compaction handed back. +func (t *Traffic) DeleteAll() (deleted int64, reclaimed int64, err error) { + t.wmu.Lock() + defer t.wmu.Unlock() + before := t.indexBytes() + trees, err := t.allHostTrees() + if err != nil { + return 0, 0, err + } + tx, err := t.db.Begin() + if err != nil { + return 0, 0, err + } + defer tx.Rollback() //nolint:errcheck // no-op once committed + if deleted, err = t.deleteWhere(tx, `1=1`); err != nil { + return 0, 0, err + } + // Staged before the commit so a filesystem failure can still abort the whole + // deletion, exactly as in DeleteHost. + stageDir, moves, err := t.stageTrees(trees) + if err != nil { + return 0, 0, errors.Join(err, restoreTrees(stageDir, moves)) + } + if err := tx.Commit(); err != nil { + return 0, 0, errors.Join(fmt.Errorf("提交流量索引删除: %w", err), restoreTrees(stageDir, moves)) + } + t.reapStage(stageDir) + if err := t.gcBlobs(); err != nil { + return deleted, 0, err + } + if err := t.compactIndex(); err != nil { + // The deletion is already durable; compaction is disk space, not + // correctness, so it must not turn a completed purge into a failed one. + log.Printf("[traffic] 压实索引失败：%v", err) + return deleted, 0, nil + } + return deleted, before - t.indexBytes(), nil +} + +// allHostTrees lists every host directory left by the pre-SQLite layout. Unlike +// hostTrees it does not go through the index, so it also finds directories whose +// rows are already gone. Underscore-prefixed entries are the store's own +// (_index, _blobs, _ca, _delete_staging) and are never hosts. +func (t *Traffic) allHostTrees() ([]string, error) { + entries, err := os.ReadDir(t.dir) + if err != nil { + return nil, err + } + var dirs []string + for _, e := range entries { + if e.IsDir() && !strings.HasPrefix(e.Name(), "_") { + dirs = append(dirs, filepath.Join(t.dir, e.Name())) + } + } + return dirs, nil +} + +// compactIndex rewrites the index into a fresh file, which is what actually +// returns its pages to the filesystem. VACUUM's runtime and temporary space +// scale with the content it keeps, so this is only reached right after DeleteAll +// has emptied the index — never as a routine step. It doubles as the conversion +// path for a database created before auto_vacuum=incremental: that pragma only +// takes effect through the VACUUM that follows it, and both must run on the same +// connection. Callers must hold wmu. +func (t *Traffic) compactIndex() error { + ctx := context.Background() + conn, err := t.db.Conn(ctx) + if err != nil { + return err + } + defer conn.Close() + if t.fts { + // A full merge, not the bounded one reclaim uses: with the index emptied + // there is nothing left to merge, so this only discards the tombstones. + if _, err := conn.ExecContext(ctx, `INSERT INTO ex_fts(ex_fts) VALUES('optimize')`); err != nil { + return fmt.Errorf("合并全文索引: %w", err) + } + } + if _, err := conn.ExecContext(ctx, `PRAGMA auto_vacuum=incremental`); err != nil { + return err + } + if _, err := conn.ExecContext(ctx, `VACUUM`); err != nil { + return fmt.Errorf("压实索引: %w", err) + } + var mode int + if err := conn.QueryRowContext(ctx, `PRAGMA auto_vacuum`).Scan(&mode); err != nil { + return err + } + // A legacy database has just been converted, so later deletions can reclaim + // space on their own instead of waiting for another purge. + t.incrementalVacuum = mode == autoVacuumIncremental + if _, err := conn.ExecContext(ctx, `PRAGMA wal_checkpoint(TRUNCATE)`); err != nil { + return fmt.Errorf("截断 WAL: %w", err) + } + return nil +} + // deleteWhere removes every trace of the exchanges matching the condition: // full-text index rows, bodies, blob references, and finally the index rows. // Order matters — each sub-select reads exchanges, so that table is emptied last. diff --git a/web/src/app/(main)/function/traffic/page.tsx b/web/src/app/(main)/function/traffic/page.tsx index 42e389a..18e5a78 100644 --- a/web/src/app/(main)/function/traffic/page.tsx +++ b/web/src/app/(main)/function/traffic/page.tsx @@ -7,6 +7,7 @@ import { ArrowUpNarrowWideIcon, ChevronLeftIcon, ChevronRightIcon, + EraserIcon, FilterXIcon, ListChecksIcon, Loader2Icon, @@ -14,6 +15,7 @@ import { SearchIcon, Trash2Icon, } from "lucide-react"; +import { toast } from "sonner"; import { HttpCodeBlock } from "@/components/http-code-block"; import { LinkTrafficDialog } from "@/components/link-traffic-dialog"; @@ -56,7 +58,9 @@ function fmtTime(ts: string) { function fmtBytes(n: number) { if (n <= 0) return "0 B"; - const units = ["B", "KB", "MB"]; + // GB matters for the reclaimed-space figure a full purge reports; a capture-heavy + // instance can hand back several. + const units = ["B", "KB", "MB", "GB"]; const i = Math.min(Math.floor(Math.log(n) / Math.log(1024)), units.length - 1); const v = n / 1024 ** i; return `${i === 0 ? v : v.toFixed(1)} ${units[i]}`; @@ -142,7 +146,7 @@ export default function TrafficPage() { const [pickerOpen, setPickerOpen] = React.useState(false); const [hostCountSortDirection, setHostCountSortDirection] = React.useState("desc"); - const [deleteMode, setDeleteMode] = React.useState<"filter" | "selected" | null>(null); // null = dialog closed + const [deleteMode, setDeleteMode] = React.useState<"filter" | "selected" | "all" | null>(null); // null = dialog closed const [deleting, setDeleting] = React.useState(false); const [reloadTick, setReloadTick] = React.useState(0); // manual refetch trigger @@ -245,22 +249,50 @@ export default function TrafficPage() { [hosts, hostCountSortDirection], ); + // "清空" for the unfiltered purge, "删除" for the host-scoped ones — the dialog's + // title and its confirm button both follow from which is in play. + const deleteVerb = deleteMode === "all" ? "清空" : "删除"; + const deleteTitle = deleteMode + ? { + all: "清空全部流量记录？", + selected: `删除选中的 ${selectedHosts.length} 个目标的全部流量？`, + filter: "删除该目标的全部流量？", + }[deleteMode] + : ""; + + // `reclaimed` only comes back from the full purge; the host-scoped deletions + // report the row count alone. + const requestDelete = (mode: "filter" | "selected" | "all"): Promise<{ deleted: number; reclaimed?: number }> => { + if (mode === "all") return api.trafficDeleteAll(); + if (mode === "selected") return api.trafficDeleteHosts(selectedHosts); + return api.trafficDeleteHost(hostQ); + }; + const confirmDelete = () => { + if (!deleteMode) return; setDeleting(true); - const p = deleteMode === "selected" ? api.trafficDeleteHosts(selectedHosts) : api.trafficDeleteHost(hostQ); - p.then(() => { - setDeleteMode(null); - setSelected(null); - setDetail(null); - if (deleteMode === "selected") { - setSelectedHosts([]); - setPickerOpen(false); - } - setPage(0); - setReloadTick((t) => t + 1); - }) - .catch(() => { + const mode = deleteMode; + requestDelete(mode) + .then((r) => { + setDeleteMode(null); + setSelected(null); + setDetail(null); + if (mode !== "filter") { + setSelectedHosts([]); + setPickerOpen(false); + } + if (mode === "all") { + // Reclaimed space is the whole point of compacting an emptied index, so say so. + const reclaimed = r.reclaimed ?? 0; + const freed = reclaimed > 0 ? `，释放 ${fmtBytes(reclaimed)} 存储` : ""; + toast.success(`已清空 ${r.deleted} 条流量${freed}`); + } + setPage(0); + setReloadTick((t) => t + 1); + }) + .catch((e) => { // Keep the confirmation open so the user can retry a failed deletion. + if (mode === "all") toast.error(`清空失败：${(e as Error).message}`); }) .finally(() => setDeleting(false)); }; @@ -434,6 +466,19 @@ export default function TrafficPage() { 删除该目标 + {/* Outline rather than a second destructive button: this one ignores every + filter, so it must not look one mis-click away from "删除该目标". */} +
- - {deleteMode === "selected" - ? `删除选中的 ${selectedHosts.length} 个目标的全部流量？` - : "删除该目标的全部流量？"} - + {deleteTitle} - {deleteMode === "selected" ? ( + {deleteMode === "all" && ( + <> + 将永久删除全部 {traffic?.count ?? 0}{" "} + 条流量记录（含请求/响应原文），忽略当前的筛选条件，此操作不可撤销。已绑定到漏洞的流量证据保存在独立的证据库中，不受影响。 +
+ + 清空后会顺带压实存储，把索引占用的磁盘空间还给系统；这期间流量录制会短暂暂停。 + + + )} + {deleteMode === "selected" && ( <> 将永久删除 {selectedHosts.length} 个目标（ @@ -750,7 +801,8 @@ export default function TrafficPage() { ）的所有流量记录（含请求/响应原文），此操作不可撤销。 - ) : ( + )} + {deleteMode === "filter" && ( <> 将永久删除 host 包含 {hostQ}{" "} 的所有流量记录（含请求/响应原文），此操作不可撤销。 @@ -768,7 +820,7 @@ export default function TrafficPage() { disabled={deleting} className="bg-destructive text-destructive-foreground hover:bg-destructive/90" > - {deleting ? "删除中…" : "确认删除"} + {deleting ? `${deleteVerb}中…` : `确认${deleteVerb}`} diff --git a/web/src/lib/api.ts b/web/src/lib/api.ts index 21431a2..da43619 100644 --- a/web/src/lib/api.ts +++ b/web/src/lib/api.ts @@ -791,6 +791,9 @@ export const api = { trafficHosts: () => get<{ hosts: TrafficHost[] }>(`/traffic/hosts`), trafficDeleteHost: (host: string) => del<{ deleted: number }>(`/traffic?host=${encodeURIComponent(host)}`), trafficDeleteHosts: (hosts: string[]) => del<{ deleted: number }>(`/traffic/hosts`, { hosts }), + // Purges every exchange and compacts the index; `reclaimed` is the bytes of + // index handed back to the filesystem. Evidence bound to findings is kept. + trafficDeleteAll: () => del<{ deleted: number; reclaimed: number }>(`/traffic/all`), // ---- app settings (runtime toggles) ---- settings: () => get(`/settings`),