Merge pull request #182 from Autumn-27/feat/traffic-clear-all

feat(traffic): 流量列表新增「清空全部」，并借空库时机压实索引

CHANGELOG.md

@@ -6,11 +6,15 @@
 
 ### 流量
 
+#### 新增的功能
+
+- **流量列表新增「清空全部」**：一次删除全部流量记录，忽略当前筛选条件，并额外清理索引已不再记录的历史 host 目录，不留残余。清空后顺带做一次全量压实（`optimize` + `VACUUM` + `wal_checkpoint(TRUNCATE)`），把索引占用的磁盘空间还给系统，完成后提示实际释放了多少。已绑定到漏洞的流量证据保存在独立的证据库中，不受影响。空库上的 `VACUUM` 几乎没有成本，因此这同时是**已有实例把索引库转成增量回收模式的途径**——清空一次之后，日常按 host 删除就能自行回收空间了。
+
 #### 修复的问题
 
 - **修复删除流量后磁盘空间不被释放**：删除只让数据不可见，空间一直留在索引文件里。SQLite 删行仅把页挂到 freelist，而索引库建库时没有启用 `auto_vacuum`，文件永不收缩；同时 `ex_fts` 是 `contentless_delete` 全文索引，`DELETE` 只写 tombstone 而不回收原 postings，不合并就永久累积——删流量反而让索引变大。由于 256KB 以下的正文全部内联在这个库里，加上 trigram 索引约为正文体积的 2 倍，抓包量大的实例会为早已删掉的流量长期占用数倍磁盘（实测抓 6MB 正文 → 索引 16MB，删光后仍是 16MB）。现在新建索引库直接启用 `auto_vacuum=incremental`，每次删除提交后在后台分块执行「全文索引增量合并 + `incremental_vacuum` + `wal_checkpoint(TRUNCATE)`」，逐步把空间还给操作系统（同一场景删除后回落到 104KB）。回收分块进行并在块间释放写锁，不阻塞流量录制；进程退出时立即让出，剩余工作在下次删除时续做。
 
-> 升级说明：`auto_vacuum` 只能在建库时设定，因此**已有实例的索引库仍是旧模式**，`incremental_vacuum` 在其上是空操作——启动时会打印一行提示。这类库升级后 tombstone 的持续增长已经止住（全文索引合并照常执行），但已占用的体积需要后续提供的「存储压缩」入口一次性回收。
+> 升级说明：`auto_vacuum` 只能在建库时设定，因此**已有实例的索引库仍是旧模式**，`incremental_vacuum` 在其上是空操作——启动时会打印一行提示。这类库升级后 tombstone 的持续增长已经止住（全文索引合并照常执行），而已占用的体积用流量列表的「清空全部」回收一次即可——那一步会把库转成增量回收模式，此后日常删除自行生效。
 
 ### LLM
 

server/server.go

@@ -784,6 +784,7 @@ func (s *Server) Handler() http.Handler {
 	mux.HandleFunc("GET /api/traffic/hosts", s.getTrafficHosts)
 	mux.HandleFunc("DELETE /api/traffic", s.deleteTraffic)
 	mux.HandleFunc("DELETE /api/traffic/hosts", s.deleteTrafficHosts)
+	mux.HandleFunc("DELETE /api/traffic/all", s.deleteAllTraffic)
 	mux.HandleFunc("GET /api/traffic/exchange", s.getTrafficExchange)
 	mux.HandleFunc("GET /api/traffic/blob", s.getTrafficBlob)
 	mux.HandleFunc("GET /api/commands", s.pgListCommands)
@@ -3185,6 +3186,25 @@ func (s *Server) deleteTrafficHosts(w http.ResponseWriter, r *http.Request) {
 	writeJSON(w, 200, map[string]any{"deleted": n})
 }
 
+// deleteAllTraffic purges every recorded exchange, then compacts the index so
+// the space is actually returned to the filesystem — an emptied index is the one
+// moment a full rewrite is cheap. Evidence already bound to a finding lives in
+// the evidence store and is deliberately left alone. Returns the number of
+// exchanges deleted and the bytes of index reclaimed.
+func (s *Server) deleteAllTraffic(w http.ResponseWriter, r *http.Request) {
+	tr := s.m.Traffic()
+	if tr == nil {
+		writeErr(w, 404, "traffic disabled")
+		return
+	}
+	n, reclaimed, err := tr.DeleteAll()
+	if err != nil {
+		writeErr(w, 500, err.Error())
+		return
+	}
+	writeJSON(w, 200, map[string]any{"deleted": n, "reclaimed": reclaimed})
+}
+
 // getTrafficExchange returns the full raw request/response of one exchange,
 // read on demand from the traffic tree (bodies are not in the paged list).
 func (s *Server) getTrafficExchange(w http.ResponseWriter, r *http.Request) {

traffic/reclaim_test.go

@@ -154,3 +154,104 @@ func TestReclaimOnLegacyIndexIsHarmless(t *testing.T) {
 		t.Fatal("旧库居然回收了空闲页，说明测试没有真的构造出旧库")
 	}
 }
+
+// TestDeleteAllPurgesAndCompacts covers the page's clear-everything action: it
+// must leave nothing behind — including host directories the index no longer
+// knows about — and it must hand the index space back, since an emptied index is
+// the one moment a full rewrite is cheap.
+func TestDeleteAllPurgesAndCompacts(t *testing.T) {
+	tr, dir := openTraffic(t)
+	bulkRecord(tr, "a.example.com", 10, 200*1024)
+	bulkRecord(tr, "b.example.com", 10, 200*1024)
+	// A text body so the full-text index has real content, and a spilled one so a
+	// blob exists to collect.
+	tr.record(newFlow("c.example.com", "GET", "/page", nil, []byte(strings.Repeat("secret-token ", 500))))
+	tr.record(newFlow("c.example.com", "GET", "/big", nil,
+		[]byte(strings.Repeat("B", maxInlineBody+1024)), withRespType("application/sql")))
+	// An orphaned legacy directory: no index row points at it, so only a
+	// clear-everything should take it.
+	orphan := filepath.Join(dir, "orphan.example.com")
+	if err := os.MkdirAll(orphan, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	grown := tr.indexBytes()
+	if grown < 5<<20 {
+		t.Fatalf("索引只有 %d 字节，样本不足", grown)
+	}
+
+	deleted, reclaimed, err := tr.DeleteAll()
+	if err != nil {
+		t.Fatalf("DeleteAll: %v", err)
+	}
+	if deleted != 22 {
+		t.Fatalf("deleted=%d，应为 22", deleted)
+	}
+	tr.reaping.Wait()
+
+	if reclaimed < grown/2 {
+		t.Fatalf("只回收了 %d 字节（删除前索引 %d）", reclaimed, grown)
+	}
+	if after := tr.indexBytes(); after > grown/8 {
+		t.Fatalf("清空后索引仍占 %d 字节（删除前 %d）", after, grown)
+	}
+	for _, q := range []string{
+		`SELECT COUNT(*) FROM exchanges`,
+		`SELECT COUNT(*) FROM exchange_bodies`,
+		`SELECT COUNT(*) FROM blob_refs`,
+	} {
+		var c int
+		if err := tr.DB().QueryRow(q).Scan(&c); err != nil {
+			t.Fatal(err)
+		}
+		if c != 0 {
+			t.Fatalf("%s = %d，应为 0", q, c)
+		}
+	}
+	if _, err := os.Stat(orphan); !os.IsNotExist(err) {
+		t.Fatalf("孤立的历史 host 目录未被清理：%v", err)
+	}
+	// Recording must keep working against the freshly rewritten file.
+	tr.record(newFlow("d.example.com", "GET", "/after", nil, []byte("清空后仍可录制")))
+	if n, err := tr.Count(); err != nil || n != 1 {
+		t.Fatalf("清空后 Count=(%d,%v)，应为 (1,nil)", n, err)
+	}
+}
+
+// TestDeleteAllConvertsLegacyIndex is why the purge compacts rather than just
+// deleting: auto_vacuum cannot be switched on after the fact except through a
+// VACUUM, and an emptied index is the cheapest place to pay for one. After this,
+// ordinary deletions reclaim space on their own.
+func TestDeleteAllConvertsLegacyIndex(t *testing.T) {
+	dir := t.TempDir()
+	old := openLegacyIndex(t, dir)
+	if err := old.Close(); err != nil {
+		t.Fatal(err)
+	}
+	tr, err := Open(dir, "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { tr.Close() })
+	if tr.incrementalVacuum {
+		t.Fatal("旧库不应报告已启用增量回收")
+	}
+
+	bulkRecord(tr, "legacy.example.com", 10, 200*1024)
+	if _, _, err := tr.DeleteAll(); err != nil {
+		t.Fatalf("DeleteAll: %v", err)
+	}
+	if !tr.incrementalVacuum {
+		t.Fatal("清空后旧库未被转换为增量回收模式")
+	}
+
+	// The converted database now reclaims on an ordinary host deletion.
+	bulkRecord(tr, "again.example.com", 10, 200*1024)
+	grown := tr.indexBytes()
+	if _, err := tr.DeleteHostsExact([]string{"again.example.com"}); err != nil {
+		t.Fatal(err)
+	}
+	tr.reaping.Wait()
+	if after := tr.indexBytes(); after > grown/4 {
+		t.Fatalf("转换后普通删除仍未回收：%d 字节（删除前 %d）", after, grown)
+	}
+}

traffic/traffic.go

@@ -1066,6 +1066,116 @@ func (t *Traffic) DeleteHost(host string) (int64, error) {
 	return n, nil
 }
 
+// DeleteAll removes every recorded exchange — the page's clear-everything
+// action. It differs from the host-scoped deletions in two ways. It sweeps host
+// directories without consulting the index, because "clear everything" should
+// leave nothing behind and a directory whose rows are already gone would
+// otherwise survive. And it ends with a full compaction: VACUUM costs what it
+// keeps, so an emptied index is the one moment it is free, and it is also the
+// only way to switch on auto_vacuum for a database created without it.
+//
+// Evidence already bound to a finding is untouched. Those bodies were copied
+// into the evidence store when they were bound, precisely so that disposable
+// traffic could be cleared without taking proof with it.
+//
+// Returns the number of exchanges deleted and how many bytes of index the
+// compaction handed back.
+func (t *Traffic) DeleteAll() (deleted int64, reclaimed int64, err error) {
+	t.wmu.Lock()
+	defer t.wmu.Unlock()
+	before := t.indexBytes()
+	trees, err := t.allHostTrees()
+	if err != nil {
+		return 0, 0, err
+	}
+	tx, err := t.db.Begin()
+	if err != nil {
+		return 0, 0, err
+	}
+	defer tx.Rollback() //nolint:errcheck // no-op once committed
+	if deleted, err = t.deleteWhere(tx, `1=1`); err != nil {
+		return 0, 0, err
+	}
+	// Staged before the commit so a filesystem failure can still abort the whole
+	// deletion, exactly as in DeleteHost.
+	stageDir, moves, err := t.stageTrees(trees)
+	if err != nil {
+		return 0, 0, errors.Join(err, restoreTrees(stageDir, moves))
+	}
+	if err := tx.Commit(); err != nil {
+		return 0, 0, errors.Join(fmt.Errorf("提交流量索引删除: %w", err), restoreTrees(stageDir, moves))
+	}
+	t.reapStage(stageDir)
+	if err := t.gcBlobs(); err != nil {
+		return deleted, 0, err
+	}
+	if err := t.compactIndex(); err != nil {
+		// The deletion is already durable; compaction is disk space, not
+		// correctness, so it must not turn a completed purge into a failed one.
+		log.Printf("[traffic] 压实索引失败：%v", err)
+		return deleted, 0, nil
+	}
+	return deleted, before - t.indexBytes(), nil
+}
+
+// allHostTrees lists every host directory left by the pre-SQLite layout. Unlike
+// hostTrees it does not go through the index, so it also finds directories whose
+// rows are already gone. Underscore-prefixed entries are the store's own
+// (_index, _blobs, _ca, _delete_staging) and are never hosts.
+func (t *Traffic) allHostTrees() ([]string, error) {
+	entries, err := os.ReadDir(t.dir)
+	if err != nil {
+		return nil, err
+	}
+	var dirs []string
+	for _, e := range entries {
+		if e.IsDir() && !strings.HasPrefix(e.Name(), "_") {
+			dirs = append(dirs, filepath.Join(t.dir, e.Name()))
+		}
+	}
+	return dirs, nil
+}
+
+// compactIndex rewrites the index into a fresh file, which is what actually
+// returns its pages to the filesystem. VACUUM's runtime and temporary space
+// scale with the content it keeps, so this is only reached right after DeleteAll
+// has emptied the index — never as a routine step. It doubles as the conversion
+// path for a database created before auto_vacuum=incremental: that pragma only
+// takes effect through the VACUUM that follows it, and both must run on the same
+// connection. Callers must hold wmu.
+func (t *Traffic) compactIndex() error {
+	ctx := context.Background()
+	conn, err := t.db.Conn(ctx)
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+	if t.fts {
+		// A full merge, not the bounded one reclaim uses: with the index emptied
+		// there is nothing left to merge, so this only discards the tombstones.
+		if _, err := conn.ExecContext(ctx, `INSERT INTO ex_fts(ex_fts) VALUES('optimize')`); err != nil {
+			return fmt.Errorf("合并全文索引: %w", err)
+		}
+	}
+	if _, err := conn.ExecContext(ctx, `PRAGMA auto_vacuum=incremental`); err != nil {
+		return err
+	}
+	if _, err := conn.ExecContext(ctx, `VACUUM`); err != nil {
+		return fmt.Errorf("压实索引: %w", err)
+	}
+	var mode int
+	if err := conn.QueryRowContext(ctx, `PRAGMA auto_vacuum`).Scan(&mode); err != nil {
+		return err
+	}
+	// A legacy database has just been converted, so later deletions can reclaim
+	// space on their own instead of waiting for another purge.
+	t.incrementalVacuum = mode == autoVacuumIncremental
+	if _, err := conn.ExecContext(ctx, `PRAGMA wal_checkpoint(TRUNCATE)`); err != nil {
+		return fmt.Errorf("截断 WAL: %w", err)
+	}
+	return nil
+}
+
 // deleteWhere removes every trace of the exchanges matching the condition:
 // full-text index rows, bodies, blob references, and finally the index rows.
 // Order matters — each sub-select reads exchanges, so that table is emptied last.

web/src/app/(main)/function/traffic/page.tsx

@@ -7,6 +7,7 @@ import {
   ArrowUpNarrowWideIcon,
   ChevronLeftIcon,
   ChevronRightIcon,
+  EraserIcon,
   FilterXIcon,
   ListChecksIcon,
   Loader2Icon,
@@ -14,6 +15,7 @@ import {
   SearchIcon,
   Trash2Icon,
 } from "lucide-react";
+import { toast } from "sonner";
 
 import { HttpCodeBlock } from "@/components/http-code-block";
 import { LinkTrafficDialog } from "@/components/link-traffic-dialog";
@@ -56,7 +58,9 @@ function fmtTime(ts: string) {
 
 function fmtBytes(n: number) {
   if (n <= 0) return "0 B";
-  const units = ["B", "KB", "MB"];
+  // GB matters for the reclaimed-space figure a full purge reports; a capture-heavy
+  // instance can hand back several.
+  const units = ["B", "KB", "MB", "GB"];
   const i = Math.min(Math.floor(Math.log(n) / Math.log(1024)), units.length - 1);
   const v = n / 1024 ** i;
   return `${i === 0 ? v : v.toFixed(1)} ${units[i]}`;
@@ -142,7 +146,7 @@ export default function TrafficPage() {
   const [pickerOpen, setPickerOpen] = React.useState(false);
   const [hostCountSortDirection, setHostCountSortDirection] = React.useState<HostCountSortDirection>("desc");
 
-  const [deleteMode, setDeleteMode] = React.useState<"filter" | "selected" | null>(null); // null = dialog closed
+  const [deleteMode, setDeleteMode] = React.useState<"filter" | "selected" | "all" | null>(null); // null = dialog closed
   const [deleting, setDeleting] = React.useState(false);
   const [reloadTick, setReloadTick] = React.useState(0); // manual refetch trigger
 
@@ -245,22 +249,50 @@ export default function TrafficPage() {
     [hosts, hostCountSortDirection],
   );
 
+  // "清空" for the unfiltered purge, "删除" for the host-scoped ones — the dialog's
+  // title and its confirm button both follow from which is in play.
+  const deleteVerb = deleteMode === "all" ? "清空" : "删除";
+  const deleteTitle = deleteMode
+    ? {
+        all: "清空全部流量记录？",
+        selected: `删除选中的 ${selectedHosts.length} 个目标的全部流量？`,
+        filter: "删除该目标的全部流量？",
+      }[deleteMode]
+    : "";
+
+  // `reclaimed` only comes back from the full purge; the host-scoped deletions
+  // report the row count alone.
+  const requestDelete = (mode: "filter" | "selected" | "all"): Promise<{ deleted: number; reclaimed?: number }> => {
+    if (mode === "all") return api.trafficDeleteAll();
+    if (mode === "selected") return api.trafficDeleteHosts(selectedHosts);
+    return api.trafficDeleteHost(hostQ);
+  };
+
   const confirmDelete = () => {
+    if (!deleteMode) return;
     setDeleting(true);
-    const p = deleteMode === "selected" ? api.trafficDeleteHosts(selectedHosts) : api.trafficDeleteHost(hostQ);
-    p.then(() => {
-      setDeleteMode(null);
-      setSelected(null);
-      setDetail(null);
-      if (deleteMode === "selected") {
-        setSelectedHosts([]);
-        setPickerOpen(false);
-      }
-      setPage(0);
-      setReloadTick((t) => t + 1);
-    })
-      .catch(() => {
+    const mode = deleteMode;
+    requestDelete(mode)
+      .then((r) => {
+        setDeleteMode(null);
+        setSelected(null);
+        setDetail(null);
+        if (mode !== "filter") {
+          setSelectedHosts([]);
+          setPickerOpen(false);
+        }
+        if (mode === "all") {
+          // Reclaimed space is the whole point of compacting an emptied index, so say so.
+          const reclaimed = r.reclaimed ?? 0;
+          const freed = reclaimed > 0 ? `，释放 ${fmtBytes(reclaimed)} 存储` : "";
+          toast.success(`已清空 ${r.deleted} 条流量${freed}`);
+        }
+        setPage(0);
+        setReloadTick((t) => t + 1);
+      })
+      .catch((e) => {
         // Keep the confirmation open so the user can retry a failed deletion.
+        if (mode === "all") toast.error(`清空失败：${(e as Error).message}`);
       })
       .finally(() => setDeleting(false));
   };
@@ -434,6 +466,19 @@ export default function TrafficPage() {
           <Trash2Icon className="size-3.5" />
           删除该目标
         </Button>
+        {/* Outline rather than a second destructive button: this one ignores every
+            filter, so it must not look one mis-click away from "删除该目标". */}
+        <Button
+          variant="outline"
+          size="sm"
+          className="h-8 text-destructive hover:bg-destructive/10 hover:text-destructive"
+          disabled={!traffic?.count || deleting}
+          title={traffic?.count ? "删除全部流量并压实存储" : "当前没有流量记录"}
+          onClick={() => setDeleteMode("all")}
+        >
+          <EraserIcon className="size-3.5" />
+          清空全部
+        </Button>
         <div className="relative max-w-sm flex-1">
           <SearchIcon className="absolute top-1/2 left-2.5 size-4 -translate-y-1/2 text-muted-foreground" />
           <Input
@@ -735,13 +780,19 @@ export default function TrafficPage() {
       >
         <AlertDialogContent>
           <AlertDialogHeader>
-            <AlertDialogTitle>
-              {deleteMode === "selected"
-                ? `删除选中的 ${selectedHosts.length} 个目标的全部流量？`
-                : "删除该目标的全部流量？"}
-            </AlertDialogTitle>
+            <AlertDialogTitle>{deleteTitle}</AlertDialogTitle>
             <AlertDialogDescription>
-              {deleteMode === "selected" ? (
+              {deleteMode === "all" && (
+                <>
+                  将永久删除全部 <span className="font-semibold tabular-nums">{traffic?.count ?? 0}</span>{" "}
+                  条流量记录（含请求/响应原文），忽略当前的筛选条件，此操作不可撤销。已绑定到漏洞的流量证据保存在独立的证据库中，不受影响。
+                  <br />
+                  <span className="text-muted-foreground">
+                    清空后会顺带压实存储，把索引占用的磁盘空间还给系统；这期间流量录制会短暂暂停。
+                  </span>
+                </>
+              )}
+              {deleteMode === "selected" && (
                 <>
                   将永久删除 <span className="font-semibold tabular-nums">{selectedHosts.length}</span> 个目标（
                   <span className="font-mono">
@@ -750,7 +801,8 @@ export default function TrafficPage() {
                   </span>
                   ）的所有流量记录（含请求/响应原文），此操作不可撤销。
                 </>
-              ) : (
+              )}
+              {deleteMode === "filter" && (
                 <>
                   将永久删除 host 包含 <span className="font-mono font-semibold">{hostQ}</span>{" "}
                   的所有流量记录（含请求/响应原文），此操作不可撤销。
@@ -768,7 +820,7 @@ export default function TrafficPage() {
               disabled={deleting}
               className="bg-destructive text-destructive-foreground hover:bg-destructive/90"
             >
-              {deleting ? "删除中…" : "确认删除"}
+              {deleting ? `${deleteVerb}中…` : `确认${deleteVerb}`}
             </AlertDialogAction>
           </AlertDialogFooter>
         </AlertDialogContent>

web/src/lib/api.ts

@@ -791,6 +791,9 @@ export const api = {
   trafficHosts: () => get<{ hosts: TrafficHost[] }>(`/traffic/hosts`),
   trafficDeleteHost: (host: string) => del<{ deleted: number }>(`/traffic?host=${encodeURIComponent(host)}`),
   trafficDeleteHosts: (hosts: string[]) => del<{ deleted: number }>(`/traffic/hosts`, { hosts }),
+  // Purges every exchange and compacts the index; `reclaimed` is the bytes of
+  // index handed back to the filesystem. Evidence bound to findings is kept.
+  trafficDeleteAll: () => del<{ deleted: number; reclaimed: number }>(`/traffic/all`),
 
   // ---- app settings (runtime toggles) ----
   settings: () => get<Settings>(`/settings`),