feat: delete traffic targets and LLM task conversations from UI

Add delete controls to the Traffic and LLM Records pages so users can purge recorded data without hand-editing the data volume. Traffic page: - Host target picker (Popover with checkboxes + select-all toggle) - Exact-match batch delete for multiple hosts (DELETE /api/traffic/hosts) - Substring delete for the host-filter input (DELETE /api/traffic?host=) - Orphaned content-addressed blob garbage collection after deletes - Distinct-hosts endpoint (GET /api/traffic/hosts) for the picker LLM Records page: - Task picker dropdown (distinct task_id + record counts) - Per-task delete (DELETE /api/llm/records?task=) Other: - Add /dist/ to .gitignore (build artifact) - Extend traffic_test.go with DeleteHost, DeleteHostsExact, Hosts tests - Fix Popover render bug (replace ScrollArea that broke height calc) - Remove fade/zoom animation on the host picker Popover
2026-08-10 11:10:58 +08:00
1 parent 5962826f2d
commit c76d0f4c93
@@ -11,12 +11,14 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
+	"io/fs"
 	"log"
 	"net"
 	"net/http"
 	"net/url"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -55,7 +57,7 @@ type Traffic struct {
 	dir   string
 	addr  string
 	db    *sql.DB
-	wmu   sync.Mutex
+	wmu   sync.Mutex // serializes record() vs DeleteHost (incl. blob GC)
 	seq   atomic.Int64
 	proxy *mproxy.Proxy
 	// pass is the set of hosts whose MITM interception failed for a proxy/protocol
@@ -186,6 +188,10 @@ func proxyCausedErr(err error) bool {
 }

 func (t *Traffic) record(f *mproxy.Flow) {
+	// The write lock covers blob + tree + index writes, so DeleteHost (and its
+	// blob GC) can run under the same lock without racing a concurrent record.
+	t.wmu.Lock()
+	defer t.wmu.Unlock()
 	host := f.Request.URL.Hostname()
 	method := f.Request.Method
 	tmpl := db.TemplatePath(f.Request.URL.EscapedPath())
@@ -221,12 +227,10 @@ func (t *Traffic) record(f *mproxy.Flow) {
 	_ = os.WriteFile(filepath.Join(exDir, "meta.json"), []byte(meta), 0o644)

 	rel, _ := filepath.Rel(t.dir, exDir)
-	t.wmu.Lock()
 	_, _ = t.db.Exec(`INSERT OR REPLACE INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
 VALUES(?,?,?,?,?,?,?,?,?,?,?)`,
 		id, now.Unix(), host, method, tmpl, f.Request.URL.String(), f.Response.StatusCode, ct,
 		len(f.Request.Body), len(f.Response.Body), rel)
-	t.wmu.Unlock()
 }

 // bodyOrBlob returns the body inline if small, else stores it content-addressed
@@ -384,6 +388,32 @@ func (t *Traffic) Get(id string) (req, resp string, err error) {
 	return string(rb), string(pb), nil
 }

+// HostCount is one distinct recorded host plus its exchange count.
+type HostCount struct {
+	Host  string `json:"host"`
+	Count int    `json:"count"`
+}
+
+// Hosts returns distinct recorded hosts with exchange counts, most recent
+// activity first — powers the page's target picker.
+func (t *Traffic) Hosts() ([]HostCount, error) {
+	rows, err := t.db.Query(`SELECT host, COUNT(*) AS n, MAX(ts) AS last FROM exchanges GROUP BY host ORDER BY last DESC`)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var out []HostCount
+	for rows.Next() {
+		var h HostCount
+		var last int64
+		if err := rows.Scan(&h.Host, &h.Count, &last); err != nil {
+			return nil, err
+		}
+		out = append(out, h)
+	}
+	return out, rows.Err()
+}
+
 // Count returns total recorded exchanges.
 func (t *Traffic) Count() (int, error) {
 	var n int
@@ -391,6 +421,124 @@ func (t *Traffic) Count() (int, error) {
 	return n, err
 }

+// DeleteHost removes every recorded exchange whose host contains the given
+// substring — mirroring the UI's host filter, so what you filtered is what gets
+// deleted: the index rows plus each matched host's file tree (<dir>/<host>/…).
+// Content-addressed blobs in _blobs are shared across hosts, so instead of
+// deleting by host they are garbage-collected afterwards: any blob no longer
+// referenced by a remaining exchange tree is removed (frees the data volume).
+// Returns rows deleted.
+func (t *Traffic) DeleteHost(host string) (int64, error) {
+	like := "%" + host + "%"
+	t.wmu.Lock()
+	defer t.wmu.Unlock()
+	// Collect distinct matched hosts first (index is the source of truth for
+	// tree names) so the trees are removed even if the DELETE row count is 0.
+	rows, err := t.db.Query(`SELECT DISTINCT host FROM exchanges WHERE host LIKE ?`, like)
+	if err != nil {
+		return 0, err
+	}
+	var hosts []string
+	for rows.Next() {
+		var h string
+		if err := rows.Scan(&h); err != nil {
+			rows.Close()
+			return 0, err
+		}
+		hosts = append(hosts, h)
+	}
+	rows.Close()
+	if err := rows.Err(); err != nil {
+		return 0, err
+	}
+	res, err := t.db.Exec(`DELETE FROM exchanges WHERE host LIKE ?`, like)
+	if err != nil {
+		return 0, err
+	}
+	n, _ := res.RowsAffected()
+	for _, h := range hosts {
+		os.RemoveAll(filepath.Join(t.dir, sanitize(h)))
+	}
+	if n > 0 {
+		_ = t.gcBlobs()
+	}
+	return n, nil
+}
+
+// DeleteHostsExact removes recorded exchanges for a set of EXACT hosts (the
+// batch path for the page's multi-select delete): index rows + each host's file
+// tree, then one blob-GC pass. Exact match — unlike DeleteHost's substring —
+// so picking "api.example.com" never sweeps "api.example.com.cn". Returns rows
+// deleted. Duplicate hosts are harmless (idempotent deletes, single tree pass).
+func (t *Traffic) DeleteHostsExact(hosts []string) (int64, error) {
+	t.wmu.Lock()
+	defer t.wmu.Unlock()
+	var deleted int64
+	removed := make(map[string]bool)
+	for _, h := range hosts {
+		res, err := t.db.Exec(`DELETE FROM exchanges WHERE host=?`, h)
+		if err != nil {
+			return deleted, err
+		}
+		n, _ := res.RowsAffected()
+		deleted += n
+		if !removed[h] {
+			os.RemoveAll(filepath.Join(t.dir, sanitize(h)))
+			removed[h] = true
+		}
+	}
+	if deleted > 0 {
+		_ = t.gcBlobs()
+	}
+	return deleted, nil
+}
+
+// gcBlobs removes content-addressed blobs in _blobs that no remaining exchange
+// tree references. References appear as "@blob sha256:<hex>" lines inside the
+// tree's request.http/response.http files (see bodyOrBlob). Best-effort: walk
+// errors only skip cleanup of the affected files. Callers must hold wmu so this
+// never races a concurrent record() writing a fresh blob + tree.
+func (t *Traffic) gcBlobs() error {
+	refs := make(map[string]struct{})
+	blobRe := regexp.MustCompile(`@blob sha256:([0-9a-f]{64})`)
+	err := filepath.WalkDir(t.dir, func(p string, d fs.DirEntry, err error) error {
+		if err != nil || d == nil {
+			return nil // skip unreadable entries
+		}
+		if d.IsDir() {
+			// _blobs/_index/_ca never contain references; skip their subtrees.
+			if p != t.dir && strings.HasPrefix(d.Name(), "_") {
+				return filepath.SkipDir
+			}
+			return nil
+		}
+		if n := d.Name(); n != "request.http" && n != "response.http" {
+			return nil
+		}
+		b, err := os.ReadFile(p)
+		if err != nil {
+			return nil
+		}
+		for _, m := range blobRe.FindAllSubmatch(b, -1) {
+			refs[string(m[1])] = struct{}{}
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+	return filepath.WalkDir(filepath.Join(t.dir, "_blobs", "sha256"), func(p string, d fs.DirEntry, err error) error {
+		if err != nil || d == nil || d.IsDir() {
+			return nil
+		}
+		h := strings.TrimSuffix(d.Name(), ".bin")
+		if _, ok := refs[h]; !ok {
+			os.Remove(p)
+		}
+		return nil
+	})
+}
+
 // query returns one page of exchange metadata filtered by host and/or a url
 // substring. Default page size is intentionally small (3) to keep tool results
 // lightweight and capped at 10; page is 0-based (page*limit offset).
@@ -0,0 +1,246 @@
+package traffic
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// TestDeleteHost verifies the delete contract: rows for hosts containing the
+// substring are removed together with their file trees, non-matching hosts are
+// untouched, and the count is right.
+func TestDeleteHost(t *testing.T) {
+	dir := t.TempDir()
+	tr, err := Open(dir, "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer tr.Close()
+
+	// Seed two hosts' index rows + trees directly (record() needs a live Flow).
+	for i, h := range []string{"a.example.com", "b.example.com"} {
+		id := fmt.Sprintf("1-%04d", i+1)
+		exDir := filepath.Join(dir, h, "GET", id)
+		if err := os.MkdirAll(exDir, 0o755); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(filepath.Join(exDir, "meta.json"), []byte(fmt.Sprintf(`{"id":%q,"host":%q}`, id, h)), 0o644); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := tr.DB().Exec(`INSERT INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
+VALUES(?,?,?,?,?,?,?,?,?,?,?)`,
+			id, i+1, h, "GET", "/", "http://"+h+"/", 200, "text/html", 0, 0, h+"/GET/"+id); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// Substring: "a.example" matches a.example.com only, leaves b.example.com.
+	n, err := tr.DeleteHost("a.example")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if n != 1 {
+		t.Fatalf("deleted=%d, want 1", n)
+	}
+	// Tree removed for the target, intact for the other host.
+	if _, err := os.Stat(filepath.Join(dir, "a.example.com")); !os.IsNotExist(err) {
+		t.Fatalf("a.example.com tree still exists (stat err=%v)", err)
+	}
+	if _, err := os.Stat(filepath.Join(dir, "b.example.com")); err != nil {
+		t.Fatalf("b.example.com tree removed: %v", err)
+	}
+	// Index reduced to the other host's single row.
+	var c int
+	if err := tr.DB().QueryRow(`SELECT COUNT(*) FROM exchanges`).Scan(&c); err != nil {
+		t.Fatal(err)
+	}
+	if c != 1 {
+		t.Fatalf("rows=%d, want 1", c)
+	}
+	// A substring matching nothing is a no-op, not an error.
+	n, err = tr.DeleteHost("nope.example")
+	if err != nil || n != 0 {
+		t.Fatalf("DeleteHost(missing)=%d, err=%v; want 0, nil", n, err)
+	}
+	// A broader substring sweeps the remaining host too.
+	if n, err = tr.DeleteHost("example.com"); err != nil || n != 1 {
+		t.Fatalf("DeleteHost(example.com)=%d, err=%v; want 1, nil", n, err)
+	}
+	c = 0
+	if err := tr.DB().QueryRow(`SELECT COUNT(*) FROM exchanges`).Scan(&c); err == nil && c != 0 {
+		t.Fatalf("rows=%d, want 0 after full sweep", c)
+	}
+}
+
+// TestHosts verifies the target picker contract: distinct hosts with counts,
+// most recent activity first.
+func TestHosts(t *testing.T) {
+	dir := t.TempDir()
+	tr, err := Open(dir, "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer tr.Close()
+	for i, row := range []struct {
+		host string
+		ts   int64
+	}{{"old.example.com", 1}, {"new.example.com", 3}, {"old.example.com", 2}} {
+		id := fmt.Sprintf("1-%04d", i+1)
+		if _, err := tr.DB().Exec(`INSERT INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
+VALUES(?,?,?,?,?,?,?,?,?,?,?)`,
+			id, row.ts, row.host, "GET", "/", "http://"+row.host+"/", 200, "text/html", 0, 0, row.host+"/GET/"+id); err != nil {
+			t.Fatal(err)
+		}
+	}
+	hosts, err := tr.Hosts()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(hosts) != 2 {
+		t.Fatalf("hosts=%d, want 2", len(hosts))
+	}
+	// newest activity (ts=3) first
+	if hosts[0].Host != "new.example.com" || hosts[0].Count != 1 {
+		t.Fatalf("hosts[0]=%+v, want new.example.com/1", hosts[0])
+	}
+	if hosts[1].Host != "old.example.com" || hosts[1].Count != 2 {
+		t.Fatalf("hosts[1]=%+v, want old.example.com/2", hosts[1])
+	}
+}
+
+// TestDeleteHostsExact verifies the batch delete: exact host match only — a
+// host whose name contains another as a substring is untouched — duplicates in
+// the batch are harmless, and the per-host trees are removed.
+func TestDeleteHostsExact(t *testing.T) {
+	dir := t.TempDir()
+	tr, err := Open(dir, "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer tr.Close()
+	seed := func(id, h string) {
+		exDir := filepath.Join(dir, h, "GET", id)
+		if err := os.MkdirAll(exDir, 0o755); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := tr.DB().Exec(`INSERT INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
+VALUES(?,?,?,?,?,?,?,?,?,?,?)`,
+			id, 1, h, "GET", "/", "http://"+h+"/", 200, "text/html", 0, 0, h+"/GET/"+id); err != nil {
+			t.Fatal(err)
+		}
+	}
+	// "api.example.com" is a substring of "api.example.com.cn".
+	seed("1-0001", "api.example.com")
+	seed("1-0002", "api.example.com.cn")
+	seed("1-0003", "shop.example.com")
+
+	// Duplicate entry in the batch must not double-delete or error.
+	n, err := tr.DeleteHostsExact([]string{"api.example.com", "api.example.com", "shop.example.com"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if n != 2 {
+		t.Fatalf("deleted=%d, want 2", n)
+	}
+	if _, err := os.Stat(filepath.Join(dir, "api.example.com")); !os.IsNotExist(err) {
+		t.Fatalf("api.example.com tree still exists: %v", err)
+	}
+	if _, err := os.Stat(filepath.Join(dir, "shop.example.com")); !os.IsNotExist(err) {
+		t.Fatalf("shop.example.com tree still exists: %v", err)
+	}
+	if _, err := os.Stat(filepath.Join(dir, "api.example.com.cn")); err != nil {
+		t.Fatalf("api.example.com.cn removed by an exact delete that shouldn't match: %v", err)
+	}
+	var c int
+	if err := tr.DB().QueryRow(`SELECT COUNT(*) FROM exchanges`).Scan(&c); err != nil {
+		t.Fatal(err)
+	}
+	if c != 1 {
+		t.Fatalf("rows=%d, want 1 (api.example.com.cn only)", c)
+	}
+}
+
+// TestDeleteHostGCBlobs verifies blob garbage collection: after a host's trees
+// are removed, blobs referenced by no remaining exchange are deleted, while
+// blobs still referenced (including shared ones) survive.
+func TestDeleteHostGCBlobs(t *testing.T) {
+	dir := t.TempDir()
+	tr, err := Open(dir, "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer tr.Close()
+
+	// Two distinct blobs + one shared blob (referenced by two hosts).
+	blobA := filepath.Join(dir, "_blobs", "sha256", "aa", "aa", strings.Repeat("a", 64)+".bin")
+	blobB := filepath.Join(dir, "_blobs", "sha256", "bb", "bb", strings.Repeat("b", 64)+".bin")
+	blobC := filepath.Join(dir, "_blobs", "sha256", "cc", "cc", strings.Repeat("c", 64)+".bin")
+	for _, b := range []string{blobA, blobB, blobC} {
+		if err := os.MkdirAll(filepath.Dir(b), 0o755); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(b, []byte("x"), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	seed := func(id, h, ref string) {
+		exDir := filepath.Join(dir, h, "GET", id)
+		if err := os.MkdirAll(exDir, 0o755); err != nil {
+			t.Fatal(err)
+		}
+		body := "no blob"
+		if ref != "" {
+			body = "@blob sha256:" + ref + " (len=1)"
+		}
+		if err := os.WriteFile(filepath.Join(exDir, "request.http"), []byte("GET / HTTP/1.1\n\n"+body), 0o644); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(filepath.Join(exDir, "response.http"), []byte("HTTP 200 OK\n"), 0o644); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := tr.DB().Exec(`INSERT INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
+VALUES(?,?,?,?,?,?,?,?,?,?,?)`,
+			id, 1, h, "GET", "/", "http://"+h+"/", 200, "text/html", 0, 0, h+"/GET/"+id); err != nil {
+			t.Fatal(err)
+		}
+	}
+	ha := strings.Repeat("a", 64)
+	hb := strings.Repeat("b", 64)
+	hc := strings.Repeat("c", 64)
+	seed("1-0001", "a.example.com", ha) // sole reference to blobA
+	seed("1-0002", "b.example.com", hb) // sole reference to blobB
+	seed("1-0003", "c.example.com", hc) // shares blobC with d
+	seed("1-0004", "d.example.com", hc)
+
+	// Delete a: blobA orphaned → removed; blobB/blobC still referenced → kept.
+	if n, err := tr.DeleteHost("a.example"); err != nil || n != 1 {
+		t.Fatalf("DeleteHost(a.example)=%d, err=%v; want 1, nil", n, err)
+	}
+	if _, err := os.Stat(blobA); !os.IsNotExist(err) {
+		t.Fatalf("orphaned blobA still exists: %v", err)
+	}
+	if _, err := os.Stat(blobB); err != nil {
+		t.Fatalf("referenced blobB removed: %v", err)
+	}
+	if _, err := os.Stat(blobC); err != nil {
+		t.Fatalf("shared blobC removed while d still references it: %v", err)
+	}
+
+	// Delete c (shares blobC with d): blobC must survive.
+	if n, err := tr.DeleteHost("c.example"); err != nil || n != 1 {
+		t.Fatalf("DeleteHost(c.example)=%d, err=%v; want 1, nil", n, err)
+	}
+	if _, err := os.Stat(blobC); err != nil {
+		t.Fatalf("shared blobC removed after deleting one sharer: %v", err)
+	}
+
+	// Delete d: last reference gone → blobC collected.
+	if n, err := tr.DeleteHost("d.example"); err != nil || n != 1 {
+		t.Fatalf("DeleteHost(d.example)=%d, err=%v; want 1, nil", n, err)
+	}
+	if _, err := os.Stat(blobC); !os.IsNotExist(err) {
+		t.Fatalf("blobC still exists after last reference removed: %v", err)
+	}
+}