mirror of
https://github.com/RuoJi6/ARTEX.git
synced 2026-10-28 01:03:50 +08:00
feat(traffic): 流量列表支持内容检索、路径/状态码/长度筛选与列排序 (#177)
- 后端 traffic.Page 改为 PageQuery:新增响应内容(FTS)、路径(url_template)、 状态码(精确/Nxx 分桶)、响应长度范围筛选,并支持按 ts/status/resp_len 排序 (尾随 id DESC 保证分页稳定);补 status/resp_len 两个索引(IF NOT EXISTS,旧库自动生效) - server getTraffic 解析 body/path/status/resp_min/resp_max/sort/order 参数 - 前端流量页新增高级筛选行(响应内容/路径/状态码/长度范围)与可排序表头, 排序偏好持久化到 localStorage;抽出公共组件 ui/sortable-head - 仅改动前端页走的 Page,不动 Agent 工具的 query 契约
This commit is contained in:
1 parent
02b1dbe6f8
commit
a4960b514e
6 files changed
+383
-24
No files matched your search
+84
-11
@@ -54,6 +54,8 @@ CREATE TABLE IF NOT EXISTS exchanges (
|
||||
CREATE INDEX IF NOT EXISTS idx_ex_host ON exchanges(host);
|
||||
CREATE INDEX IF NOT EXISTS idx_ex_tmpl ON exchanges(host, url_template);
|
||||
CREATE INDEX IF NOT EXISTS idx_ex_ts ON exchanges(ts);
|
||||
CREATE INDEX IF NOT EXISTS idx_ex_status ON exchanges(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_ex_resp ON exchanges(resp_len);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS exchange_bodies (
|
||||
id TEXT PRIMARY KEY,
|
||||
@@ -694,13 +696,53 @@ func ftsQuote(s string) string {
|
||||
return `"` + strings.ReplaceAll(s, `"`, `""`) + `"`
|
||||
}
|
||||
|
||||
// Page returns one page of exchange metadata filtered by an optional host
|
||||
// substring, exact method, and a free-text query q that matches across the
|
||||
// indexed metadata columns (host/url/method/content-type/status) and, when the
|
||||
// term is long enough for the trigram index, across captured request/response
|
||||
// text as well. Also returns the total number of rows matching that filter (for
|
||||
// the UI's pagination). Newest first. Bodies are never included in the rows.
|
||||
func (t *Traffic) Page(host, method, q string, page, size int) (rows []ExchangeMeta, total int, err error) {
|
||||
// PageQuery bundles the optional filters and sort order for Page. Every filter
|
||||
// is combined with AND; an empty string skips that filter, and RespMin/RespMax
|
||||
// of -1 skip the response-size bounds. An empty Sort/Order defaults to
|
||||
// newest-first by timestamp.
|
||||
type PageQuery struct {
|
||||
Host string // host substring
|
||||
Method string // exact method (case-insensitive)
|
||||
Query string // broad search box: metadata columns OR captured text
|
||||
Body string // content box: captured request/response text only (full-text)
|
||||
Path string // url_template substring — the vulnerability's path
|
||||
Status string // exact code ("404") or class bucket ("4xx")
|
||||
RespMin int64 // minimum resp_len, or -1 when unset
|
||||
RespMax int64 // maximum resp_len, or -1 when unset
|
||||
Sort string // ts | status | resp_len (default ts)
|
||||
Order string // asc | desc (default desc)
|
||||
}
|
||||
|
||||
// sortColumns whitelists the columns Page may order by, so the caller-supplied
|
||||
// Sort can never reach the SQL as anything but one of these fixed names.
|
||||
var sortColumns = map[string]string{"ts": "ts", "status": "status", "resp_len": "resp_len"}
|
||||
|
||||
// statusFilter turns a status token into a SQL condition: an exact code ("404")
|
||||
// matches that status, an "Nxx" class ("4xx") matches the whole hundreds band.
|
||||
// ok is false for an empty or unrecognized token.
|
||||
func statusFilter(s string) (cond string, args []any, ok bool) {
|
||||
s = strings.ToLower(strings.TrimSpace(s))
|
||||
if s == "" {
|
||||
return "", nil, false
|
||||
}
|
||||
if len(s) == 3 && s[0] >= '1' && s[0] <= '5' && s[1] == 'x' && s[2] == 'x' {
|
||||
base := int(s[0]-'0') * 100
|
||||
return "status>=? AND status<?", []any{base, base + 100}, true
|
||||
}
|
||||
if n, err := strconv.Atoi(s); err == nil {
|
||||
return "status=?", []any{n}, true
|
||||
}
|
||||
return "", nil, false
|
||||
}
|
||||
|
||||
// Page returns one page of exchange metadata for the traffic list, filtered by
|
||||
// the criteria in f (all optional, combined with AND) and ordered by the
|
||||
// requested column. Query is the broad search box (metadata columns OR, when the
|
||||
// term is long enough for the trigram index, captured request/response text);
|
||||
// Body narrows to exchanges whose captured text matches, via that same index.
|
||||
// Also returns the total number of rows matching the filter (for the UI's
|
||||
// pagination). Bodies are never included in the rows.
|
||||
func (t *Traffic) Page(f PageQuery, page, size int) (rows []ExchangeMeta, total int, err error) {
|
||||
if size <= 0 || size > 500 {
|
||||
size = 100
|
||||
}
|
||||
@@ -718,13 +760,25 @@ func (t *Traffic) Page(host, method, q string, page, size int) (rows []ExchangeM
|
||||
where += cond
|
||||
args = append(args, vs...)
|
||||
}
|
||||
if h := strings.TrimSpace(host); h != "" {
|
||||
if h := strings.TrimSpace(f.Host); h != "" {
|
||||
add("host LIKE ?", "%"+h+"%")
|
||||
}
|
||||
if m := strings.TrimSpace(method); m != "" {
|
||||
if m := strings.TrimSpace(f.Method); m != "" {
|
||||
add("method=?", strings.ToUpper(m))
|
||||
}
|
||||
if s := strings.TrimSpace(q); s != "" {
|
||||
if p := strings.TrimSpace(f.Path); p != "" {
|
||||
add("url_template LIKE ?", "%"+p+"%")
|
||||
}
|
||||
if cond, sargs, ok := statusFilter(f.Status); ok {
|
||||
add(cond, sargs...)
|
||||
}
|
||||
if f.RespMin >= 0 {
|
||||
add("resp_len>=?", f.RespMin)
|
||||
}
|
||||
if f.RespMax >= 0 {
|
||||
add("resp_len<=?", f.RespMax)
|
||||
}
|
||||
if s := strings.TrimSpace(f.Query); s != "" {
|
||||
like := "%" + s + "%"
|
||||
const meta = "host LIKE ? OR url LIKE ? OR url_template LIKE ? OR method LIKE ? OR content_type LIKE ? OR CAST(status AS TEXT) LIKE ?"
|
||||
// Metadata match OR full-text match: one search box, widest recall. Terms
|
||||
@@ -735,11 +789,30 @@ func (t *Traffic) Page(host, method, q string, page, size int) (rows []ExchangeM
|
||||
add("("+meta+")", like, like, like, like, like, like)
|
||||
}
|
||||
}
|
||||
// Body is the dedicated content box: it only searches captured request/response
|
||||
// text, so it goes straight to the full-text index with no metadata fallback. A
|
||||
// term too short for the trigram tokenizer cannot be served and is skipped
|
||||
// rather than guessed at — the UI hints at the three-character minimum.
|
||||
if b := strings.TrimSpace(f.Body); b != "" {
|
||||
if cond, arg, ok := t.ftsFilter(b); ok {
|
||||
add(cond, arg)
|
||||
}
|
||||
}
|
||||
if err = t.db.QueryRow(`SELECT COUNT(*) FROM exchanges`+where, args...).Scan(&total); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
col := sortColumns[strings.ToLower(strings.TrimSpace(f.Sort))]
|
||||
if col == "" {
|
||||
col = "ts"
|
||||
}
|
||||
dir := "DESC"
|
||||
if strings.EqualFold(strings.TrimSpace(f.Order), "asc") {
|
||||
dir = "ASC"
|
||||
}
|
||||
// id embeds capture timestamp + sequence, so a trailing id DESC makes the order
|
||||
// total and pagination stable even when the sort column has many ties.
|
||||
sel := `SELECT id,ts,host,method,url_template,url,status,content_type,resp_len,path FROM exchanges` +
|
||||
where + ` ORDER BY ts DESC LIMIT ? OFFSET ?`
|
||||
where + ` ORDER BY ` + col + ` ` + dir + `, id DESC LIMIT ? OFFSET ?`
|
||||
qargs := append(append([]any{}, args...), size, page*size)
|
||||
rs, err := t.db.Query(sel, qargs...)
|
||||
if err != nil {
|
||||
|
||||
@@ -360,7 +360,7 @@ func TestPageSearchesBodies(t *testing.T) {
|
||||
tr.record(newFlow("api.example.com", "POST", "/v1/login", nil, []byte(`{"error":"invalid credentials"}`)))
|
||||
tr.record(newFlow("api.example.com", "GET", "/v1/health", nil, []byte(`{"status":"ok"}`)))
|
||||
|
||||
rows, total, err := tr.Page("", "", "invalid credentials", 0, 100)
|
||||
rows, total, err := tr.Page(PageQuery{Query: "invalid credentials", RespMin: -1, RespMax: -1}, 0, 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -368,11 +368,52 @@ func TestPageSearchesBodies(t *testing.T) {
|
||||
t.Fatalf("正文关键词命中 total=%d rows=%d,应为 1/1", total, len(rows))
|
||||
}
|
||||
// Metadata matching still works alongside it.
|
||||
if _, total, err := tr.Page("", "", "health", 0, 100); err != nil || total != 1 {
|
||||
if _, total, err := tr.Page(PageQuery{Query: "health", RespMin: -1, RespMax: -1}, 0, 100); err != nil || total != 1 {
|
||||
t.Fatalf("URL 关键词 total=%d err=%v,应为 1", total, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPageFiltersAndSort covers the issue #177 additions: status-class/exact
|
||||
// filtering, response-size bounds, path (url_template) filtering, and
|
||||
// server-side sorting by resp_len.
|
||||
func TestPageFiltersAndSort(t *testing.T) {
|
||||
tr, _ := openTraffic(t)
|
||||
status := func(code int) flowOpt { return func(f *mproxy.Flow) { f.Response.StatusCode = code } }
|
||||
// Three exchanges with distinct status codes and response sizes.
|
||||
tr.record(newFlow("api.example.com", "GET", "/api/users", nil, make([]byte, 10), status(200)))
|
||||
tr.record(newFlow("api.example.com", "GET", "/api/admin", nil, make([]byte, 100), status(404)))
|
||||
tr.record(newFlow("api.example.com", "GET", "/api/users/1", nil, make([]byte, 50), status(500)))
|
||||
|
||||
// Status class band.
|
||||
if rows, _, err := tr.Page(PageQuery{Status: "4xx", RespMin: -1, RespMax: -1}, 0, 100); err != nil || len(rows) != 1 || rows[0].Status != 404 {
|
||||
t.Fatalf("status=4xx 应命中 1 条 404,得 %d 条 err=%v", len(rows), err)
|
||||
}
|
||||
// Exact status.
|
||||
if rows, _, err := tr.Page(PageQuery{Status: "500", RespMin: -1, RespMax: -1}, 0, 100); err != nil || len(rows) != 1 || rows[0].Status != 500 {
|
||||
t.Fatalf("status=500 应命中 1 条,得 %d 条 err=%v", len(rows), err)
|
||||
}
|
||||
// Response-size lower bound (>=60 keeps only the 100-byte row).
|
||||
if rows, _, err := tr.Page(PageQuery{RespMin: 60, RespMax: -1}, 0, 100); err != nil || len(rows) != 1 || rows[0].RespLen != 100 {
|
||||
t.Fatalf("resp_min=60 应命中 1 条 100B,得 %d 条 err=%v", len(rows), err)
|
||||
}
|
||||
// Path (url_template) filter narrows to the /api/admin exchange.
|
||||
if rows, _, err := tr.Page(PageQuery{Path: "/api/admin", RespMin: -1, RespMax: -1}, 0, 100); err != nil || len(rows) != 1 || rows[0].Status != 404 {
|
||||
t.Fatalf("path=/api/admin 应命中 1 条,得 %d 条 err=%v", len(rows), err)
|
||||
}
|
||||
// Sort by response length, ascending then descending.
|
||||
asc, _, err := tr.Page(PageQuery{RespMin: -1, RespMax: -1, Sort: "resp_len", Order: "asc"}, 0, 100)
|
||||
if err != nil || len(asc) != 3 {
|
||||
t.Fatalf("resp_len asc 应返回 3 条,得 %d 条 err=%v", len(asc), err)
|
||||
}
|
||||
if asc[0].RespLen != 10 || asc[1].RespLen != 50 || asc[2].RespLen != 100 {
|
||||
t.Fatalf("resp_len asc 顺序错误:%d,%d,%d", asc[0].RespLen, asc[1].RespLen, asc[2].RespLen)
|
||||
}
|
||||
desc, _, err := tr.Page(PageQuery{RespMin: -1, RespMax: -1, Sort: "resp_len", Order: "desc"}, 0, 100)
|
||||
if err != nil || len(desc) != 3 || desc[0].RespLen != 100 || desc[2].RespLen != 10 {
|
||||
t.Fatalf("resp_len desc 顺序错误 err=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryHostPortAndURLForms(t *testing.T) {
|
||||
tr, _ := openTraffic(t)
|
||||
tr.record(newFlow("api.example.com:8082", "GET", "/admin", nil, []byte("8082")))
|
||||
|
||||
Reference in new issue
Block a user