fix(traffic): match recorded hosts with port-aware search (#114)
This commit is contained in:
RuoJi6 authored and 
GitHub committed
1 parent
f2589ad47c
commit
7c8b55ca6c
4 files changed
+124
-4
No files matched your search
@@ -10,10 +10,22 @@ import (
|
||||
|
||||
"github.com/Autumn-27/artex/agent"
|
||||
"github.com/Autumn-27/artex/db"
|
||||
"github.com/Autumn-27/artex/traffic"
|
||||
actool "github.com/Autumn-27/norma/tool"
|
||||
)
|
||||
|
||||
func (s *Server) seedFindingWorkflowTools() {
|
||||
const hostSearchDescriptionFlag = "finding_workflow_tools_v3_host_search_description"
|
||||
if value, _, _ := s.m.pg.GetSetting(hostSearchDescriptionFlag); value != "true" {
|
||||
// Only replace the original built-in text. A user-edited description is
|
||||
// authoritative and must survive upgrades.
|
||||
legacy := "查询记录代理已抓取的目标流量（必须指定 host，可再按 URL 子串或正文关键词过滤）。body_contains 会在已抓取的请求/响应头与正文中做全文搜索，支持任意子串和中文（至少 3 个字符），可用来找响应里的密码、密钥、报错、内网地址等。仅返回极轻量索引(id/method/url/status/resp_len)，不含任何响应内容。默认只返回 3 条、每页最多 10 条；结果多时用 page 翻页（page=0 起）；要看某条的请求/响应原文用 traffic_get(id)。回看已访问资源、找端点先用它，避免重复 curl 同一 URL。"
|
||||
if _, err := s.m.pg.Exec(`UPDATE tools SET description=$1,updated_at=now() WHERE key='traffic_search' AND system AND description=$2`, traffic.TrafficSearchDescription, legacy); err != nil {
|
||||
log.Printf("[evidence] upgrade traffic_search description: %v", err)
|
||||
return
|
||||
}
|
||||
_ = s.m.pg.SetSetting(hostSearchDescriptionFlag, "true")
|
||||
}
|
||||
const flag = "finding_workflow_tools_v2_reporter"
|
||||
if value, _, _ := s.m.pg.GetSetting(flag); value == "true" {
|
||||
return
|
||||
|
||||
@@ -246,6 +246,9 @@ func TestFindingWorkflowMigrationPreservesUserConfiguration(t *testing.T) {
|
||||
}
|
||||
search, _ := pg.GetTool("traffic_search")
|
||||
get, _ := pg.GetTool("traffic_get")
|
||||
if !strings.Contains(search.Description, "支持裸主机、主机:端口或完整 URL") {
|
||||
t.Fatal("traffic_search description migration missing host/port guidance")
|
||||
}
|
||||
if search.Enabled || !contains(search.Agents, "reporter") || get.Enabled || len(get.Agents) != 1 || get.Agents[0] != "custom-agent" {
|
||||
t.Fatal("default/custom reader binding migration incorrect")
|
||||
}
|
||||
|
||||
+62
-4
@@ -23,6 +23,7 @@ import (
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
@@ -101,6 +102,11 @@ const (
|
||||
maxBlobRead = 8 * 1024
|
||||
)
|
||||
|
||||
// TrafficSearchDescription is persisted into the tool catalog for new and
|
||||
// upgraded installations. Keep it in the traffic package so the runtime tool
|
||||
// and the catalog migration cannot drift apart.
|
||||
const TrafficSearchDescription = "查询记录代理已抓取的目标流量（必须指定 host；支持裸主机、主机:端口或完整 URL，可再按 URL 子串或正文关键词过滤）。指定端口时只返回该服务的流量，避免同一 IP 的不同端口串包。body_contains 会在已抓取的请求/响应头与正文中做全文搜索，支持任意子串和中文（至少 3 个字符）。仅返回极轻量索引(id/method/url/status/resp_len)，不含响应内容；结果非空后必须用 traffic_get 逐条核实请求/响应，再把确实支持当前漏洞的 ID 交给 bind_finding_traffic。默认只返回 3 条、每页最多 10 条；结果多时用 page 翻页。"
|
||||
|
||||
// Traffic runs the recording proxy and owns the file tree + index.
|
||||
type Traffic struct {
|
||||
dir string
|
||||
@@ -1474,8 +1480,24 @@ func (t *Traffic) query(host, contains, bodyContains string, page, limit int) ([
|
||||
q := `SELECT id,ts,host,method,url_template,url,status,content_type,resp_len,path FROM exchanges WHERE 1=1`
|
||||
args := []any{}
|
||||
if host != "" {
|
||||
hostName, port, err := normalizeSearchHost(host)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
q += ` AND host=?`
|
||||
args = append(args, host)
|
||||
args = append(args, hostName)
|
||||
if port != "" {
|
||||
// Recorded rows historically store URL.Hostname() (without the port),
|
||||
// so constrain the original URL authority as a compatibility fallback.
|
||||
// New and old captures therefore share the same search contract.
|
||||
authority := net.JoinHostPort(hostName, port)
|
||||
q += ` AND (url LIKE ? OR url LIKE ? OR url LIKE ?)`
|
||||
args = append(args,
|
||||
"%://"+authority+"/%",
|
||||
"%://"+authority+"?%",
|
||||
"%://"+authority,
|
||||
)
|
||||
}
|
||||
}
|
||||
if contains != "" {
|
||||
q += ` AND (url LIKE ? OR url_template LIKE ?)`
|
||||
@@ -1510,6 +1532,42 @@ func (t *Traffic) query(host, contains, bodyContains string, page, limit int) ([
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// normalizeSearchHost accepts the forms agents commonly have at hand while
|
||||
// keeping the stored host (URL.Hostname()) as the canonical host key. A port,
|
||||
// when supplied, is applied to the URL authority so captures from different
|
||||
// services on the same IP cannot be mixed.
|
||||
func normalizeSearchHost(raw string) (host, port string, err error) {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return "", "", errors.New("host 为必填参数")
|
||||
}
|
||||
if strings.Contains(raw, "://") {
|
||||
u, parseErr := url.Parse(raw)
|
||||
if parseErr != nil || u.Host == "" {
|
||||
return "", "", fmt.Errorf("无法解析 host：%q", raw)
|
||||
}
|
||||
host, port = u.Hostname(), u.Port()
|
||||
} else if h, p, splitErr := net.SplitHostPort(raw); splitErr == nil {
|
||||
host, port = h, p
|
||||
} else if strings.HasPrefix(raw, "[") && strings.HasSuffix(raw, "]") {
|
||||
host = strings.TrimSuffix(strings.TrimPrefix(raw, "["), "]")
|
||||
} else {
|
||||
host = raw
|
||||
}
|
||||
host = strings.Trim(strings.TrimSpace(host), "[]")
|
||||
if host == "" {
|
||||
return "", "", fmt.Errorf("无法解析 host：%q", raw)
|
||||
}
|
||||
if port != "" {
|
||||
p, parseErr := strconv.Atoi(port)
|
||||
if parseErr != nil || p < 1 || p > 65535 {
|
||||
return "", "", fmt.Errorf("端口无效：%q", port)
|
||||
}
|
||||
port = strconv.Itoa(p)
|
||||
}
|
||||
return strings.ToLower(host), port, nil
|
||||
}
|
||||
|
||||
// Tools exposes traffic lookup to work agents so they query already-captured
|
||||
// traffic instead of re-curling the same resource (token + dedup win).
|
||||
func (t *Traffic) Tools() []actool.CoreTool {
|
||||
@@ -1520,11 +1578,11 @@ func (t *Traffic) Tools() []actool.CoreTool {
|
||||
|
||||
search := actool.Build(actool.Spec{
|
||||
Name: "traffic_search",
|
||||
Description: "查询记录代理已抓取的目标流量（必须指定 host，可再按 URL 子串或正文关键词过滤）。body_contains 会在已抓取的请求/响应头与正文中做全文搜索，支持任意子串和中文（至少 3 个字符），可用来找响应里的密码、密钥、报错、内网地址等。仅返回极轻量索引(id/method/url/status/resp_len)，不含任何响应内容。默认只返回 3 条、每页最多 10 条；结果多时用 page 翻页（page=0 起）；要看某条的请求/响应原文用 traffic_get(id)。回看已访问资源、找端点先用它，避免重复 curl 同一 URL。",
|
||||
Description: TrafficSearchDescription,
|
||||
Schema: map[string]any{
|
||||
"type": "object",
|
||||
"properties": map[string]any{
|
||||
"host": map[string]any{"type": "string", "description": "按主机过滤（必填，如 '107.172.96.177:8082'）"},
|
||||
"host": map[string]any{"type": "string", "description": "按主机过滤（必填；如 '107.172.96.177'、'107.172.96.177:8082' 或 'http://107.172.96.177:8082/path'）"},
|
||||
"contains": map[string]any{"type": "string", "description": "URL 子串过滤（可选，如 'api' / 'login'）"},
|
||||
"body_contains": map[string]any{"type": "string", "description": "正文全文搜索（可选，至少 3 个字符），匹配请求/响应的头与正文，如 'password' / 'root:x:0' / '内网测试'"},
|
||||
"limit": map[string]any{"type": "integer", "description": "每页条数，默认 3，最大 10"},
|
||||
@@ -1543,7 +1601,7 @@ func (t *Traffic) Tools() []actool.CoreTool {
|
||||
}
|
||||
_ = json.Unmarshal(in, &a)
|
||||
if strings.TrimSpace(a.Host) == "" {
|
||||
return actool.Errorf("host 为必填参数：请指定要查询的主机（如 '107.172.96.177:8082'），避免全库扫描。"), nil
|
||||
return actool.Errorf("host 为必填参数：请指定裸主机、主机:端口或完整 URL，避免全库扫描。"), nil
|
||||
}
|
||||
rows, err := t.query(a.Host, a.Contains, a.BodyContains, a.Page, a.Limit)
|
||||
if err != nil {
|
||||
|
||||
@@ -373,6 +373,53 @@ func TestPageSearchesBodies(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryHostPortAndURLForms(t *testing.T) {
|
||||
tr, _ := openTraffic(t)
|
||||
tr.record(newFlow("api.example.com:8082", "GET", "/admin", nil, []byte("8082")))
|
||||
tr.record(newFlow("api.example.com:8088", "GET", "/admin", nil, []byte("8088")))
|
||||
tr.record(newFlow("[2001:db8::1]:8443", "GET", "/admin", nil, []byte("8443")))
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
host string
|
||||
want int
|
||||
}{
|
||||
{name: "bare host", host: "api.example.com", want: 2},
|
||||
{name: "host and port", host: "api.example.com:8082", want: 1},
|
||||
{name: "full URL", host: "http://api.example.com:8088/admin", want: 1},
|
||||
{name: "IPv6 host and port", host: "[2001:db8::1]:8443", want: 1},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
rows, err := tr.query(tc.host, "", "", 0, 10)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(rows) != tc.want {
|
||||
t.Fatalf("query(%q) returned %d rows, want %d", tc.host, len(rows), tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeSearchHost(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
raw, host, port string
|
||||
}{
|
||||
{raw: "API.Example.com", host: "api.example.com"},
|
||||
{raw: "api.example.com:8088", host: "api.example.com", port: "8088"},
|
||||
{raw: "https://[2001:db8::1]:8443/path", host: "2001:db8::1", port: "8443"},
|
||||
{raw: "[2001:db8::1]", host: "2001:db8::1"},
|
||||
} {
|
||||
host, port, err := normalizeSearchHost(tc.raw)
|
||||
if err != nil {
|
||||
t.Fatalf("normalizeSearchHost(%q): %v", tc.raw, err)
|
||||
}
|
||||
if host != tc.host || port != tc.port {
|
||||
t.Fatalf("normalizeSearchHost(%q)=(%q,%q), want (%q,%q)", tc.raw, host, port, tc.host, tc.port)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestTruncateUTF8 guards the preview cut: never split a multi-byte rune.
|
||||
func TestTruncateUTF8(t *testing.T) {
|
||||
s := "内网测试账号"
|
||||
|
||||
Reference in new issue
Block a user