feat(findings): 发现页新增「按资产」视图
左侧资产树 + 右侧该子树下的发现,与另外两个视图共用同一张表和同一批筛选。 后端 - db/finding_assets.go:按当前筛选构建资产树。只收有发现的资产,祖先链逐层 补齐(endpoint→service→subdomain/ip→root_domain),父子优先级与 BuildCoverageGraph 同源;计数沿祖先链去重累加,一条发现挂多个资产不会让 父节点重复计数 - FindingFilter 加 AssetScope:选中节点=选中整棵子树,解析成资产 id 集合后 下推 SQL,列表 / 分组 / 导出三个入口共用,导出「当前筛选」自动带上 - 「未关联资产」同时收 asset_ids 为空与指向已删资产的发现,与树上桶的口径 一致(只判空数组会让桶上的数字大于点开后的条数) - findingFilterFromQuery 统一四处的筛选解析;新增 GIN 索引 idx_findings_asset_ids,按资产反查发现不再全表扫 前端 - 树默认展开一层,支持过滤(匹配项自动展开)、面包屑下钻、手动刷新 - 每层只显示相对上一层的增量:子域名去根域名后缀、服务显示 `https :443`、 接口只留路径。完整值在 title 与面包屑,搜索仍按完整值匹配 - 不轮询:进入视图 / 改筛选 / 页内改动发现 / 点刷新时才查 - 节点超上限时整层丢弃接口/服务并提示,计数已在上层不会丢
This commit is contained in:
Autumn-27 committed
1 parent
796dbf5600
commit
23cfb9ed4e
11 files changed
+1582
-82
No files matched your search
@@ -0,0 +1,669 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Finding asset tree — the「按资产」view of the global findings list.
|
||||
//
|
||||
// 层级与 BuildCoverageGraph 同源(company → root_domain/ip/app → subdomain →
|
||||
// service → endpoint),但那份是「某个任务的范围内资产」的力导向图,这份是
|
||||
// 「全库有发现的资产」的树:只收有发现的资产及其祖先链,节点带子树聚合计数。
|
||||
// 两者的父子优先级规则必须保持一致,改一处请对照 task_scope.go 改另一处。
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// FindingUnassignedAsset 是「未关联资产」的节点 key,也是列表接口的筛选哨兵:
|
||||
// 命中 asset_ids 为空、或所指资产已被删除的发现。
|
||||
const FindingUnassignedAsset = "__none__"
|
||||
|
||||
// findingAssetTreeMaxNodes 是返回给前端的节点上限。超出时自底向上丢弃整层
|
||||
// (endpoint 优先,其次 service):它们的计数已经累加进父节点,丢节点不丢数字。
|
||||
const findingAssetTreeMaxNodes = 3000
|
||||
|
||||
// FindingAssetNode 是资产树的一个节点。Key 与覆盖图同构:资产行是 "a:<id>"、
|
||||
// 企业是 "c:<id>"、没有资产行的根域名是合成的 "r:<domain>"、未关联桶是 "__none__"。
|
||||
type FindingAssetNode struct {
|
||||
Key string `json:"key"`
|
||||
Parent string `json:"parent,omitempty"`
|
||||
Kind string `json:"kind"` // company|root_domain|subdomain|ip|service|app|endpoint|none
|
||||
Label string `json:"label"`
|
||||
AssetID int64 `json:"asset_id,omitempty"`
|
||||
CompanyID int64 `json:"company_id,omitempty"`
|
||||
// Self 是直接挂在该资产上的发现数;Total 含全部子孙且按 finding 去重
|
||||
// (一个发现挂多个资产时,只在共同祖先上计一次)。
|
||||
Self int `json:"self"`
|
||||
Total int `json:"total"`
|
||||
Critical int `json:"critical"`
|
||||
High int `json:"high"`
|
||||
Medium int `json:"medium"`
|
||||
Low int `json:"low"`
|
||||
LastFoundAt time.Time `json:"last_found_at"`
|
||||
}
|
||||
|
||||
// FindingAssetTree 是整棵树的一次性快照。Nodes 已排好序:同一父节点下按发现数
|
||||
// 降序、标签升序,「未关联资产」恒在最后。
|
||||
type FindingAssetTree struct {
|
||||
Nodes []FindingAssetNode `json:"nodes"`
|
||||
FindingTotal int `json:"finding_total"`
|
||||
// Truncated=true 表示为控制体积丢弃了 DroppedKinds 里的层级。
|
||||
Truncated bool `json:"truncated"`
|
||||
DroppedKinds []string `json:"dropped_kinds,omitempty"`
|
||||
}
|
||||
|
||||
// assetRow 是构树需要的资产字段子集。
|
||||
type assetRow struct {
|
||||
id int64
|
||||
kind string
|
||||
companyID int64
|
||||
domain string
|
||||
rootDomain string
|
||||
ip string
|
||||
url string
|
||||
port int
|
||||
serviceType string
|
||||
appName string
|
||||
}
|
||||
|
||||
func (a *assetRow) coverageNode() CoverageGraphNode {
|
||||
return CoverageGraphNode{
|
||||
Kind: a.kind, Domain: a.domain, RootDomain: a.rootDomain, IP: a.ip,
|
||||
URL: a.url, Port: a.port, ServiceType: a.serviceType, AppName: a.appName,
|
||||
}
|
||||
}
|
||||
|
||||
// label 复用覆盖图的标签规则(URL > domain > ip > app_name > root_domain)。
|
||||
func (a *assetRow) label() string {
|
||||
n := a.coverageNode()
|
||||
n.Key = assetKey(a.id)
|
||||
return coverageNodeLabel(&n)
|
||||
}
|
||||
|
||||
// hostPort 与覆盖图一致:优先 domain,其次 URL 里的 host,最后 ip。
|
||||
func (a *assetRow) hostPort() (string, int) {
|
||||
n := a.coverageNode()
|
||||
return hostPortOf(&n)
|
||||
}
|
||||
|
||||
const findingAssetSelectCols = `a.id, a.type, COALESCE(a.company_id,0),
|
||||
COALESCE(a.domain,''), COALESCE(a.root_domain,''), COALESCE(a.ip,''),
|
||||
COALESCE(a.url,''), COALESCE(a.port,0), COALESCE(a.service_type,''),
|
||||
COALESCE(a.app_name,'')`
|
||||
|
||||
func scanAssetRows(rows interface {
|
||||
Next() bool
|
||||
Scan(...any) error
|
||||
Err() error
|
||||
Close() error
|
||||
}) ([]*assetRow, error) {
|
||||
defer rows.Close()
|
||||
var out []*assetRow
|
||||
for rows.Next() {
|
||||
a := &assetRow{}
|
||||
if err := rows.Scan(&a.id, &a.kind, &a.companyID, &a.domain, &a.rootDomain,
|
||||
&a.ip, &a.url, &a.port, &a.serviceType, &a.appName); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, a)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// findingAssetHit 是一条发现在构树阶段需要的最小信息。
|
||||
type findingAssetHit struct {
|
||||
severity string
|
||||
ts time.Time
|
||||
assetIDs []int64
|
||||
}
|
||||
|
||||
// BuildFindingAssetTree 按当前筛选构建资产树。AssetScope 自身不参与(否则树会随
|
||||
// 选中节点塌缩成一条链)。
|
||||
func (d *DB) BuildFindingAssetTree(f FindingFilter) (*FindingAssetTree, error) {
|
||||
return d.buildFindingAssetTree(f, findingAssetTreeMaxNodes)
|
||||
}
|
||||
|
||||
// buildFindingAssetTree 是带节点上限的内部实现。maxNodes<=0 表示不截断——解析
|
||||
// AssetScope 时必须用这个模式,否则被丢掉的 endpoint 会让子树 id 集合不全。
|
||||
func (d *DB) buildFindingAssetTree(f FindingFilter, maxNodes int) (*FindingAssetTree, error) {
|
||||
f.AssetScope = ""
|
||||
f.assetIDs, f.assetNone, f.assetMiss = nil, false, false
|
||||
where, args := f.where()
|
||||
|
||||
rows, err := d.Query(`SELECT COALESCE(f.severity,''), f.created_at,
|
||||
COALESCE(f.asset_ids::text,'[]')
|
||||
FROM findings f LEFT JOIN tasks t ON f.task_id = t.id`+where, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hits := []findingAssetHit{}
|
||||
assetIDs := map[int64]bool{}
|
||||
for rows.Next() {
|
||||
var h findingAssetHit
|
||||
var aidsJSON string
|
||||
if err := rows.Scan(&h.severity, &h.ts, &aidsJSON); err != nil {
|
||||
rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
_ = json.Unmarshal([]byte(aidsJSON), &h.assetIDs)
|
||||
for _, id := range h.assetIDs {
|
||||
if id > 0 {
|
||||
assetIDs[id] = true
|
||||
}
|
||||
}
|
||||
hits = append(hits, h)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
rows.Close()
|
||||
|
||||
tree := &FindingAssetTree{Nodes: []FindingAssetNode{}, FindingTotal: len(hits)}
|
||||
byID, err := d.loadFindingAssetRows(assetIDs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
nodes, parentOf := d.assembleFindingAssetNodes(byID)
|
||||
if err := d.attachCompanyNodes(nodes, parentOf); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 计数:一条发现沿它每个资产的祖先链向上,收集去重后的 key 集合再逐个 +1,
|
||||
// 所以父节点不会因为一条发现挂了多个子资产而重复计数。
|
||||
unassigned := &FindingAssetNode{Key: FindingUnassignedAsset, Kind: "none", Label: "未关联资产"}
|
||||
touched := map[string]bool{}
|
||||
for _, h := range hits {
|
||||
clear(touched)
|
||||
var direct []*FindingAssetNode
|
||||
for _, id := range h.assetIDs {
|
||||
node := nodes[assetKey(id)]
|
||||
if node == nil {
|
||||
continue
|
||||
}
|
||||
direct = append(direct, node)
|
||||
for key := node.Key; key != ""; key = parentOf[key] {
|
||||
touched[key] = true
|
||||
}
|
||||
}
|
||||
if len(direct) == 0 {
|
||||
countFinding(unassigned, h)
|
||||
unassigned.Self++
|
||||
continue
|
||||
}
|
||||
for _, node := range direct {
|
||||
node.Self++
|
||||
}
|
||||
for key := range touched {
|
||||
countFinding(nodes[key], h)
|
||||
}
|
||||
}
|
||||
|
||||
for _, node := range nodes {
|
||||
if node.Total > 0 {
|
||||
tree.Nodes = append(tree.Nodes, *node)
|
||||
}
|
||||
}
|
||||
if unassigned.Total > 0 {
|
||||
tree.Nodes = append(tree.Nodes, *unassigned)
|
||||
}
|
||||
sortFindingAssetNodes(tree.Nodes)
|
||||
truncateFindingAssetTree(tree, maxNodes)
|
||||
return tree, nil
|
||||
}
|
||||
|
||||
// countFinding 把一条发现累加到节点上(总数 / 严重度分桶 / 最近发现时间)。
|
||||
func countFinding(n *FindingAssetNode, h findingAssetHit) {
|
||||
if n == nil {
|
||||
return
|
||||
}
|
||||
n.Total++
|
||||
switch h.severity {
|
||||
case "critical":
|
||||
n.Critical++
|
||||
case "high":
|
||||
n.High++
|
||||
case "medium":
|
||||
n.Medium++
|
||||
case "low":
|
||||
n.Low++
|
||||
}
|
||||
if h.ts.After(n.LastFoundAt) {
|
||||
n.LastFoundAt = h.ts
|
||||
}
|
||||
}
|
||||
|
||||
// loadFindingAssetRows 读取命中的资产行,并逐轮补齐祖先(service 的宿主域名/IP、
|
||||
// 子域名的根域名)。祖先自身可能没有任何发现,但树需要它们才能成形。
|
||||
func (d *DB) loadFindingAssetRows(ids map[int64]bool) (map[int64]*assetRow, error) {
|
||||
byID := map[int64]*assetRow{}
|
||||
if len(ids) == 0 {
|
||||
return byID, nil
|
||||
}
|
||||
idList := make([]int64, 0, len(ids))
|
||||
for id := range ids {
|
||||
idList = append(idList, id)
|
||||
}
|
||||
rows, err := d.Query(`SELECT `+findingAssetSelectCols+` FROM assets a WHERE a.id = ANY($1::bigint[])`, idList)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
found, err := scanAssetRows(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, a := range found {
|
||||
byID[a.id] = a
|
||||
}
|
||||
|
||||
// 每轮找出还缺父节点的宿主标识,批量补一层;层数固定(endpoint→service→
|
||||
// subdomain/ip→root_domain),4 轮足够收敛。
|
||||
for range 4 {
|
||||
want := missingParents(byID)
|
||||
if want.empty() {
|
||||
break
|
||||
}
|
||||
added, err := d.loadAssetsByHost(want, byID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if added == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
return byID, nil
|
||||
}
|
||||
|
||||
// missingHosts 是一轮补齐里要去库里找的宿主标识,按目标资产类型分开。
|
||||
type missingHosts struct {
|
||||
services []string // endpoint 的宿主(找 service 行)
|
||||
domains []string // service/endpoint 的宿主域名(找 subdomain 行)
|
||||
ips []string // service/endpoint 的宿主 IP(找 ip 行)
|
||||
roots []string // 子域名的根域名(找 root_domain 行)
|
||||
}
|
||||
|
||||
func (m missingHosts) empty() bool {
|
||||
return len(m.services) == 0 && len(m.domains) == 0 && len(m.ips) == 0 && len(m.roots) == 0
|
||||
}
|
||||
|
||||
// missingParents 汇总还没被加载的宿主:service(供 endpoint 挂靠)、子域名/IP(供
|
||||
// service 与 endpoint 挂靠)与根域名(供子域名挂靠)。
|
||||
func missingParents(byID map[int64]*assetRow) missingHosts {
|
||||
haveService := map[string]bool{}
|
||||
haveDomain := map[string]bool{}
|
||||
haveIP := map[string]bool{}
|
||||
haveRoot := map[string]bool{}
|
||||
for _, a := range byID {
|
||||
switch a.kind {
|
||||
case "service":
|
||||
if host, _ := a.hostPort(); host != "" {
|
||||
haveService[host] = true
|
||||
}
|
||||
case "subdomain":
|
||||
haveDomain[a.domain] = true
|
||||
case "ip":
|
||||
haveIP[a.ip] = true
|
||||
case "root_domain":
|
||||
haveRoot[a.domain] = true
|
||||
}
|
||||
}
|
||||
wantService := map[string]bool{}
|
||||
wantDomain := map[string]bool{}
|
||||
wantIP := map[string]bool{}
|
||||
wantRoot := map[string]bool{}
|
||||
for _, a := range byID {
|
||||
switch a.kind {
|
||||
case "service", "endpoint":
|
||||
host, _ := a.hostPort()
|
||||
// endpoint 先找同宿主的 service;端口对不上的 service 会以 Total=0
|
||||
// 被最终过滤掉,不会污染树。
|
||||
if a.kind == "endpoint" && host != "" && !haveService[host] {
|
||||
wantService[host] = true
|
||||
}
|
||||
if host != "" && !haveDomain[host] && !haveIP[host] && !haveRoot[host] {
|
||||
if isIPLiteral(host) {
|
||||
wantIP[host] = true
|
||||
} else {
|
||||
wantDomain[host] = true
|
||||
}
|
||||
}
|
||||
if a.ip != "" && !haveIP[a.ip] {
|
||||
wantIP[a.ip] = true
|
||||
}
|
||||
case "subdomain":
|
||||
if a.rootDomain != "" && !haveRoot[a.rootDomain] {
|
||||
wantRoot[a.rootDomain] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return missingHosts{
|
||||
services: keysOf(wantService),
|
||||
domains: keysOf(wantDomain),
|
||||
ips: keysOf(wantIP),
|
||||
roots: keysOf(wantRoot),
|
||||
}
|
||||
}
|
||||
|
||||
func keysOf(m map[string]bool) []string {
|
||||
if len(m) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// isIPLiteral 粗判一个 host 是不是 IP 字面量(用于决定去 ip 还是 subdomain 表找宿主)。
|
||||
func isIPLiteral(host string) bool {
|
||||
if strings.Contains(host, ":") {
|
||||
return true // IPv6
|
||||
}
|
||||
if host == "" {
|
||||
return false
|
||||
}
|
||||
for _, part := range strings.Split(host, ".") {
|
||||
if part == "" {
|
||||
return false
|
||||
}
|
||||
if _, err := strconv.Atoi(part); err != nil {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return strings.Count(host, ".") == 3
|
||||
}
|
||||
|
||||
// loadAssetsByHost 按宿主标识批量补齐资产行,返回本轮新增的行数。
|
||||
func (d *DB) loadAssetsByHost(want missingHosts, byID map[int64]*assetRow) (int, error) {
|
||||
added := 0
|
||||
load := func(q string, arg []string) error {
|
||||
if len(arg) == 0 {
|
||||
return nil
|
||||
}
|
||||
rows, err := d.Query(q, arg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
found, err := scanAssetRows(rows)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, a := range found {
|
||||
if _, ok := byID[a.id]; ok {
|
||||
continue
|
||||
}
|
||||
byID[a.id] = a
|
||||
added++
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if err := load(`SELECT `+findingAssetSelectCols+` FROM assets a
|
||||
WHERE a.type='service' AND (a.domain = ANY($1::text[]) OR a.ip = ANY($1::text[]))`, want.services); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := load(`SELECT `+findingAssetSelectCols+` FROM assets a
|
||||
WHERE a.type='subdomain' AND a.domain = ANY($1::text[])`, want.domains); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := load(`SELECT `+findingAssetSelectCols+` FROM assets a
|
||||
WHERE a.type='ip' AND a.ip = ANY($1::text[])`, want.ips); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := load(`SELECT `+findingAssetSelectCols+` FROM assets a
|
||||
WHERE a.type='root_domain' AND a.domain = ANY($1::text[])`, want.roots); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return added, nil
|
||||
}
|
||||
|
||||
// assembleFindingAssetNodes 把资产行变成节点并连上父子关系。父节点缺位时(库里
|
||||
// 根本没有那条根域名资产)合成 "r:<domain>" 占位节点,与覆盖图的处理一致。
|
||||
func (d *DB) assembleFindingAssetNodes(byID map[int64]*assetRow) (map[string]*FindingAssetNode, map[string]string) {
|
||||
nodes := map[string]*FindingAssetNode{}
|
||||
parentOf := map[string]string{}
|
||||
rootByDomain := map[string]string{}
|
||||
subByDomain := map[string]string{}
|
||||
ipByAddr := map[string]string{}
|
||||
svcByHost := map[string]string{}
|
||||
svcByHostPort := map[string]string{}
|
||||
|
||||
for _, a := range byID {
|
||||
key := assetKey(a.id)
|
||||
nodes[key] = &FindingAssetNode{
|
||||
Key: key, Kind: a.kind, Label: a.label(),
|
||||
AssetID: a.id, CompanyID: a.companyID,
|
||||
}
|
||||
switch a.kind {
|
||||
case "root_domain":
|
||||
if a.domain != "" {
|
||||
rootByDomain[a.domain] = key
|
||||
}
|
||||
case "subdomain":
|
||||
if a.domain != "" {
|
||||
subByDomain[a.domain] = key
|
||||
}
|
||||
case "ip":
|
||||
if a.ip != "" {
|
||||
ipByAddr[a.ip] = key
|
||||
}
|
||||
case "service":
|
||||
if host, port := a.hostPort(); host != "" {
|
||||
svcByHost[host] = key
|
||||
svcByHostPort[host+"|"+strconv.Itoa(port)] = key
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 子域名的根域名在库里没有资产行时,合成一个占位根,免得子域名散成顶层。
|
||||
for _, a := range byID {
|
||||
if a.kind != "subdomain" || a.rootDomain == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := rootByDomain[a.rootDomain]; ok {
|
||||
continue
|
||||
}
|
||||
key := "r:" + a.rootDomain
|
||||
nodes[key] = &FindingAssetNode{Key: key, Kind: "root_domain", Label: a.rootDomain}
|
||||
rootByDomain[a.rootDomain] = key
|
||||
}
|
||||
|
||||
firstOf := func(keys ...string) string {
|
||||
for _, k := range keys {
|
||||
if k != "" {
|
||||
if _, ok := nodes[k]; ok {
|
||||
return k
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
for _, a := range byID {
|
||||
key := assetKey(a.id)
|
||||
var parent string
|
||||
switch a.kind {
|
||||
case "subdomain":
|
||||
parent = firstOf(rootByDomain[a.rootDomain])
|
||||
case "service":
|
||||
host, _ := a.hostPort()
|
||||
parent = firstOf(subByDomain[a.domain], subByDomain[host],
|
||||
ipByAddr[a.ip], ipByAddr[host], rootByDomain[a.rootDomain], rootByDomain[host])
|
||||
case "endpoint":
|
||||
host, port := a.hostPort()
|
||||
parent = firstOf(svcByHostPort[host+"|"+strconv.Itoa(port)], svcByHost[host],
|
||||
subByDomain[host], subByDomain[a.domain], ipByAddr[host], ipByAddr[a.ip],
|
||||
rootByDomain[a.rootDomain], rootByDomain[host])
|
||||
}
|
||||
if parent != "" && parent != key {
|
||||
parentOf[key] = parent
|
||||
nodes[key].Parent = parent
|
||||
}
|
||||
}
|
||||
return nodes, parentOf
|
||||
}
|
||||
|
||||
// attachCompanyNodes 给顶层资产(根域名 / IP / 应用)补企业父节点——只有资产确实
|
||||
// 归属了企业才会出现企业层,没归属的资产仍然自己就是顶层。
|
||||
func (d *DB) attachCompanyNodes(nodes map[string]*FindingAssetNode, parentOf map[string]string) error {
|
||||
want := map[int64]bool{}
|
||||
for _, n := range nodes {
|
||||
if n.Parent != "" || n.CompanyID <= 0 {
|
||||
continue
|
||||
}
|
||||
switch n.Kind {
|
||||
case "root_domain", "ip", "app":
|
||||
want[n.CompanyID] = true
|
||||
}
|
||||
}
|
||||
if len(want) == 0 {
|
||||
return nil
|
||||
}
|
||||
ids := make([]int64, 0, len(want))
|
||||
for id := range want {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
rows, err := d.Query(`SELECT id, COALESCE(name,'') FROM companies WHERE id = ANY($1::bigint[])`, ids)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
names := map[int64]string{}
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
var name string
|
||||
if err := rows.Scan(&id, &name); err != nil {
|
||||
return err
|
||||
}
|
||||
names[id] = name
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
for id, name := range names {
|
||||
key := companyKey(id)
|
||||
if _, ok := nodes[key]; ok {
|
||||
continue
|
||||
}
|
||||
if name == "" {
|
||||
name = "企业 #" + strconv.FormatInt(id, 10)
|
||||
}
|
||||
nodes[key] = &FindingAssetNode{Key: key, Kind: "company", Label: name, CompanyID: id}
|
||||
}
|
||||
for _, n := range nodes {
|
||||
if n.Parent != "" || n.CompanyID <= 0 || n.Kind == "company" {
|
||||
continue
|
||||
}
|
||||
switch n.Kind {
|
||||
case "root_domain", "ip", "app":
|
||||
key := companyKey(n.CompanyID)
|
||||
if _, ok := nodes[key]; !ok {
|
||||
continue
|
||||
}
|
||||
n.Parent = key
|
||||
parentOf[n.Key] = key
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// sortFindingAssetNodes 排序:发现多的在前,同数按标签;「未关联资产」恒在最后。
|
||||
// 前端按数组顺序挂子节点,所以只要同一父节点下的相对顺序正确即可。
|
||||
func sortFindingAssetNodes(nodes []FindingAssetNode) {
|
||||
sort.SliceStable(nodes, func(i, j int) bool {
|
||||
a, b := nodes[i], nodes[j]
|
||||
if (a.Kind == "none") != (b.Kind == "none") {
|
||||
return b.Kind == "none"
|
||||
}
|
||||
if a.Total != b.Total {
|
||||
return a.Total > b.Total
|
||||
}
|
||||
return a.Label < b.Label
|
||||
})
|
||||
}
|
||||
|
||||
// truncateFindingAssetTree 在节点过多时整层丢弃(先 endpoint 再 service)。计数已
|
||||
// 累加到父节点,丢的只是可展开的细节层级。
|
||||
func truncateFindingAssetTree(tree *FindingAssetTree, maxNodes int) {
|
||||
if maxNodes <= 0 || len(tree.Nodes) <= maxNodes {
|
||||
return
|
||||
}
|
||||
for _, kind := range []string{"endpoint", "service"} {
|
||||
kept := tree.Nodes[:0]
|
||||
for _, n := range tree.Nodes {
|
||||
if n.Kind == kind {
|
||||
continue
|
||||
}
|
||||
kept = append(kept, n)
|
||||
}
|
||||
tree.Nodes = kept
|
||||
tree.Truncated = true
|
||||
tree.DroppedKinds = append(tree.DroppedKinds, kind)
|
||||
if len(tree.Nodes) <= maxNodes {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// applyAssetScope 把 AssetScope(节点 key)解析成可用于 SQL 的资产 id 集合。选中
|
||||
// 一个节点等于选中它的整棵子树,所以要先把树建出来再收集子孙。
|
||||
func (d *DB) applyAssetScope(f FindingFilter) (FindingFilter, error) {
|
||||
scope := strings.TrimSpace(f.AssetScope)
|
||||
f.assetIDs, f.assetNone, f.assetMiss = nil, false, false
|
||||
if scope == "" {
|
||||
return f, nil
|
||||
}
|
||||
if scope == FindingUnassignedAsset {
|
||||
f.assetNone = true
|
||||
return f, nil
|
||||
}
|
||||
// 不截断:被丢掉的 endpoint 同样要参与 id 收集,否则列表会少数据。
|
||||
tree, err := d.buildFindingAssetTree(f, 0)
|
||||
if err != nil {
|
||||
return f, err
|
||||
}
|
||||
children := map[string][]FindingAssetNode{}
|
||||
byKey := map[string]FindingAssetNode{}
|
||||
for _, n := range tree.Nodes {
|
||||
byKey[n.Key] = n
|
||||
children[n.Parent] = append(children[n.Parent], n)
|
||||
}
|
||||
if _, ok := byKey[scope]; !ok {
|
||||
// 选中的节点在当前筛选下已经不存在,结果应当为空而不是退化成不过滤。
|
||||
f.assetMiss = true
|
||||
return f, nil
|
||||
}
|
||||
seen := map[string]bool{scope: true}
|
||||
queue := []string{scope}
|
||||
for len(queue) > 0 {
|
||||
key := queue[0]
|
||||
queue = queue[1:]
|
||||
if id := byKey[key].AssetID; id > 0 {
|
||||
f.assetIDs = append(f.assetIDs, id)
|
||||
}
|
||||
for _, child := range children[key] {
|
||||
if seen[child.Key] {
|
||||
continue
|
||||
}
|
||||
seen[child.Key] = true
|
||||
queue = append(queue, child.Key)
|
||||
}
|
||||
}
|
||||
if len(f.assetIDs) == 0 {
|
||||
f.assetMiss = true
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// assetIDContainments 把资产 id 变成 jsonb 包含判断的右操作数集合,配合
|
||||
// idx_findings_asset_ids(GIN jsonb_path_ops)使用。
|
||||
func assetIDContainments(ids []int64) []string {
|
||||
out := make([]string, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
out = append(out, "["+strconv.FormatInt(id, 10)+"]")
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,233 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// cleanupTreeFixtures 删除一个用例造出来的资产与发现。必须用 defer 注册(而不是
|
||||
// t.Cleanup):t.Cleanup 跑在测试函数返回之后,那时 defer d.Close() 已经把连接关了,
|
||||
// 清理会静默失败并把脏数据留在共享开发库里。
|
||||
func cleanupTreeFixtures(d *DB, taskID int64, rootDomains ...string) {
|
||||
d.Exec(`DELETE FROM assets WHERE root_domain = ANY($1::text[])`, rootDomains) //nolint:errcheck
|
||||
d.DeleteFindingsByTask(taskID) //nolint:errcheck
|
||||
}
|
||||
|
||||
// seedTreeAsset inserts one asset row.
|
||||
func seedTreeAsset(t *testing.T, d *DB, kind string, cols map[string]any) int64 {
|
||||
t.Helper()
|
||||
names := []string{"type"}
|
||||
values := []any{kind}
|
||||
placeholders := []string{"$1"}
|
||||
for k, v := range cols {
|
||||
values = append(values, v)
|
||||
names = append(names, k)
|
||||
placeholders = append(placeholders, "$"+strconv.Itoa(len(values)))
|
||||
}
|
||||
q := "INSERT INTO assets(" + strings.Join(names, ",") + ") VALUES (" +
|
||||
strings.Join(placeholders, ",") + ") RETURNING id"
|
||||
var id int64
|
||||
if err := d.QueryRow(q, values...).Scan(&id); err != nil {
|
||||
t.Fatalf("seed %s asset: %v", kind, err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
func nodeByKey(tree *FindingAssetTree, key string) *FindingAssetNode {
|
||||
for i := range tree.Nodes {
|
||||
if tree.Nodes[i].Key == key {
|
||||
return &tree.Nodes[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// TestBuildFindingAssetTree covers the whole shape of the「按资产」tree: the
|
||||
// root→subdomain→service→endpoint chain gets rebuilt from a finding that only
|
||||
// points at the leaf, ancestors aggregate their subtree, assets without any
|
||||
// finding stay out, and a finding whose asset row is gone lands in the
|
||||
// unassigned bucket.
|
||||
func TestBuildFindingAssetTree(t *testing.T) {
|
||||
d, err := Open(testDSN(t))
|
||||
if err != nil {
|
||||
t.Skipf("postgres unavailable (%v) — skipping", err)
|
||||
}
|
||||
defer d.Close()
|
||||
|
||||
tk, err := d.CreateTask("资产树测试", "目标", nil, 0, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.DeleteTask(tk.ID)
|
||||
|
||||
const root = "tree-test.example"
|
||||
const sub = "api.tree-test.example"
|
||||
defer cleanupTreeFixtures(d, tk.ID, root)
|
||||
rootID := seedTreeAsset(t, d, "root_domain", map[string]any{"domain": root, "root_domain": root})
|
||||
subID := seedTreeAsset(t, d, "subdomain", map[string]any{"domain": sub, "root_domain": root})
|
||||
svcID := seedTreeAsset(t, d, "service", map[string]any{
|
||||
"domain": sub, "root_domain": root, "url": "https://" + sub, "port": 443, "service_type": "http",
|
||||
})
|
||||
epID := seedTreeAsset(t, d, "endpoint", map[string]any{
|
||||
"domain": sub, "root_domain": root, "url": "https://" + sub + "/admin", "port": 443, "method": "GET",
|
||||
})
|
||||
// 同域名下另一个服务,不挂任何发现 —— 不应出现在树里。
|
||||
seedTreeAsset(t, d, "service", map[string]any{
|
||||
"domain": sub, "root_domain": root, "url": "http://" + sub + ":8080", "port": 8080, "service_type": "http",
|
||||
})
|
||||
|
||||
// 只把发现挂在最深的 endpoint 上,祖先链要靠构树自己补出来。
|
||||
if _, err := d.AddFinding(tk.ID, 0, "XSS", "反射型 XSS", "high", "s", "e", "w", []int64{epID}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// 直接挂在服务上的一条,用来验证 Self 与 Total 的区别。
|
||||
if _, err := d.AddFinding(tk.ID, 0, "Info", "信息泄露", "low", "s", "e", "w", []int64{svcID}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// 资产行不存在(已删除资产)→ 未关联桶。
|
||||
if _, err := d.AddFinding(tk.ID, 0, "Misc", "孤儿", "medium", "s", "e", "w", []int64{999000111}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
tree, err := d.BuildFindingAssetTree(FindingFilter{TaskID: strconv.FormatInt(tk.ID, 10)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if tree.FindingTotal != 3 {
|
||||
t.Fatalf("finding_total: want 3, got %d", tree.FindingTotal)
|
||||
}
|
||||
|
||||
rootNode := nodeByKey(tree, assetKey(rootID))
|
||||
subNode := nodeByKey(tree, assetKey(subID))
|
||||
svcNode := nodeByKey(tree, assetKey(svcID))
|
||||
epNode := nodeByKey(tree, assetKey(epID))
|
||||
for name, n := range map[string]*FindingAssetNode{
|
||||
"root": rootNode, "subdomain": subNode, "service": svcNode, "endpoint": epNode,
|
||||
} {
|
||||
if n == nil {
|
||||
t.Fatalf("%s node missing from tree", name)
|
||||
}
|
||||
}
|
||||
|
||||
// 父子链:endpoint → service → subdomain → root_domain。
|
||||
if epNode.Parent != svcNode.Key {
|
||||
t.Errorf("endpoint parent: want %s, got %s", svcNode.Key, epNode.Parent)
|
||||
}
|
||||
if svcNode.Parent != subNode.Key {
|
||||
t.Errorf("service parent: want %s, got %s", subNode.Key, svcNode.Parent)
|
||||
}
|
||||
if subNode.Parent != rootNode.Key {
|
||||
t.Errorf("subdomain parent: want %s, got %s", rootNode.Key, subNode.Parent)
|
||||
}
|
||||
if rootNode.Parent != "" {
|
||||
t.Errorf("root parent: want top level, got %s", rootNode.Parent)
|
||||
}
|
||||
|
||||
// 聚合:根域名两条(endpoint 的 high + service 的 low),service 自身一条、子树两条。
|
||||
if rootNode.Total != 2 || rootNode.High != 1 || rootNode.Low != 1 {
|
||||
t.Errorf("root totals: want 2/high1/low1, got %d/high%d/low%d", rootNode.Total, rootNode.High, rootNode.Low)
|
||||
}
|
||||
if rootNode.Self != 0 {
|
||||
t.Errorf("root self: want 0 (只是祖先), got %d", rootNode.Self)
|
||||
}
|
||||
if svcNode.Total != 2 || svcNode.Self != 1 {
|
||||
t.Errorf("service total/self: want 2/1, got %d/%d", svcNode.Total, svcNode.Self)
|
||||
}
|
||||
if epNode.Total != 1 || epNode.Self != 1 {
|
||||
t.Errorf("endpoint total/self: want 1/1, got %d/%d", epNode.Total, epNode.Self)
|
||||
}
|
||||
|
||||
// 没有发现的兄弟服务不进树。
|
||||
for _, n := range tree.Nodes {
|
||||
if n.Label == "http://"+sub+":8080" {
|
||||
t.Errorf("asset without findings should be hidden: %+v", n)
|
||||
}
|
||||
}
|
||||
|
||||
// 未关联桶收下那条指向已删资产的发现。
|
||||
none := nodeByKey(tree, FindingUnassignedAsset)
|
||||
if none == nil || none.Total != 1 || none.Medium != 1 {
|
||||
t.Fatalf("unassigned bucket: want 1 medium, got %+v", none)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFindingAssetScopeFilter verifies选中一个节点 narrows the findings list to
|
||||
// that node's whole subtree, and that the unassigned sentinel works too.
|
||||
func TestFindingAssetScopeFilter(t *testing.T) {
|
||||
d, err := Open(testDSN(t))
|
||||
if err != nil {
|
||||
t.Skipf("postgres unavailable (%v) — skipping", err)
|
||||
}
|
||||
defer d.Close()
|
||||
|
||||
tk, err := d.CreateTask("资产筛选测试", "目标", nil, 0, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.DeleteTask(tk.ID)
|
||||
|
||||
const root = "scope-test.example"
|
||||
const sub = "api.scope-test.example"
|
||||
const other = "other-scope-test.example"
|
||||
defer cleanupTreeFixtures(d, tk.ID, root, other)
|
||||
rootID := seedTreeAsset(t, d, "root_domain", map[string]any{"domain": root, "root_domain": root})
|
||||
subID := seedTreeAsset(t, d, "subdomain", map[string]any{"domain": sub, "root_domain": root})
|
||||
otherID := seedTreeAsset(t, d, "root_domain", map[string]any{"domain": other, "root_domain": other})
|
||||
|
||||
if _, err := d.AddFinding(tk.ID, 0, "A", "子域名上的", "high", "s", "e", "w", []int64{subID}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := d.AddFinding(tk.ID, 0, "B", "别的根域名上的", "high", "s", "e", "w", []int64{otherID}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := d.AddFinding(tk.ID, 0, "C", "没有资产的", "high", "s", "e", "w", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// 指向已删除资产的发现,和 asset_ids 为空的一样属于「未关联」——树的桶收下它,
|
||||
// 列表筛选也必须查得出来,两处口径不一致会让桶上的数字大于点开后的条数。
|
||||
if _, err := d.AddFinding(tk.ID, 0, "D", "资产已删除", "high", "s", "e", "w", []int64{999000333}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
base := FindingFilter{TaskID: strconv.FormatInt(tk.ID, 10)}
|
||||
cases := []struct {
|
||||
name string
|
||||
scope string
|
||||
want int
|
||||
}{
|
||||
{"整棵子树", assetKey(rootID), 1}, // 根域名下只有子域名那条
|
||||
{"叶子节点", assetKey(subID), 1}, // 子域名自身
|
||||
{"另一棵树", assetKey(otherID), 1}, // 互不串味
|
||||
{"未关联", FindingUnassignedAsset, 2}, // asset_ids 为空的 + 指向已删资产的
|
||||
{"不存在的节点", "a:999000222", 0}, // 当前筛选下没有该节点 → 空结果,不是不过滤
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
f := base
|
||||
f.AssetScope = tc.scope
|
||||
items, total, err := d.ListFindingsPage(f, 1, 50)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if total != tc.want || len(items) != tc.want {
|
||||
t.Fatalf("scope %s: want %d findings, got total=%d items=%d", tc.scope, tc.want, total, len(items))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// 不带 scope 时四条都在。
|
||||
if _, total, err := d.ListFindingsPage(base, 1, 50); err != nil || total != 4 {
|
||||
t.Fatalf("unscoped: want 4, got %d (%v)", total, err)
|
||||
}
|
||||
|
||||
// 树上未关联桶的计数必须与点开后查到的条数一致 —— 这正是两处口径分家时会崩的断言。
|
||||
tree, err := d.BuildFindingAssetTree(base)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
none := nodeByKey(tree, FindingUnassignedAsset)
|
||||
if none == nil || none.Total != 2 {
|
||||
t.Fatalf("unassigned bucket count: want 2, got %+v", none)
|
||||
}
|
||||
}
|
||||
+40
-1
@@ -148,6 +148,14 @@ type FindingFilter struct {
|
||||
TaskID string // 任务 id(字符串形式;空/非法 = 不按任务筛选)
|
||||
Query string // 名称/类型/摘要/证据/报告正文的模糊检索关键词
|
||||
Sort string // "severity" | "time"
|
||||
// AssetScope 是资产树的节点 key(a:<id> / c:<id> / r:<domain> / __none__),
|
||||
// 选中一个节点等于选中它的整棵子树。空 = 不按资产筛选。
|
||||
AssetScope string
|
||||
|
||||
// 下面三个由 applyAssetScope 从 AssetScope 解析而来,调用方不用设置。
|
||||
assetIDs []int64 // 子树里所有资产 id
|
||||
assetNone bool // 只要「未关联资产」的发现
|
||||
assetMiss bool // 选中的节点在当前筛选下不存在 → 结果恒空
|
||||
}
|
||||
|
||||
// FindingUnassignedTask is the task filter sentinel for findings whose task is
|
||||
@@ -178,6 +186,25 @@ func (f FindingFilter) where() (string, []any) {
|
||||
args = append(args, tid)
|
||||
conds = append(conds, fmt.Sprintf("f.task_id = $%d", len(args)))
|
||||
}
|
||||
// 资产筛选:asset_ids 是 jsonb 数组,@> ANY(...) 能走 idx_findings_asset_ids。
|
||||
switch {
|
||||
case f.assetMiss:
|
||||
conds = append(conds, "FALSE")
|
||||
case f.assetNone:
|
||||
// 「未关联资产」= asset_ids 为空,或者里面的 id 一个都不在 assets 表里
|
||||
// (资产已被删除)。两类都进资产树的未关联桶,这里必须同样收下,否则桶上
|
||||
// 的计数会大于点开后能查到的条数。
|
||||
conds = append(conds, `(
|
||||
jsonb_array_length(COALESCE(f.asset_ids, '[]'::jsonb)) = 0
|
||||
OR NOT EXISTS (
|
||||
SELECT 1 FROM jsonb_array_elements_text(f.asset_ids) e(v)
|
||||
JOIN assets a ON a.id = e.v::bigint
|
||||
)
|
||||
)`)
|
||||
case len(f.assetIDs) > 0:
|
||||
args = append(args, assetIDContainments(f.assetIDs))
|
||||
conds = append(conds, fmt.Sprintf("f.asset_ids @> ANY($%d::jsonb[])", len(args)))
|
||||
}
|
||||
if query := strings.TrimSpace(f.Query); query != "" {
|
||||
escaped := strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`).Replace(query)
|
||||
args = append(args, "%"+escaped+"%")
|
||||
@@ -205,6 +232,10 @@ func (d *DB) ListFindingsPage(f FindingFilter, page, pageSize int) ([]*DBFinding
|
||||
if pageSize <= 0 {
|
||||
pageSize = 20
|
||||
}
|
||||
f, err := d.applyAssetScope(f)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
where, args := f.where()
|
||||
|
||||
var total int
|
||||
@@ -264,6 +295,10 @@ func (d *DB) ListFindingGroups(f FindingFilter, page, pageSize int) ([]FindingGr
|
||||
if pageSize <= 0 {
|
||||
pageSize = 10
|
||||
}
|
||||
f, err := d.applyAssetScope(f)
|
||||
if err != nil {
|
||||
return nil, 0, 0, err
|
||||
}
|
||||
where, args := f.where()
|
||||
grouped := ` FROM findings f LEFT JOIN tasks t ON f.task_id=t.id` + where +
|
||||
` GROUP BY t.id, t.name, t.description, t.status, t.paused, t.queued`
|
||||
@@ -444,7 +479,11 @@ func (d *DB) ListFindingsForExport(f FindingFilter, ids []int64) ([]*DBFinding,
|
||||
WHERE f.id IN (` + strings.Join(ph, ",") + `)
|
||||
` + order
|
||||
} else {
|
||||
where, wargs := f.where()
|
||||
scoped, err := d.applyAssetScope(f)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
where, wargs := scoped.where()
|
||||
q = `SELECT ` + cols + `
|
||||
FROM findings f
|
||||
LEFT JOIN tasks t ON f.task_id = t.id` + where + `
|
||||
|
||||
@@ -960,6 +960,8 @@ ALTER TABLE findings ADD COLUMN IF NOT EXISTS report TEXT NOT NULL DEFAULT '';
|
||||
CREATE INDEX IF NOT EXISTS idx_findings_task ON findings(task_id, created_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_findings_time ON findings(created_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_findings_status ON findings(status, created_at DESC);
|
||||
-- 「按资产」视图靠 asset_ids @> '[<id>]' 反查发现,没有这个 GIN 索引就是全表扫。
|
||||
CREATE INDEX IF NOT EXISTS idx_findings_asset_ids ON findings USING GIN(asset_ids jsonb_path_ops);
|
||||
|
||||
-- =====================================================================
|
||||
-- M. 后端日志持久化
|
||||
|
||||
Reference in new issue
Block a user