From 8e9b126972a7d27fa64b49d14a7f4701ce1f53fd Mon Sep 17 00:00:00 2001 From: Autumn-27 <53844290+Autumn-27@users.noreply.github.com> Date: Wed, 19 Aug 2026 09:17:13 -0400 Subject: [PATCH] =?UTF-8?q?feat(assets):=20=E4=BB=BB=E5=8A=A1/=E4=BC=81?= =?UTF-8?q?=E4=B8=9A=E8=B5=84=E4=BA=A7=E5=88=97=E8=A1=A8=E6=94=B9=E6=9C=8D?= =?UTF-8?q?=E5=8A=A1=E7=AB=AF=E5=88=86=E9=A1=B5,=E5=8E=BB=E6=8E=89=2050=20?= =?UTF-8?q?=E6=9D=A1=E4=B8=8A=E9=99=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - db: QueryByTask/QueryByCompany 加 offset 参数,limit<=0 不再兜底成 50/200 而是不限量;新增 CountByTask/CountByCompany/CountsByTypeForTask; 抽出共享 pageClause 与 scanTypeCounts。 - server: /assets 的 task_id/company_id 分支改用真实 total(不再是 len(assets));/assets/counts 支持 ?task_id= 返回按类型计数。 - web: taskAssets(taskId,type,limit,offset) 返回 {assets,total}; assetCounts(taskId?);assets-tab 改为只取当前分类当前页,分类徽章走 counts 接口,分页器 100/200 真正可用,轮询变轻。 - 补 TestQueryByTaskPaging/TestQueryByCompanyPaging 回归(60 条 25/页翻页 不重不漏)。 --- db/assets.go | 76 ++++++++-- db/assets_test.go | 143 +++++++++++++++++- server/assets.go | 27 +++- .../tasks/detail/_tabs/assets-tab.tsx | 72 +++++---- web/src/lib/api.ts | 12 +- 5 files changed, 268 insertions(+), 62 deletions(-) diff --git a/db/assets.go b/db/assets.go index fee5d94..3cf1c1b 100644 --- a/db/assets.go +++ b/db/assets.go @@ -913,10 +913,8 @@ func (s *AssetStore) CountByType(typ string) (int, error) { } // QueryByCompany returns assets for a company, optionally filtered by type. -func (s *AssetStore) QueryByCompany(companyID int64, typ string, limit int) ([]*Asset, error) { - if limit <= 0 { - limit = 200 - } +// limit <= 0 means no limit — pair it with CountByCompany for server-side paging. +func (s *AssetStore) QueryByCompany(companyID int64, typ string, limit, offset int) ([]*Asset, error) { q := `SELECT id, type, company_id, array_to_json(task_ids)::text, COALESCE(domain,''), COALESCE(root_domain,''), COALESCE(ip,''), COALESCE(c_segment::text,''), port, @@ -933,8 +931,7 @@ FROM assets WHERE company_id = $1` args = append(args, typ) q += fmt.Sprintf(` AND type = $%d`, len(args)) } - args = append(args, limit) - q += fmt.Sprintf(` ORDER BY last_seen DESC LIMIT $%d`, len(args)) + q += pageClause(&args, limit, offset) rows, err := s.db.Query(q, args...) if err != nil { return nil, err @@ -943,11 +940,38 @@ FROM assets WHERE company_id = $1` return scanAssets(rows) } -// QueryByTask returns assets rows that have a given task_id in task_ids. -func (s *AssetStore) QueryByTask(taskID int64, typ string, limit int) ([]*Asset, error) { - if limit <= 0 { - limit = 200 +// CountByCompany returns the full number of assets attributed to a company +// (optionally filtered by type), ignoring limit/offset. +func (s *AssetStore) CountByCompany(companyID int64, typ string) (int, error) { + q := `SELECT count(*) FROM assets WHERE company_id = $1` + args := []any{companyID} + if typ != "" { + args = append(args, typ) + q += fmt.Sprintf(` AND type = $%d`, len(args)) } + var n int + err := s.db.QueryRow(q, args...).Scan(&n) + return n, err +} + +// pageClause appends limit/offset placeholders to args and returns the trailing +// ORDER BY … LIMIT … OFFSET fragment. limit <= 0 means no limit. +func pageClause(args *[]any, limit, offset int) string { + q := ` ORDER BY last_seen DESC` + if limit > 0 { + *args = append(*args, limit) + q += fmt.Sprintf(` LIMIT $%d`, len(*args)) + } + if offset > 0 { + *args = append(*args, offset) + q += fmt.Sprintf(` OFFSET $%d`, len(*args)) + } + return q +} + +// QueryByTask returns assets rows that have a given task_id in task_ids. +// limit <= 0 means no limit — pair it with CountByTask for server-side paging. +func (s *AssetStore) QueryByTask(taskID int64, typ string, limit, offset int) ([]*Asset, error) { q := `SELECT id, type, company_id, array_to_json(task_ids)::text, COALESCE(domain,''), COALESCE(root_domain,''), COALESCE(ip,''), COALESCE(c_segment::text,''), port, @@ -964,8 +988,7 @@ FROM assets WHERE $1 = ANY(task_ids)` args = append(args, typ) q += fmt.Sprintf(` AND type = $%d`, len(args)) } - args = append(args, limit) - q += fmt.Sprintf(` ORDER BY last_seen DESC LIMIT $%d`, len(args)) + q += pageClause(&args, limit, offset) rows, err := s.db.Query(q, args...) if err != nil { return nil, err @@ -974,6 +997,30 @@ FROM assets WHERE $1 = ANY(task_ids)` return scanAssets(rows) } +// CountByTask returns the full number of assets attached to a task (optionally +// filtered by type), ignoring limit/offset — lets the UI paginate on the server. +func (s *AssetStore) CountByTask(taskID int64, typ string) (int, error) { + q := `SELECT count(*) FROM assets WHERE $1 = ANY(task_ids)` + args := []any{taskID} + if typ != "" { + args = append(args, typ) + q += fmt.Sprintf(` AND type = $%d`, len(args)) + } + var n int + err := s.db.QueryRow(q, args...).Scan(&n) + return n, err +} + +// CountsByTypeForTask returns asset counts per type for a single task. +func (s *AssetStore) CountsByTypeForTask(taskID int64) (map[string]int, error) { + rows, err := s.db.Query(`SELECT type, COUNT(*) FROM assets WHERE $1 = ANY(task_ids) GROUP BY type`, taskID) + if err != nil { + return nil, err + } + defer rows.Close() + return scanTypeCounts(rows) +} + const assetSelectCols = `SELECT id, type, company_id, array_to_json(task_ids)::text, COALESCE(domain,''), COALESCE(root_domain,''), COALESCE(ip,''), COALESCE(c_segment::text,''), port, @@ -1072,6 +1119,11 @@ func (s *AssetStore) CountsByType() (map[string]int, error) { return nil, err } defer rows.Close() + return scanTypeCounts(rows) +} + +// scanTypeCounts scans a (type, count) result set into a map. +func scanTypeCounts(rows *sql.Rows) (map[string]int, error) { out := map[string]int{} for rows.Next() { var typ string diff --git a/db/assets_test.go b/db/assets_test.go index 4c5b5b2..2245192 100644 --- a/db/assets_test.go +++ b/db/assets_test.go @@ -1,6 +1,7 @@ package db import ( + "fmt" "testing" ) @@ -545,7 +546,7 @@ func TestQueryByTask(t *testing.T) { } defer deleteAsset(d, id) - assets, err := av2.QueryByTask(taskID, "", 10) + assets, err := av2.QueryByTask(taskID, "", 10, 0) if err != nil { t.Fatal(err) } @@ -558,6 +559,73 @@ func TestQueryByTask(t *testing.T) { if !found { t.Error("QueryByTask: inserted asset not found") } + + n, err := av2.CountByTask(taskID, "") + if err != nil { + t.Fatal(err) + } + if n < 1 { + t.Errorf("CountByTask: got %d, want >= 1", n) + } + if counts, err := av2.CountsByTypeForTask(taskID); err != nil { + t.Fatal(err) + } else if counts["root_domain"] < 1 { + t.Errorf("CountsByTypeForTask: root_domain = %d, want >= 1", counts["root_domain"]) + } + + // offset past the end must return nothing, not fall back to page 1 + rest, err := av2.QueryByTask(taskID, "", 10, n) + if err != nil { + t.Fatal(err) + } + if len(rest) != 0 { + t.Errorf("QueryByTask offset=%d: got %d rows, want 0", n, len(rest)) + } +} + +// 任务资产列表按页取,不再被固定条数截断:60 条资产用 25/页要能完整翻出来。 +func TestQueryByTaskPaging(t *testing.T) { + d, av2, _ := testSetup(t) + defer d.Close() + + const taskID = int64(99998) + const n = 60 + for i := 0; i < n; i++ { + id, err := av2.UpsertRootDomain(UpsertRootDomainReq{ + Domain: fmt.Sprintf("paging-%02d.querybytask.test", i), + TaskID: taskID, + }) + if err != nil { + t.Fatal(err) + } + defer deleteAsset(d, id) + } + + total, err := av2.CountByTask(taskID, "root_domain") + if err != nil { + t.Fatal(err) + } + if total != n { + t.Fatalf("CountByTask: got %d, want %d", total, n) + } + + const size = 25 + seen := map[int64]bool{} + for offset := 0; offset < total; offset += size { + page, err := av2.QueryByTask(taskID, "root_domain", size, offset) + if err != nil { + t.Fatal(err) + } + for _, a := range page { + if seen[a.ID] { + t.Errorf("offset=%d: asset %d returned twice", offset, a.ID) + } + seen[a.ID] = true + } + } + if len(seen) != n { + t.Errorf("paged through %d assets, want %d", len(seen), n) + } } func TestQueryByCompany(t *testing.T) { @@ -582,7 +650,7 @@ func TestQueryByCompany(t *testing.T) { t.Fatal(err) } - assets, err := av2.QueryByCompany(companyID, "root_domain", 10) + assets, err := av2.QueryByCompany(companyID, "root_domain", 10, 0) if err != nil { t.Fatal(err) } @@ -595,6 +663,77 @@ func TestQueryByCompany(t *testing.T) { if !found { t.Error("QueryByCompany: attributed asset not found") } + + n, err := av2.CountByCompany(companyID, "root_domain") + if err != nil { + t.Fatal(err) + } + if n < 1 { + t.Errorf("CountByCompany: got %d, want >= 1", n) + } + + // offset past the end must return nothing, not fall back to page 1 + rest, err := av2.QueryByCompany(companyID, "root_domain", 10, n) + if err != nil { + t.Fatal(err) + } + if len(rest) != 0 { + t.Errorf("QueryByCompany offset=%d: got %d rows, want 0", n, len(rest)) + } +} + +// 企业资产列表同样按页取,不被固定条数截断。 +func TestQueryByCompanyPaging(t *testing.T) { + d, av2, cs := testSetup(t) + defer d.Close() + + companyID, _, err := cs.UpsertCompany("QueryByCompanyPagingCorp", "") + if err != nil { + t.Fatal(err) + } + defer cleanupCompany(d, companyID) + + if added, _, _, errs := cs.AddScope(companyID, []string{"qbc-paging.io"}, "test"); added != 1 { + t.Fatalf("AddScope: added=%d, errors=%v", added, errs) + } + + // UpsertSubdomain 会顺带建根域名资产,一并清掉 + defer d.Exec(`DELETE FROM assets WHERE root_domain = 'qbc-paging.io'`) + + const n = 60 + for i := 0; i < n; i++ { + if _, err := av2.UpsertSubdomain(UpsertSubdomainReq{ + Domain: fmt.Sprintf("paging-%02d.qbc-paging.io", i), + }); err != nil { + t.Fatal(err) + } + } + + total, err := av2.CountByCompany(companyID, "subdomain") + if err != nil { + t.Fatal(err) + } + if total != n { + t.Fatalf("CountByCompany: got %d, want %d", total, n) + } + + const size = 25 + seen := map[int64]bool{} + for offset := 0; offset < total; offset += size { + page, err := av2.QueryByCompany(companyID, "subdomain", size, offset) + if err != nil { + t.Fatal(err) + } + for _, a := range page { + if seen[a.ID] { + t.Errorf("offset=%d: asset %d returned twice", offset, a.ID) + } + seen[a.ID] = true + } + } + if len(seen) != n { + t.Errorf("paged through %d assets, want %d", len(seen), n) + } } func TestCountsByType(t *testing.T) { diff --git a/server/assets.go b/server/assets.go index 6712a8a..62a794e 100644 --- a/server/assets.go +++ b/server/assets.go @@ -217,6 +217,9 @@ func (s *Server) reattribute(w http.ResponseWriter, r *http.Request) { // GET /api/assets // ===================================================================== +// defaultAssetPageSize is the page size used when the caller omits ?limit. +const defaultAssetPageSize = 50 + func (s *Server) listAssets(w http.ResponseWriter, r *http.Request) { as := s.assetStore() if as == nil { @@ -227,8 +230,10 @@ func (s *Server) listAssets(w http.ResponseWriter, r *http.Request) { typ := q.Get("type") limit, _ := strconv.Atoi(q.Get("limit")) offset, _ := strconv.Atoi(q.Get("offset")) + // limit/offset are page controls, not a cap: `total` always reports the full + // match count so callers can page through everything. if limit <= 0 { - limit = 50 + limit = defaultAssetPageSize } var assets []*db.Asset @@ -247,11 +252,15 @@ func (s *Server) listAssets(w http.ResponseWriter, r *http.Request) { taskID, _ := strconv.ParseInt(q.Get("task_id"), 10, 64) switch { case companyID > 0: - assets, err = as.QueryByCompany(companyID, typ, limit) - total = len(assets) + assets, err = as.QueryByCompany(companyID, typ, limit, offset) + if err == nil { + total, err = as.CountByCompany(companyID, typ) + } case taskID > 0: - assets, err = as.QueryByTask(taskID, typ, limit) - total = len(assets) + assets, err = as.QueryByTask(taskID, typ, limit, offset) + if err == nil { + total, err = as.CountByTask(taskID, typ) + } default: if typ == "" { typ = "root_domain" @@ -284,7 +293,13 @@ func (s *Server) assetCounts(w http.ResponseWriter, r *http.Request) { writeErr(w, 503, "database unavailable") return } - counts, err := as.CountsByType() + var counts map[string]int + var err error + if taskID, _ := strconv.ParseInt(r.URL.Query().Get("task_id"), 10, 64); taskID > 0 { + counts, err = as.CountsByTypeForTask(taskID) + } else { + counts, err = as.CountsByType() + } if err != nil { writeErr(w, 500, err.Error()) return diff --git a/web/src/app/(main)/function/tasks/detail/_tabs/assets-tab.tsx b/web/src/app/(main)/function/tasks/detail/_tabs/assets-tab.tsx index ffb1d51..7d066c6 100644 --- a/web/src/app/(main)/function/tasks/detail/_tabs/assets-tab.tsx +++ b/web/src/app/(main)/function/tasks/detail/_tabs/assets-tab.tsx @@ -152,50 +152,51 @@ const TABS: { key: string; label: string; icon: LucideIcon }[] = [ ]; export function AssetsTab({ taskId }: { taskId: string }) { - const [assets, setAssets] = React.useState([]); + // 服务端分页:只取当前分类的当前页,分类总数走 /assets/counts?task_id=,不做数量截断 + const [rows, setRows] = React.useState([]); + const [total, setTotal] = React.useState(0); + const [counts, setCounts] = React.useState>({}); const [loaded, setLoaded] = React.useState(false); const [tab, setTab] = React.useState("root_domain"); const [page, setPage] = React.useState(0); const [size, setSize] = React.useState(50); - React.useEffect(() => setPage(0), [tab, size]); + // 换分类/换页大小:回到第一页,并清掉旧行(各分类列不同,不能串用) + React.useEffect(() => { + setPage(0); + setRows([]); + }, [tab, size]); React.useEffect(() => { let alive = true; const load = async () => { try { - const results = await Promise.all( - TABS.map((t) => api.taskAssets(taskId, t.key)), - ); + const [cur, c] = await Promise.all([ + api.taskAssets(taskId, tab, size, page * size), + api.assetCounts(taskId), + ]); if (!alive) return; - setAssets(results.flat()); - setLoaded(true); + setRows(cur.assets); + setTotal(cur.total); + setCounts(c ?? {}); } catch { + // 保留上一次的数据 + } finally { if (alive) setLoaded(true); } }; load(); const timer = setInterval(load, 10_000); return () => { alive = false; clearInterval(timer); }; - }, [taskId]); + }, [taskId, tab, page, size]); - const byType = React.useMemo(() => { - const m: Record = { root_domain: [], ip: [], subdomain: [], service: [], endpoint: [] }; - for (const a of assets) { - if (a.type in m) m[a.type].push(a); - } - return m; - }, [assets]); + // 轮询期间资产变少时,别停在越界的空页上 + React.useEffect(() => { + const maxPage = Math.max(0, Math.ceil(total / size) - 1); + if (page > maxPage) setPage(maxPage); + }, [total, size, page]); - const counts: Record = { - root_domain: byType.root_domain.length, - ip: byType.ip.length, - subdomain: byType.subdomain.length, - service: byType.service.length, - endpoint: byType.endpoint.length, - }; - - const totalAll = assets.length; + const totalAll = TABS.reduce((n, t) => n + (counts[t.key] ?? 0), 0); if (!loaded) { return
加载中…
; @@ -209,9 +210,6 @@ export function AssetsTab({ taskId }: { taskId: string }) { ); } - const cur = byType[tab] ?? []; - const slice = cur.slice(page * size, page * size + size); - return ( {/* left nav */} @@ -238,8 +236,8 @@ export function AssetsTab({ taskId }: { taskId: string }) {
- - {slice.map((a) => ( + + {rows.map((a) => ( {a.domain} {a.icp || "—"} @@ -249,8 +247,8 @@ export function AssetsTab({ taskId }: { taskId: string }) { - - {slice.map((a) => ( + + {rows.map((a) => ( {a.ip} {a.c_segment || "—"} @@ -269,8 +267,8 @@ export function AssetsTab({ taskId }: { taskId: string }) { - - {slice.map((a) => ( + + {rows.map((a) => ( {a.domain} {a.root_domain || "—"} @@ -284,13 +282,13 @@ export function AssetsTab({ taskId }: { taskId: string }) { - {slice.map((a) => { + {rows.map((a) => { const isHttp = a.service_type === "http"; const addr = isHttp ? (a.url || "") @@ -330,8 +328,8 @@ export function AssetsTab({ taskId }: { taskId: string }) { - - {slice.map((a) => ( + + {rows.map((a) => ( {a.url || "—"} diff --git a/web/src/lib/api.ts b/web/src/lib/api.ts index e14ae4d..d2eb7ec 100644 --- a/web/src/lib/api.ts +++ b/web/src/lib/api.ts @@ -202,13 +202,15 @@ export const api = { get<{ count: number; total: number; assets: Asset[] }>( `/assets?dsl=${encodeURIComponent(dsl)}${type ? `&type=${encodeURIComponent(type)}` : ""}&limit=${limit}&offset=${offset}`, ).then((r) => ({ assets: r?.assets ?? [], total: r?.total ?? r?.count ?? 0 })), - assetCounts: () => get>("/assets/counts"), + assetCounts: (taskId = "") => + get>(`/assets/counts${taskId ? `?task_id=${encodeURIComponent(taskId)}` : ""}`), deleteAssets: (ids: number[]) => http<{ deleted: number }>("/assets", { method: "DELETE", body: JSON.stringify({ ids }) }), - // legacy — kept for task-specific views; hits the same endpoint with task_id filter - taskAssets: (taskId: string, type = "") => - get<{ count: number; assets: Asset[] }>(`/assets?task_id=${taskId}&type=${type}`) - .then((r) => r?.assets ?? []), + // task-scoped view of the same endpoint — server-side paginated like `assets` + taskAssets: (taskId: string, type = "", limit = 50, offset = 0) => + get<{ count: number; total: number; assets: Asset[] }>( + `/assets?task_id=${encodeURIComponent(taskId)}&type=${encodeURIComponent(type)}&limit=${limit}&offset=${offset}`, + ).then((r) => ({ assets: r?.assets ?? [], total: r?.total ?? r?.count ?? 0 })), // ---- companies (企业 + 资产范围；归属唯一来源) ---- companies: () => get("/companies").then(arr),