feat(assets): 任务/企业资产列表改服务端分页,去掉 50 条上限

- db: QueryByTask/QueryByCompany 加 offset 参数,limit<=0 不再兜底成
  50/200 而是不限量;新增 CountByTask/CountByCompany/CountsByTypeForTask;
  抽出共享 pageClause 与 scanTypeCounts。
- server: /assets 的 task_id/company_id 分支改用真实 total(不再是
  len(assets));/assets/counts 支持 ?task_id= 返回按类型计数。
- web: taskAssets(taskId,type,limit,offset) 返回 {assets,total};
  assetCounts(taskId?);assets-tab 改为只取当前分类当前页,分类徽章走
  counts 接口,分页器 100/200 真正可用,轮询变轻。
- 补 TestQueryByTaskPaging/TestQueryByCompanyPaging 回归(60 条 25/页翻页
  不重不漏)。
This commit is contained in:
Autumn-27 committed 2026-08-19 09:17:13 -04:00
1 parent 5244500fee
commit 8e9b126972
5 files changed
+268 -62

No files matched your search

+64 -12
View File
@@ -913,10 +913,8 @@ func (s *AssetStore) CountByType(typ string) (int, error) {
}
// QueryByCompany returns assets for a company, optionally filtered by type.
func (s *AssetStore) QueryByCompany(companyID int64, typ string, limit int) ([]*Asset, error) {
if limit <= 0 {
limit = 200
}
// limit <= 0 means no limit — pair it with CountByCompany for server-side paging.
func (s *AssetStore) QueryByCompany(companyID int64, typ string, limit, offset int) ([]*Asset, error) {
q := `SELECT id, type, company_id, array_to_json(task_ids)::text,
COALESCE(domain,''), COALESCE(root_domain,''), COALESCE(ip,''),
COALESCE(c_segment::text,''), port,
@@ -933,8 +931,7 @@ FROM assets WHERE company_id = $1`
args = append(args, typ)
q += fmt.Sprintf(` AND type = $%d`, len(args))
}
args = append(args, limit)
q += fmt.Sprintf(` ORDER BY last_seen DESC LIMIT $%d`, len(args))
q += pageClause(&args, limit, offset)
rows, err := s.db.Query(q, args...)
if err != nil {
return nil, err
@@ -943,11 +940,38 @@ FROM assets WHERE company_id = $1`
return scanAssets(rows)
}
// QueryByTask returns assets rows that have a given task_id in task_ids.
func (s *AssetStore) QueryByTask(taskID int64, typ string, limit int) ([]*Asset, error) {
if limit <= 0 {
limit = 200
// CountByCompany returns the full number of assets attributed to a company
// (optionally filtered by type), ignoring limit/offset.
func (s *AssetStore) CountByCompany(companyID int64, typ string) (int, error) {
q := `SELECT count(*) FROM assets WHERE company_id = $1`
args := []any{companyID}
if typ != "" {
args = append(args, typ)
q += fmt.Sprintf(` AND type = $%d`, len(args))
}
var n int
err := s.db.QueryRow(q, args...).Scan(&n)
return n, err
}
// pageClause appends limit/offset placeholders to args and returns the trailing
// ORDER BY … LIMIT … OFFSET fragment. limit <= 0 means no limit.
func pageClause(args *[]any, limit, offset int) string {
q := ` ORDER BY last_seen DESC`
if limit > 0 {
*args = append(*args, limit)
q += fmt.Sprintf(` LIMIT $%d`, len(*args))
}
if offset > 0 {
*args = append(*args, offset)
q += fmt.Sprintf(` OFFSET $%d`, len(*args))
}
return q
}
// QueryByTask returns assets rows that have a given task_id in task_ids.
// limit <= 0 means no limit — pair it with CountByTask for server-side paging.
func (s *AssetStore) QueryByTask(taskID int64, typ string, limit, offset int) ([]*Asset, error) {
q := `SELECT id, type, company_id, array_to_json(task_ids)::text,
COALESCE(domain,''), COALESCE(root_domain,''), COALESCE(ip,''),
COALESCE(c_segment::text,''), port,
@@ -964,8 +988,7 @@ FROM assets WHERE $1 = ANY(task_ids)`
args = append(args, typ)
q += fmt.Sprintf(` AND type = $%d`, len(args))
}
args = append(args, limit)
q += fmt.Sprintf(` ORDER BY last_seen DESC LIMIT $%d`, len(args))
q += pageClause(&args, limit, offset)
rows, err := s.db.Query(q, args...)
if err != nil {
return nil, err
@@ -974,6 +997,30 @@ FROM assets WHERE $1 = ANY(task_ids)`
return scanAssets(rows)
}
// CountByTask returns the full number of assets attached to a task (optionally
// filtered by type), ignoring limit/offset — lets the UI paginate on the server.
func (s *AssetStore) CountByTask(taskID int64, typ string) (int, error) {
q := `SELECT count(*) FROM assets WHERE $1 = ANY(task_ids)`
args := []any{taskID}
if typ != "" {
args = append(args, typ)
q += fmt.Sprintf(` AND type = $%d`, len(args))
}
var n int
err := s.db.QueryRow(q, args...).Scan(&n)
return n, err
}
// CountsByTypeForTask returns asset counts per type for a single task.
func (s *AssetStore) CountsByTypeForTask(taskID int64) (map[string]int, error) {
rows, err := s.db.Query(`SELECT type, COUNT(*) FROM assets WHERE $1 = ANY(task_ids) GROUP BY type`, taskID)
if err != nil {
return nil, err
}
defer rows.Close()
return scanTypeCounts(rows)
}
const assetSelectCols = `SELECT id, type, company_id, array_to_json(task_ids)::text,
COALESCE(domain,''), COALESCE(root_domain,''), COALESCE(ip,''),
COALESCE(c_segment::text,''), port,
@@ -1072,6 +1119,11 @@ func (s *AssetStore) CountsByType() (map[string]int, error) {
return nil, err
}
defer rows.Close()
return scanTypeCounts(rows)
}
// scanTypeCounts scans a (type, count) result set into a map.
func scanTypeCounts(rows *sql.Rows) (map[string]int, error) {
out := map[string]int{}
for rows.Next() {
var typ string
+141 -2
View File
@@ -1,6 +1,7 @@
package db
import (
"fmt"
"testing"
)
@@ -545,7 +546,7 @@ func TestQueryByTask(t *testing.T) {
}
defer deleteAsset(d, id)
assets, err := av2.QueryByTask(taskID, "", 10)
assets, err := av2.QueryByTask(taskID, "", 10, 0)
if err != nil {
t.Fatal(err)
}
@@ -558,6 +559,73 @@ func TestQueryByTask(t *testing.T) {
if !found {
t.Error("QueryByTask: inserted asset not found")
}
n, err := av2.CountByTask(taskID, "")
if err != nil {
t.Fatal(err)
}
if n < 1 {
t.Errorf("CountByTask: got %d, want >= 1", n)
}
if counts, err := av2.CountsByTypeForTask(taskID); err != nil {
t.Fatal(err)
} else if counts["root_domain"] < 1 {
t.Errorf("CountsByTypeForTask: root_domain = %d, want >= 1", counts["root_domain"])
}
// offset past the end must return nothing, not fall back to page 1
rest, err := av2.QueryByTask(taskID, "", 10, n)
if err != nil {
t.Fatal(err)
}
if len(rest) != 0 {
t.Errorf("QueryByTask offset=%d: got %d rows, want 0", n, len(rest))
}
}
// 任务资产列表按页取,不再被固定条数截断:60 条资产用 25/页要能完整翻出来。
func TestQueryByTaskPaging(t *testing.T) {
d, av2, _ := testSetup(t)
defer d.Close()
const taskID = int64(99998)
const n = 60
for i := 0; i < n; i++ {
id, err := av2.UpsertRootDomain(UpsertRootDomainReq{
Domain: fmt.Sprintf("paging-%02d.querybytask.test", i),
TaskID: taskID,
})
if err != nil {
t.Fatal(err)
}
defer deleteAsset(d, id)
}
total, err := av2.CountByTask(taskID, "root_domain")
if err != nil {
t.Fatal(err)
}
if total != n {
t.Fatalf("CountByTask: got %d, want %d", total, n)
}
const size = 25
seen := map[int64]bool{}
for offset := 0; offset < total; offset += size {
page, err := av2.QueryByTask(taskID, "root_domain", size, offset)
if err != nil {
t.Fatal(err)
}
for _, a := range page {
if seen[a.ID] {
t.Errorf("offset=%d: asset %d returned twice", offset, a.ID)
}
seen[a.ID] = true
}
}
if len(seen) != n {
t.Errorf("paged through %d assets, want %d", len(seen), n)
}
}
func TestQueryByCompany(t *testing.T) {
@@ -582,7 +650,7 @@ func TestQueryByCompany(t *testing.T) {
t.Fatal(err)
}
assets, err := av2.QueryByCompany(companyID, "root_domain", 10)
assets, err := av2.QueryByCompany(companyID, "root_domain", 10, 0)
if err != nil {
t.Fatal(err)
}
@@ -595,6 +663,77 @@ func TestQueryByCompany(t *testing.T) {
if !found {
t.Error("QueryByCompany: attributed asset not found")
}
n, err := av2.CountByCompany(companyID, "root_domain")
if err != nil {
t.Fatal(err)
}
if n < 1 {
t.Errorf("CountByCompany: got %d, want >= 1", n)
}
// offset past the end must return nothing, not fall back to page 1
rest, err := av2.QueryByCompany(companyID, "root_domain", 10, n)
if err != nil {
t.Fatal(err)
}
if len(rest) != 0 {
t.Errorf("QueryByCompany offset=%d: got %d rows, want 0", n, len(rest))
}
}
// 企业资产列表同样按页取,不被固定条数截断。
func TestQueryByCompanyPaging(t *testing.T) {
d, av2, cs := testSetup(t)
defer d.Close()
companyID, _, err := cs.UpsertCompany("QueryByCompanyPagingCorp", "")
if err != nil {
t.Fatal(err)
}
defer cleanupCompany(d, companyID)
if added, _, _, errs := cs.AddScope(companyID, []string{"qbc-paging.io"}, "test"); added != 1 {
t.Fatalf("AddScope: added=%d, errors=%v", added, errs)
}
// UpsertSubdomain 会顺带建根域名资产,一并清掉
defer d.Exec(`DELETE FROM assets WHERE root_domain = 'qbc-paging.io'`)
const n = 60
for i := 0; i < n; i++ {
if _, err := av2.UpsertSubdomain(UpsertSubdomainReq{
Domain: fmt.Sprintf("paging-%02d.qbc-paging.io", i),
}); err != nil {
t.Fatal(err)
}
}
total, err := av2.CountByCompany(companyID, "subdomain")
if err != nil {
t.Fatal(err)
}
if total != n {
t.Fatalf("CountByCompany: got %d, want %d", total, n)
}
const size = 25
seen := map[int64]bool{}
for offset := 0; offset < total; offset += size {
page, err := av2.QueryByCompany(companyID, "subdomain", size, offset)
if err != nil {
t.Fatal(err)
}
for _, a := range page {
if seen[a.ID] {
t.Errorf("offset=%d: asset %d returned twice", offset, a.ID)
}
seen[a.ID] = true
}
}
if len(seen) != n {
t.Errorf("paged through %d assets, want %d", len(seen), n)
}
}
func TestCountsByType(t *testing.T) {
+21 -6
View File
@@ -217,6 +217,9 @@ func (s *Server) reattribute(w http.ResponseWriter, r *http.Request) {
// GET /api/assets
// =====================================================================
// defaultAssetPageSize is the page size used when the caller omits ?limit.
const defaultAssetPageSize = 50
func (s *Server) listAssets(w http.ResponseWriter, r *http.Request) {
as := s.assetStore()
if as == nil {
@@ -227,8 +230,10 @@ func (s *Server) listAssets(w http.ResponseWriter, r *http.Request) {
typ := q.Get("type")
limit, _ := strconv.Atoi(q.Get("limit"))
offset, _ := strconv.Atoi(q.Get("offset"))
// limit/offset are page controls, not a cap: `total` always reports the full
// match count so callers can page through everything.
if limit <= 0 {
limit = 50
limit = defaultAssetPageSize
}
var assets []*db.Asset
@@ -247,11 +252,15 @@ func (s *Server) listAssets(w http.ResponseWriter, r *http.Request) {
taskID, _ := strconv.ParseInt(q.Get("task_id"), 10, 64)
switch {
case companyID > 0:
assets, err = as.QueryByCompany(companyID, typ, limit)
total = len(assets)
assets, err = as.QueryByCompany(companyID, typ, limit, offset)
if err == nil {
total, err = as.CountByCompany(companyID, typ)
}
case taskID > 0:
assets, err = as.QueryByTask(taskID, typ, limit)
total = len(assets)
assets, err = as.QueryByTask(taskID, typ, limit, offset)
if err == nil {
total, err = as.CountByTask(taskID, typ)
}
default:
if typ == "" {
typ = "root_domain"
@@ -284,7 +293,13 @@ func (s *Server) assetCounts(w http.ResponseWriter, r *http.Request) {
writeErr(w, 503, "database unavailable")
return
}
counts, err := as.CountsByType()
var counts map[string]int
var err error
if taskID, _ := strconv.ParseInt(r.URL.Query().Get("task_id"), 10, 64); taskID > 0 {
counts, err = as.CountsByTypeForTask(taskID)
} else {
counts, err = as.CountsByType()
}
if err != nil {
writeErr(w, 500, err.Error())
return
@@ -152,50 +152,51 @@ const TABS: { key: string; label: string; icon: LucideIcon }[] = [
];
export function AssetsTab({ taskId }: { taskId: string }) {
const [assets, setAssets] = React.useState<Asset[]>([]);
// 服务端分页:只取当前分类的当前页,分类总数走 /assets/counts?task_id=,不做数量截断
const [rows, setRows] = React.useState<Asset[]>([]);
const [total, setTotal] = React.useState(0);
const [counts, setCounts] = React.useState<Record<string, number>>({});
const [loaded, setLoaded] = React.useState(false);
const [tab, setTab] = React.useState("root_domain");
const [page, setPage] = React.useState(0);
const [size, setSize] = React.useState(50);
React.useEffect(() => setPage(0), [tab, size]);
// 换分类/换页大小:回到第一页,并清掉旧行(各分类列不同,不能串用)
React.useEffect(() => {
setPage(0);
setRows([]);
}, [tab, size]);
React.useEffect(() => {
let alive = true;
const load = async () => {
try {
const results = await Promise.all(
TABS.map((t) => api.taskAssets(taskId, t.key)),
);
const [cur, c] = await Promise.all([
api.taskAssets(taskId, tab, size, page * size),
api.assetCounts(taskId),
]);
if (!alive) return;
setAssets(results.flat());
setLoaded(true);
setRows(cur.assets);
setTotal(cur.total);
setCounts(c ?? {});
} catch {
// 保留上一次的数据
} finally {
if (alive) setLoaded(true);
}
};
load();
const timer = setInterval(load, 10_000);
return () => { alive = false; clearInterval(timer); };
}, [taskId]);
}, [taskId, tab, page, size]);
const byType = React.useMemo(() => {
const m: Record<string, Asset[]> = { root_domain: [], ip: [], subdomain: [], service: [], endpoint: [] };
for (const a of assets) {
if (a.type in m) m[a.type].push(a);
}
return m;
}, [assets]);
// 轮询期间资产变少时,别停在越界的空页上
React.useEffect(() => {
const maxPage = Math.max(0, Math.ceil(total / size) - 1);
if (page > maxPage) setPage(maxPage);
}, [total, size, page]);
const counts: Record<string, number> = {
root_domain: byType.root_domain.length,
ip: byType.ip.length,
subdomain: byType.subdomain.length,
service: byType.service.length,
endpoint: byType.endpoint.length,
};
const totalAll = assets.length;
const totalAll = TABS.reduce((n, t) => n + (counts[t.key] ?? 0), 0);
if (!loaded) {
return <div className="py-20 text-center text-sm text-muted-foreground">加载中…</div>;
@@ -209,9 +210,6 @@ export function AssetsTab({ taskId }: { taskId: string }) {
);
}
const cur = byType[tab] ?? [];
const slice = cur.slice(page * size, page * size + size);
return (
<Tabs orientation="vertical" value={tab} onValueChange={setTab} className="items-start gap-4">
{/* left nav */}
@@ -238,8 +236,8 @@ export function AssetsTab({ taskId }: { taskId: string }) {
<div className="min-w-0 flex-1 flex flex-col gap-4">
<TabsContent value="root_domain" className="mt-0 flex min-h-0 flex-1 flex-col">
<AssetCard cols={["域名", "ICP 备案"]} total={cur.length} page={page} size={size} onSize={setSize} onPage={setPage}>
{slice.map((a) => (
<AssetCard cols={["域名", "ICP 备案"]} total={total} page={page} size={size} onSize={setSize} onPage={setPage}>
{rows.map((a) => (
<TableRow key={a.id}>
<TableCell className="font-mono text-xs font-medium">{a.domain}</TableCell>
<TableCell className="text-xs">{a.icp || "—"}</TableCell>
@@ -249,8 +247,8 @@ export function AssetsTab({ taskId }: { taskId: string }) {
</TabsContent>
<TabsContent value="ip" className="mt-0 flex min-h-0 flex-1 flex-col">
<AssetCard cols={["IP", "C段", "绑定域名", "开放端口"]} total={cur.length} page={page} size={size} onSize={setSize} onPage={setPage}>
{slice.map((a) => (
<AssetCard cols={["IP", "C段", "绑定域名", "开放端口"]} total={total} page={page} size={size} onSize={setSize} onPage={setPage}>
{rows.map((a) => (
<TableRow key={a.id}>
<TableCell className="font-mono text-xs font-medium">{a.ip}</TableCell>
<TableCell className="font-mono text-xs">{a.c_segment || "—"}</TableCell>
@@ -269,8 +267,8 @@ export function AssetsTab({ taskId }: { taskId: string }) {
</TabsContent>
<TabsContent value="subdomain" className="mt-0 flex min-h-0 flex-1 flex-col">
<AssetCard cols={["域名", "根域名", "解析类型", "解析值"]} total={cur.length} page={page} size={size} onSize={setSize} onPage={setPage}>
{slice.map((a) => (
<AssetCard cols={["域名", "根域名", "解析类型", "解析值"]} total={total} page={page} size={size} onSize={setSize} onPage={setPage}>
{rows.map((a) => (
<TableRow key={a.id}>
<TableCell className="font-mono text-xs font-medium">{a.domain}</TableCell>
<TableCell className="font-mono text-xs">{a.root_domain || "—"}</TableCell>
@@ -284,13 +282,13 @@ export function AssetsTab({ taskId }: { taskId: string }) {
<TabsContent value="service" className="mt-0 flex min-h-0 flex-1 flex-col">
<AssetCard
cols={["地址 / 服务", "状态码", "标题", "响应长度", "指纹", "认证"]}
total={cur.length}
total={total}
page={page}
size={size}
onSize={setSize}
onPage={setPage}
>
{slice.map((a) => {
{rows.map((a) => {
const isHttp = a.service_type === "http";
const addr = isHttp
? (a.url || "")
@@ -330,8 +328,8 @@ export function AssetsTab({ taskId }: { taskId: string }) {
</TabsContent>
<TabsContent value="endpoint" className="mt-0 flex min-h-0 flex-1 flex-col">
<AssetCard cols={["方法", "完整地址", "参数"]} total={cur.length} page={page} size={size} onSize={setSize} onPage={setPage}>
{slice.map((a) => (
<AssetCard cols={["方法", "完整地址", "参数"]} total={total} page={page} size={size} onSize={setSize} onPage={setPage}>
{rows.map((a) => (
<TableRow key={a.id}>
<TableCell className="w-16"><MethodBadge method={a.method || ""} /></TableCell>
<TableCell className="max-w-sm truncate font-mono text-xs" title={a.url}>{a.url || "—"}</TableCell>
+7 -5
View File
@@ -202,13 +202,15 @@ export const api = {
get<{ count: number; total: number; assets: Asset[] }>(
`/assets?dsl=${encodeURIComponent(dsl)}${type ? `&type=${encodeURIComponent(type)}` : ""}&limit=${limit}&offset=${offset}`,
).then((r) => ({ assets: r?.assets ?? [], total: r?.total ?? r?.count ?? 0 })),
assetCounts: () => get<Record<string, number>>("/assets/counts"),
assetCounts: (taskId = "") =>
get<Record<string, number>>(`/assets/counts${taskId ? `?task_id=${encodeURIComponent(taskId)}` : ""}`),
deleteAssets: (ids: number[]) =>
http<{ deleted: number }>("/assets", { method: "DELETE", body: JSON.stringify({ ids }) }),
// legacy — kept for task-specific views; hits the same endpoint with task_id filter
taskAssets: (taskId: string, type = "") =>
get<{ count: number; assets: Asset[] }>(`/assets?task_id=${taskId}&type=${type}`)
.then((r) => r?.assets ?? []),
// task-scoped view of the same endpoint — server-side paginated like `assets`
taskAssets: (taskId: string, type = "", limit = 50, offset = 0) =>
get<{ count: number; total: number; assets: Asset[] }>(
`/assets?task_id=${encodeURIComponent(taskId)}&type=${encodeURIComponent(type)}&limit=${limit}&offset=${offset}`,
).then((r) => ({ assets: r?.assets ?? [], total: r?.total ?? r?.count ?? 0 })),
// ---- companies (企业 + 资产范围;归属唯一来源) ----
companies: () => get<Company[]>("/companies").then(arr),