From dfbbb33069a05e21275f65d69b7ea34135e35307 Mon Sep 17 00:00:00 2001 From: Autumn-27 <53844290+Autumn-27@users.noreply.github.com> Date: Mon, 24 Aug 2026 10:32:10 -0400 Subject: [PATCH] =?UTF-8?q?fix(assets):=20ip=20=E5=AD=97=E6=AE=B5=E6=8B=92?= =?UTF-8?q?=E6=94=B6=E4=B8=BB=E6=9C=BA=E5=90=8D,=E5=B9=B6=E5=91=8A?= =?UTF-8?q?=E8=AF=89=E8=B0=83=E7=94=A8=E6=96=B9=E8=AF=A5=E6=80=8E=E4=B9=88?= =?UTF-8?q?=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 上一次修复让非法 ip 不再打挂企业归属重算,但脏数据仍在持续产生。这次从源头 拦住:assets.ip 会被转成 inet 参与网段归属,存主机名的资产对所有 IP/CIDR 范围规则都是隐形的。 - db 层新增 ValidateAssetIP + ErrAssetIPInvalid,在 UpsertIP / UpsertHTTPService / UpsertOtherService / UpsertEndpoint 四个入口校验。 放在 db 层是因为 Agent 工具和资产 API 都经由它们,不必写两遍;内部调用方 (enrich 的解析结果、task_assets 的 ParseCIDR 输出、RecordValue 已 ParseIP 过滤的分支)传的本就是合法地址,不受影响。 - 错误信息给的是改法而不只是「非法」:带上收到的原值,并指出主机名应改用 type=subdomain 填 domain,或先解析 A/AAAA 记录再填 ip。insert_assets 的 错误是逐条返回的({index, error}),同批其余资产照常入库,Agent 拿到后能 定位到具体那条并在下一轮自行修正。 - 同步收紧 insert_assets 的工具描述与 ip 字段说明,在提示层先挡一道。 - 空 ip 仍然合法:service / endpoint 的 ip 是可选字段。 验证: 新增 db 层四类型拒收测试与 agent 层「一批混入一条非法、其余照常入库」 测试,均连 PG 实跑通过;干净库 go test -p 1 ./... 无新增失败(仅剩既有的 TestCompanyScopeInvalid)。 --- CHANGELOG.md | 4 +++ agent/insert_assets_test.go | 54 +++++++++++++++++++++++++++++++++++++ agent/tools_insert.go | 4 +-- db/assets.go | 34 +++++++++++++++++++++++ db/assets_test.go | 53 ++++++++++++++++++++++++++++++++++++ 5 files changed, 147 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6cb56e9..9fe4eb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,10 @@ - 无法解析的 `ip` 不再静默跳过：企业范围保存后会明确提示有多少条资产的 `ip` 不是合法 IP（含具体 id 和取值），提醒这些资产不会被 IP/CIDR 规则归属；同一信息也写入服务端日志，覆盖没有前端响应的路径（删除企业、scopesentry 同步、Agent 写入）。 - 任务测试范围的 IP/CIDR 匹配同步改用 `try_inet()`，替换原先只检查字符集的正则守卫（`abc.def` 这类全 hex 字母的主机名此前仍会漏过并触发同样的报错）。 +#### 修改的功能 + +- `ip`、`service`、`endpoint` 资产的 `ip` 字段不再接受主机名：`insert_assets` 和资产 API 会逐条返回带 `index` 的错误，并直接给出改正方法（改用 `type=subdomain` 填 `domain`，或先解析 A/AAAA 记录），Agent 可据此自行修正后重插。同批次中的其他资产不受影响，照常入库。 + ## [0.3.3] - 2026-08-23 ### Worker diff --git a/agent/insert_assets_test.go b/agent/insert_assets_test.go index 1a4a195..573562d 100644 --- a/agent/insert_assets_test.go +++ b/agent/insert_assets_test.go @@ -3,6 +3,7 @@ package agent import ( "context" "encoding/json" + "strings" "testing" "github.com/Autumn-27/artex/db" @@ -394,3 +395,56 @@ func TestInsertAssetsDedup(t *testing.T) { t.Errorf("dedup: want same ID on double insert, got %v vs %v", id1, id2) } } + +// ===================================================================== +// TestInsertAssetsRejectsHostnameIPPerItem +// 一批里混入 ip 填了主机名的一条 → 只有那条失败，其余照常入库， +// 且错误里带得上 index 和改正方法，Agent 下一轮能自己修好。 +// ===================================================================== +func TestInsertAssetsRejectsHostnameIPPerItem(t *testing.T) { + d := testDB(t) + defer d.Close() + + ts := NewToolSet(nil, "") + ts.SetAssetStore(d.Assets(), d.Companies()) + defer d.Exec(`DELETE FROM assets WHERE ip IN ('198.51.100.23','cdn.badip-test.com') OR domain='badip-test.com'`) + + out := callInsertAssets(t, ts, map[string]any{ + "assets": []any{ + map[string]any{"type": "root_domain", "domain": "badip-test.com"}, + map[string]any{"type": "ip", "ip": "cdn.badip-test.com"}, + map[string]any{"type": "ip", "ip": "198.51.100.23"}, + }, + }) + + // The two valid entries must survive the bad one — a whole-batch failure + // would make the agent re-send assets that were already fine. + results, _ := out["results"].([]any) + if len(results) != 2 { + t.Fatalf("results=%v, want the 2 valid assets", out["results"]) + } + + errsRaw, _ := out["errors"].([]any) + if len(errsRaw) != 1 { + t.Fatalf("errors=%v, want exactly the invalid entry", out["errors"]) + } + entry, _ := errsRaw[0].(map[string]any) + if index, _ := entry["index"].(float64); int(index) != 1 { + t.Fatalf("error index=%v, want 1", entry["index"]) + } + message, _ := entry["error"].(string) + for _, want := range []string{"cdn.badip-test.com", "type=subdomain", "A/AAAA"} { + if !strings.Contains(message, want) { + t.Fatalf("error message %q lacks %q — agent cannot act on it", message, want) + } + } + + // The rejected value must not have reached the table. + var stored int + if err := d.QueryRow(`SELECT count(*) FROM assets WHERE ip='cdn.badip-test.com'`).Scan(&stored); err != nil { + t.Fatal(err) + } + if stored != 0 { + t.Fatalf("rejected hostname still stored in assets.ip (%d rows)", stored) + } +} diff --git a/agent/tools_insert.go b/agent/tools_insert.go index 5b4d58c..8c6fb5b 100644 --- a/agent/tools_insert.go +++ b/agent/tools_insert.go @@ -79,7 +79,7 @@ func (t *ToolSet) insertAssets() actool.CoreTool { "发现新资产时立即批量插入资产到资产库。支持一次提交多种资产类型：root_domain/ip/subdomain/app/service/endpoint。\n"+ "每条资产的字段说明：\n"+ "• root_domain：domain(根域名, 必填)、icp(备案, 可选)\n"+ - "• ip：ip(IP地址, 必填)、bound_domains([域名])、open_ports([{port,service}])\n"+ + "• ip：ip(IP地址, 必填, 必须是 IPv4/IPv6 地址而非主机名)、bound_domains([域名])、open_ports([{port,service}])\n"+ "• subdomain：domain(子域名, 必填)、record_type(A/AAAA/CNAME等)、record_value(记录值)、icp(可选)\n"+ "• app：app_name(应用名, 必填)、bundle_id(bundle id, 可选)、category、description、app_icp、company_id(归属企业 id, 可选)\n"+ "• service(http)：url(必填)、technologies([指纹])、status_code、content_length、page_title、favicon_mmh3、auth([{type,username,password,...}])、service_ip\n"+ @@ -108,7 +108,7 @@ func (t *ToolSet) insertAssets() actool.CoreTool { "description": "DNS 解析值列表（subdomain 可选，如 [\"1.2.3.4\",\"2.3.4.5\"]）", }, // ip - "ip": str("IP 地址（ip 类型必填；service/endpoint 类型可填，用于关联 IP）"), + "ip": str("IP 地址，必须是 IPv4/IPv6 地址，不能填主机名（主机名请用 type=subdomain 的 domain 字段）；ip 类型必填；service/endpoint 类型可填，用于关联 IP"), "bound_domains": map[string]any{ "type": "array", "items": map[string]any{"type": "string"}, diff --git a/db/assets.go b/db/assets.go index ec3a7cb..ebb2534 100644 --- a/db/assets.go +++ b/db/assets.go @@ -3,6 +3,7 @@ package db import ( "database/sql" "encoding/json" + "errors" "fmt" "net" "net/url" @@ -300,6 +301,27 @@ RETURNING id`, domain, icpVal, companyID, taskIDs).Scan(&id) return id, err } +// ErrAssetIPInvalid marks a non-address value in an asset's ip field. Both +// insert_assets and the asset API report it per item with the item index, so one +// bad entry never costs the rest of the batch. +var ErrAssetIPInvalid = errors.New("invalid asset ip") + +// ValidateAssetIP keeps hostnames out of assets.ip. Network attribution casts +// that column to inet (see try_inet in schema.sql), so a hostname stored here is +// silently invisible to every IP/CIDR scope rule — the asset simply never gets +// attributed and nobody can tell why. The message states the fix rather than +// just the fault, so an agent can correct the item on its next turn. An empty +// value is accepted: the ip field is optional for service and endpoint assets. +func ValidateAssetIP(value string) error { + if value == "" || net.ParseIP(value) != nil { + return nil + } + return fmt.Errorf( + "%w: ip 必须是 IPv4/IPv6 地址，收到 %q。若这是主机名，请改用 type=subdomain 并填 domain 字段；"+ + "若确实要登记地址，请先解析出 A/AAAA 记录，再用解析出的地址填 ip", + ErrAssetIPInvalid, value) +} + // ===================================================================== // UpsertIP // ===================================================================== @@ -317,6 +339,9 @@ func (s *AssetStore) UpsertIP(req UpsertIPReq) (int64, error) { if req.IP == "" { return 0, fmt.Errorf("ip is required") } + if err := ValidateAssetIP(req.IP); err != nil { + return 0, err + } if s.tx == nil { return s.withCompanyScopeMutation(func(scoped *AssetStore) (int64, error) { return scoped.UpsertIP(req) @@ -659,6 +684,9 @@ func (s *AssetStore) UpsertHTTPService(req UpsertHTTPServiceReq) (int64, error) if req.URL == "" { return 0, fmt.Errorf("url is required") } + if err := ValidateAssetIP(req.IP); err != nil { + return 0, err + } if s.tx == nil { return s.withCompanyScopeMutation(func(scoped *AssetStore) (int64, error) { return scoped.UpsertHTTPService(req) @@ -793,6 +821,9 @@ func (s *AssetStore) UpsertOtherService(req UpsertOtherServiceReq) (int64, error if req.Domain == "" && req.IP == "" { return 0, fmt.Errorf("domain or ip is required") } + if err := ValidateAssetIP(req.IP); err != nil { + return 0, err + } if req.Port == 0 { return 0, fmt.Errorf("port is required") } @@ -936,6 +967,9 @@ func (s *AssetStore) UpsertEndpoint(req UpsertEndpointReq) (int64, error) { if req.Method == "" { return 0, fmt.Errorf("method is required") } + if err := ValidateAssetIP(req.IP); err != nil { + return 0, err + } if s.tx == nil { return s.withCompanyScopeMutation(func(scoped *AssetStore) (int64, error) { return scoped.UpsertEndpoint(req) diff --git a/db/assets_test.go b/db/assets_test.go index c2ae2e5..9bebb97 100644 --- a/db/assets_test.go +++ b/db/assets_test.go @@ -1,8 +1,10 @@ package db import ( + "errors" "fmt" "slices" + "strings" "testing" "time" ) @@ -946,3 +948,54 @@ func TestAssetPaginationUsesStableIDTieBreaker(t *testing.T) { t.Fatalf("DSL pagination order=%v want=%v", got, want) } } + +// TestAssetIPRejectsHostname pins the write-side guard added after a hostname in +// assets.ip broke company attribution: every asset type that persists an ip must +// refuse a non-address, and the message must tell the caller how to fix it. +func TestAssetIPRejectsHostname(t *testing.T) { + d, av2, _ := testSetup(t) + defer d.Close() + + hostname := "d63cd476.cdn.ucloud.com.cn" + cases := []struct { + name string + call func() (int64, error) + }{ + {"ip", func() (int64, error) { + return av2.UpsertIP(UpsertIPReq{IP: hostname}) + }}, + {"http service", func() (int64, error) { + return av2.UpsertHTTPService(UpsertHTTPServiceReq{URL: "https://example.com/", IP: hostname}) + }}, + {"other service", func() (int64, error) { + return av2.UpsertOtherService(UpsertOtherServiceReq{IP: hostname, Port: 22, ServiceName: "ssh"}) + }}, + {"endpoint", func() (int64, error) { + return av2.UpsertEndpoint(UpsertEndpointReq{URL: "https://example.com/a", Method: "GET", IP: hostname}) + }}, + } + for _, tc := range cases { + id, err := tc.call() + if !errors.Is(err, ErrAssetIPInvalid) { + deleteAsset(d, id) + t.Fatalf("%s: err=%v, want %v", tc.name, err, ErrAssetIPInvalid) + } + // The caller has to learn the remedy, not just that something was wrong. + if !strings.Contains(err.Error(), hostname) || !strings.Contains(err.Error(), "type=subdomain") { + t.Fatalf("%s: message is not actionable: %v", tc.name, err) + } + } + + // An empty ip stays legal for the types where it is optional, and a real + // address is still accepted. + id, err := av2.UpsertHTTPService(UpsertHTTPServiceReq{URL: "https://ip-guard.example.com/"}) + if err != nil { + t.Fatalf("empty ip rejected: %v", err) + } + deleteAsset(d, id) + id, err = av2.UpsertIP(UpsertIPReq{IP: "203.0.113.7"}) + if err != nil { + t.Fatalf("valid ip rejected: %v", err) + } + deleteAsset(d, id) +}