diff --git a/.env.example b/.env.example index 78de041..9e885e3 100644 --- a/.env.example +++ b/.env.example @@ -14,5 +14,5 @@ ARTEX_LLM_MODEL= ARTEX_LLM_BASE_URL= ARTEX_LLM_PROXY= -# HTTP User-Agent；目标请求留空不发送，自更新留空使用通用客户端值 +# HTTP User-Agent；留空时从浏览器 UA 样本中随机抽选，填写则使用固定值 ARTEX_HTTP_USER_AGENT= diff --git a/HTTP_USER_AGENT.md b/HTTP_USER_AGENT.md index 2063068..fdf7cad 100644 --- a/HTTP_USER_AGENT.md +++ b/HTTP_USER_AGENT.md @@ -12,17 +12,23 @@ | 静态采集 / 同源爬虫模板 | 移除 `spa-api-recon` / `spa-api-recon spider`，避免 urllib 自动填入自己的默认 UA；入口、下载、重定向及重试沿用同一配置。 | | HTTP 自定义工具 | 原先没有 ARTEX 专属 UA，但未设置时会由 Go 填充默认值。现采用统一策略，模板显式请求头优先。 | | 请求重放器 | 原先没有 ARTEX 专属 UA。现采用统一策略，用户显式请求头（包括空值）优先，保持原请求的配置能力。 | -| 记录代理 | 转发缺失 UA 的请求时，Go 会补入默认客户端标识。现保留省略语义，录制中不展示内部用于抑制默认值的空头。保留客户端已有 UA。 | -| GitHub 自更新 | 额外发现查询版本、下载校验和与发布包共用的两处 `artex-selfupdate`。均已移除；GitHub API 要求非空 UA，未配置时使用通用 `Go-http-client/1.1`，不含 ARTEX 名称。 | +| 记录代理 | 转发缺失 UA 的请求时，Go 会补入默认客户端标识。现保留显式省略语义，录制中不展示内部用于抑制默认值的空头。保留客户端已选择的随机或固定 UA。 | +| GitHub 自更新 | 额外发现查询版本、下载校验和与发布包共用的两处 `artex-selfupdate`。均已移除；与其他请求统一采用随机浏览器样本或显式配置的非空 UA，不含默认 ARTEX 名称。 | | 前端 mock | `ARTEX-worker/1.0` 仅为示例 HTTP 文本，不会发出请求；已删除该行，避免误导。 | ## 配置 -`ARTEX_HTTP_USER_AGENT` 统一配置以上通道，默认空值使目标请求不发送此头；自更新采用表中非空兼容值。Go 服务重启后生效；Docker 在 `.env` 设置后重建容器。直接运行 Python 模板时可继承同名环境变量。 +默认采用随机 UA 池方案，Go 与两个独立 Python 模板内置相同的 6 个完整浏览器 UA 样本，覆盖 Chrome / Firefox / Safari 的桌面格式。随机选择完整样本，允许重复，不拼接随机版本，不调用 AI 服务。 -Python 模板的 `--user-agent VALUE` 优先于环境变量，`--user-agent ''` 强制省略。自定义 HTTP 工具 / 重放器的显式 UA 同样优先于全局值。配置仅接受可打印 ASCII，去除首尾空格；控制字符和非 ASCII 值按空值处理，防止头注入。 +- Go 每次 HTTP 逻辑请求抽选一次；WebFetch 每次工具调用重新抽选，重定向、代理请求及失败后的直连重试复用同一值。 +- Python 脚本每次运行抽选一次，入口、并发资源下载及重试复用同一值；独立调用 `fetch()` 时每次调用单独选择。 +- `ARTEX_HTTP_USER_AGENT` 非空且合法时覆盖随机值；留空、仅空格或非法值回退随机池。Go 使用系统加密随机源；若随机源出错，保留池中首个非空有效样本。 +- Python 的 `--user-agent VALUE` 优先于环境变量，`--user-agent ''` 仍显式省略；自定义 HTTP 工具 / 重放器显式提供的 UA（包括空值）同样优先，保留原请求的控制能力。 +- 固定值只接受可打印 ASCII，去除首尾空格；控制字符和非 ASCII 值回退随机池，防止头注入。 -不随机轮换、不自动冒充浏览器。可以复用授权目标已有流量中的真实值。省略或修改 UA 不能保证客户端不可识别，第三方出口代理也可能再次添加或改写请求头。 +Go 服务重启后使用新配置；Docker 在 `.env` 设置后重建容器。直接运行 Python 模板时继承同名环境变量。已同步 agent OPSEC 提示，避免旧提示与平台随机策略冲突；遇到限速等异常时仍停止，不以换 UA 继续当前方向。 + +UA 随机化不会同步改变 TLS、Cookie 或其他浏览器特征，不能保证客户端不可识别；第三方出口代理也可能再次改写请求头。 ## 检查过的其他通道与边界 @@ -40,15 +46,21 @@ Python 模板的 `--user-agent VALUE` 优先于环境变量，`--user-agent ''` ## 回归验证 -- Go：检查真实 HTTP 线上头、配置值 / 空值、代理及失败直连重试、重定向、CA 信任、静态资源限制、HTML / RECON 解析、agent session 注册、自定义工具 / 重放器覆盖、自更新查询及下载和代理录制。 -- Python：本地 HTTP 服务验证默认 / 环境变量 / CLI 覆盖 / 显式空值、非法头值、重定向、Cookie 保留及产物生成；不访问真实目标。 +- Go：检查真实 HTTP 线上头、随机样本选择 / 固定值覆盖 / 显式空值、代理及失败直连重试、重定向、CA 信任、静态资源限制、HTML / RECON 解析、agent session 注册、自定义工具 / 重放器覆盖、自更新查询及下载和代理录制。 +- Python：本地 HTTP 服务验证默认随机 / 环境变量 / CLI 覆盖 / 显式空值、非法头值、重定向、Cookie 保留及产物生成；不访问真实目标。 本地命令： ```sh go test ./internal/httpidentity ./internal/webfetch ./enrich ./agent ./selfupdate ./traffic go test ./server -run 'TestProbe|TestRepeater|TestRunHTTPTool|TestHTTPTool|TestTargetHTTPIdentity' -count=1 +go test -race ./internal/httpidentity ./internal/webfetch ./enrich ./traffic python3 -m unittest discover -s skills/api-recon/scripts -p 'test_*.py' -v ``` -服务端完整套件另需隔离 PostgreSQL 测试库；未配置数据库时其中的数据库集成用例会失败。 +以上专项回归、Go race 检查及 Python 3 项测试均通过。 + +2026-11-11 使用项目提供的本地 PostgreSQL，在新建的独立测试库中执行了 `go test ./... -count=1 -p 1 -timeout 180s`，测试库用后删除。完整套件未全部通过： + +- `agent` 的 `TestGraphOverviewExpandsAssociatedCompanyScope`、`TestCaptureApprovalLifecycle` 和 `db` 的 `TestInterceptApprovalFilters`、`TestSideTaskArchiveVersions` 失败；在原提交 `17c2fc5` 的独立副本和另一新建测试库中复现了相同失败。 +- `server` 首次运行有 3 个用例在临时目录清理阶段报 `directory not empty`，涉及 `TestInheritedActivityDetailAndRelationDeletion`、`TestTaskMetadataPatchReturnsRenameAndPin`、`TestTaskMetadataPatchGlobalAssetSharing`，不是 UA 断言失败。再次在新建隔离库中运行完整 `server` 套件时这 3 项通过，但 `TestCoreTaskLifecyclePG` 报 `expected goal nodes seeded, got 0`。原提交的完整 `server` 套件通过；修改后完整 `server` 套件仍未获得全绿结果。 diff --git a/README.md b/README.md index 1b4c191..0933bda 100644 --- a/README.md +++ b/README.md @@ -314,7 +314,7 @@ CGO_ENABLED=0 go build -tags embedui -o artex ./cmd/artex **LLM**：`export ANTHROPIC_API_KEY=sk-...`（或 `OPENAI_API_KEY`），也可在 UI 的「LLM 配置」页填写。 可选：`ARTEX_LLM_PROVIDER` / `ARTEX_LLM_MODEL` / `ARTEX_LLM_BASE_URL` / `ARTEX_LLM_PROXY`。 -**HTTP User-Agent**：内置 WebFetch（worker / planner / main / chat，包括代理失败后的直连重试）、资产 HTTP 自动补全、出口连通性探测、HTTP 自定义工具、请求重放器及可选的 `harvest_static.py` / `spider_mpa.py` 默认不发送 User-Agent，不再携带工具名称。通过 `ARTEX_HTTP_USER_AGENT` 设置固定值（例如复用授权目标已有流量中的值）；直接运行时导出环境变量并重启服务，Docker 部署在 `.env` 配置后重建容器。两个 Python 模板还支持 `--user-agent '已有流量中的值'`，优先于环境变量；`--user-agent ''` 强制留空。值会去除首尾空格，只接受可打印 ASCII，含控制字符或非 ASCII 的配置忽略并按空值处理。不随机轮换或自动伪装浏览器；省略此请求头不能保证无法识别客户端。自定义工具和重放器中显式提供的请求头优先于全局值。GitHub 自更新查询、校验和及包下载同样移除了工具名，未配置时使用通用 `Go-http-client/1.1` 以满足 GitHub 非空请求头要求。记录代理保留请求头省略语义。此设置不影响浏览器、SDK 联网搜索、LLM / MCP / Webhook 服务通信或外部独立工具；本仓库未包含 phpfinger，其 `--user-agent` 需在该工具中单独配置。 +**HTTP User-Agent**：内置 WebFetch（worker / planner / main / chat）、资产 HTTP 自动补全、出口连通性探测、HTTP 自定义工具、请求重放器、GitHub 自更新和可选的 `harvest_static.py` / `spider_mpa.py` 默认从 6 个浏览器 UA 样本中随机抽选，不携带工具名称。Go 每次逻辑请求选择一次，重定向及 WebFetch 的代理失败直连重试复用同一个值；Python 脚本每次运行选择一次，入口、下载与重试保持一致。随机抽选允许重复，不依赖 AI 服务。通过 `ARTEX_HTTP_USER_AGENT` 可指定固定值；留空或非法值使用随机策略。直接运行时导出环境变量并重启服务，Docker 在 `.env` 配置后重建容器。Python 的 `--user-agent VALUE`、自定义工具和重放器中的显式 UA 优先，显式空值仍可省略此头。值去除首尾空格，仅接受可打印 ASCII。记录代理保留已有 UA 及显式省略语义。UA 随机化不代表 TLS、Cookie 或其他浏览器特征同步改变，也不保证无法识别客户端。此设置不影响浏览器、SDK 联网搜索、LLM / MCP / Webhook 服务通信或外部独立工具；本仓库未包含 phpfinger，其 `--user-agent` 需在该工具中单独配置。 完整排查范围、额外发现及未接管的服务通信见 [HTTP 请求标识排查](HTTP_USER_AGENT.md)。 diff --git a/agent/prompt.go b/agent/prompt.go index adc60aa..5b89972 100644 --- a/agent/prompt.go +++ b/agent/prompt.go @@ -21,9 +21,9 @@ const opsecPromptBlock = `【OPSEC 准则（硬约束，仅次于授权范围和 - 先复用已有流量、浏览器会话和已验证证据；能通过 traffic_search/traffic_get 或浏览器完成时，不要重复直接请求。 - 所有目标交互必须经过平台配置的记录代理或浏览器工具，确保请求可追溯。 -- 不得使用裸 curl、urllib、requests 等工具的默认请求头直接访问目标。命令行请求必须尽量复用目标已有流量中的 User-Agent、Accept、Referer、Cookie 等请求特征；没有真实依据时不要伪造身份、设备或浏览器指纹。 +- 不得使用裸 curl、urllib、requests 等工具的默认请求头直接访问目标。优先复用目标已有流量中的 User-Agent、Accept、Referer、Cookie 等请求特征；平台 HTTP 工具和采集模板的默认 UA 由系统随机策略提供，同一次请求的重定向、重试必须保持一致，明确配置的 UA 优先。 - 不得使用 -k、--ignore-https-errors、verify=False 等方式绕过证书校验；平台代理 CA 已配置时应正常验证。 -- 不得随机轮换 User-Agent、批量并发、快速循环、无节制重试、端口横扫、目录爆破或大规模扫描；遇到限速、验证码、WAF 或异常响应时立即停止当前方向。 +- 不得批量并发、快速循环、无节制重试、端口横扫、目录爆破或大规模扫描；遇到限速、验证码、WAF 或异常响应时立即停止当前方向，不得通过更换 UA 继续当前方向。 - OPSEC 不扩大授权范围；目标、主机、端口、路径或账号不在授权范围内时不得访问。 - 请求和响应是否留痕由平台代理和浏览器负责；不要额外复制完整请求头、Cookie 或响应正文。` diff --git a/agent/webfetch_test.go b/agent/webfetch_test.go index dcabe00..1d0eac8 100644 --- a/agent/webfetch_test.go +++ b/agent/webfetch_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "strings" "testing" "github.com/Autumn-27/norma/agentcore" @@ -42,7 +43,9 @@ func TestAgentSessionUsesConfiguredWebFetch(t *testing.T) { calls := 0 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { calls++ - if r.UserAgent() != ua { + if ua == "" && !strings.HasPrefix(r.UserAgent(), "Mozilla/5.0 ") { + t.Errorf("random UA missing: %q", r.UserAgent()) + } else if ua != "" && r.UserAgent() != ua { t.Errorf("session sent User-Agent %q, want %q", r.UserAgent(), ua) } w.Write([]byte("fixture")) diff --git a/enrich/enrich_test.go b/enrich/enrich_test.go index e4ddbd6..90ec255 100644 --- a/enrich/enrich_test.go +++ b/enrich/enrich_test.go @@ -4,6 +4,7 @@ import ( "errors" "net/http" "net/http/httptest" + "strings" "testing" ) @@ -15,12 +16,11 @@ func TestProbeUserAgent(t *testing.T) { calls := 0 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { calls++ - if got := r.UserAgent(); got != ua { + if got := r.UserAgent(); ua == "" && !strings.HasPrefix(got, "Mozilla/5.0 ") { + t.Errorf("random UA missing: %q", got) + } else if ua != "" && got != ua { t.Errorf("User-Agent = %q, want %q", got, ua) } - if _, present := r.Header["User-Agent"]; ua == "" && present { - t.Error("empty User-Agent present on wire") - } http.Redirect(w, r, "/stop", http.StatusFound) })) defer server.Close() diff --git a/internal/httpidentity/useragent.go b/internal/httpidentity/useragent.go index 5c06366..1eedabd 100644 --- a/internal/httpidentity/useragent.go +++ b/internal/httpidentity/useragent.go @@ -2,36 +2,50 @@ package httpidentity import ( + "crypto/rand" + "io" + "math/big" "net/http" "os" "strings" ) -// UserAgent returns the configured value, or an empty string to omit the header. -// Use printable ASCII for consistent behavior across Go and the Python templates. -// Ignore malformed values rather than allowing header injection. +// Keep these valid browser UA samples in sync with the standalone Python +// templates. Select whole samples rather than inventing incompatible versions. +var userAgents = [...]string{ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) Gecko/20100101 Firefox/128.0", + "Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.6 Safari/605.1.15", +} + +// UserAgent returns the configured printable ASCII value, or a random browser +// sample when unset, empty or malformed. Call once per logical request and reuse +// the selected value for redirects and retries. func UserAgent() string { value := strings.Trim(os.Getenv("ARTEX_HTTP_USER_AGENT"), " ") for _, c := range value { if c < 32 || c > 126 { - return "" + return randomUserAgent(rand.Reader) } } - return value + if value != "" { + return value + } + return randomUserAgent(rand.Reader) } -// Apply must set even an empty value: deleting the header makes net/http send -// its own Go-http-client default instead of omitting User-Agent on the wire. +func randomUserAgent(source io.Reader) string { + index, err := rand.Int(source, big.NewInt(int64(len(userAgents)))) + if err != nil { + return userAgents[0] // Keep a valid nonempty UA if system entropy fails. + } + return userAgents[index.Int64()] +} + +// Apply selects an identity for this request; net/http preserves it on redirects. func Apply(req *http.Request) { req.Header.Set("User-Agent", UserAgent()) } - -// ApplyRequired keeps a nonempty, generic library identity for services that -// require User-Agent (notably the GitHub API), without an ARTEX identifier. -func ApplyRequired(req *http.Request) { - value := UserAgent() - if value == "" { - value = "Go-http-client/1.1" - } - req.Header.Set("User-Agent", value) -} diff --git a/internal/httpidentity/useragent_test.go b/internal/httpidentity/useragent_test.go index eb74fd7..df874d5 100644 --- a/internal/httpidentity/useragent_test.go +++ b/internal/httpidentity/useragent_test.go @@ -1,8 +1,10 @@ package httpidentity import ( + "bytes" "net/http" "net/http/httptest" + "slices" "testing" ) @@ -17,12 +19,12 @@ func TestApplyOnWire(t *testing.T) { t.Run(tc.name, func(t *testing.T) { t.Setenv("ARTEX_HTTP_USER_AGENT", tc.value) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if got := r.Header.Get("User-Agent"); got != tc.want { + got := r.UserAgent() + if tc.want == "" && !slices.Contains(userAgents[:], got) { + t.Errorf("default User-Agent is not a browser sample: %q", got) + } else if tc.want != "" && got != tc.want { t.Errorf("User-Agent = %q, want %q", got, tc.want) } - if _, present := r.Header["User-Agent"]; tc.want == "" && present { - t.Error("empty User-Agent must be absent on the wire") - } if r.Header.Get("X-Injected") != "" { t.Error("header injection") } @@ -41,3 +43,15 @@ func TestApplyOnWire(t *testing.T) { }) } } + +func TestRandomUserAgentSelection(t *testing.T) { + // Exercise every choice deterministically without a statistical/flaky test. + for index, want := range userAgents { + if got := randomUserAgent(bytes.NewReader([]byte{byte(index)})); got != want { + t.Errorf("choice %d = %q, want %q", index, got, want) + } + } + if got := randomUserAgent(bytes.NewReader(nil)); got != userAgents[0] { + t.Errorf("entropy failure fallback = %q", got) + } +} diff --git a/internal/webfetch/webfetch.go b/internal/webfetch/webfetch.go index 0942871..2a8cba3 100644 --- a/internal/webfetch/webfetch.go +++ b/internal/webfetch/webfetch.go @@ -122,7 +122,6 @@ func staticAssetExt(rawURL string) string { } func webFetchRunner(cfg actool.WebFetchConfig) func(context.Context, json.RawMessage, *actool.ToolContext) (actool.Result, error) { - userAgent := httpidentity.UserAgent() client := newFetchClient(cfg) // direct is the no-proxy fallback used when a proxied fetch fails for a // proxy-caused reason (closed tunnel / failed CONNECT / TLS or HTTP2 mismatch): @@ -152,6 +151,7 @@ func webFetchRunner(cfg actool.WebFetchConfig) func(context.Context, json.RawMes if err != nil { return actool.Errorf("Error: " + err.Error()), nil } + userAgent := httpidentity.UserAgent() req.Header.Set("User-Agent", userAgent) resp, err := client.Do(req) // One direct retry when the proxied attempt failed for a proxy-caused reason. diff --git a/internal/webfetch/webfetch_test.go b/internal/webfetch/webfetch_test.go index 1760163..bf575e9 100644 --- a/internal/webfetch/webfetch_test.go +++ b/internal/webfetch/webfetch_test.go @@ -12,6 +12,7 @@ import ( "path/filepath" "reflect" "strings" + "sync" "sync/atomic" "testing" @@ -37,13 +38,21 @@ func TestIdentityAcrossRoutes(t *testing.T) { t.Run(route+"/"+ua, func(t *testing.T) { t.Setenv("ARTEX_HTTP_USER_AGENT", ua) var targetCalls, proxyCalls atomic.Int32 + var mu sync.Mutex + var selected string checkUA := func(r *http.Request) { - if got := r.Header.Get("User-Agent"); got != ua { + mu.Lock() + defer mu.Unlock() + got := r.UserAgent() + if ua == "" && !strings.HasPrefix(got, "Mozilla/5.0 ") { + t.Errorf("random UA missing: %q", got) + } else if ua != "" && got != ua { t.Errorf("User-Agent = %q, want %q", got, ua) } - if _, present := r.Header["User-Agent"]; ua == "" && present { - t.Error("empty User-Agent present on wire") + if selected != "" && selected != got { + t.Errorf("redirect/retry UA changed from %q to %q", selected, got) } + selected = got } target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { targetCalls.Add(1) @@ -137,6 +146,26 @@ func TestTLSAndStaticAssetGuard(t *testing.T) { } } +func TestIdentitySelectedForEachCall(t *testing.T) { + t.Setenv("ARTEX_HTTP_USER_AGENT", "Before-Tool-Creation") + tool := New(actool.WebFetchConfig{}) + seen := make(chan string, 2) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + seen <- r.UserAgent() + fmt.Fprint(w, "ok") + })) + defer server.Close() + for _, want := range []string{"First-Call", "Second-Call"} { + t.Setenv("ARTEX_HTTP_USER_AGENT", want) + if result := callFetch(t, tool, server.URL, false); result.IsError { + t.Fatal(result.Flatten()) + } + if got := <-seen; got != want { + t.Fatalf("identity cached across calls: got %q, want %q", got, want) + } + } +} + func TestSDKMetadataPreserved(t *testing.T) { base := actool.NewWebFetch(actool.WebFetchConfig{}) local := New(actool.WebFetchConfig{}) diff --git a/selfupdate/github.go b/selfupdate/github.go index 4dd79e0..3b09438 100644 --- a/selfupdate/github.go +++ b/selfupdate/github.go @@ -127,7 +127,7 @@ func FetchLatest(ctx context.Context, c *http.Client) (*Release, error) { return nil, err } req.Header.Set("Accept", "application/vnd.github+json") - httpidentity.ApplyRequired(req) + httpidentity.Apply(req) resp, err := c.Do(req) if err != nil { diff --git a/selfupdate/github_test.go b/selfupdate/github_test.go index 1a2e07b..7b8c953 100644 --- a/selfupdate/github_test.go +++ b/selfupdate/github_test.go @@ -59,7 +59,7 @@ func TestFetchLatestMissingRelease(t *testing.T) { func TestUpdateUserAgent(t *testing.T) { for _, tc := range []struct{ env, want string }{ - {"", "Go-http-client/1.1"}, + {"", ""}, {"Captured-UA/1.0", "Captured-UA/1.0"}, } { t.Run(tc.env, func(t *testing.T) { @@ -67,7 +67,9 @@ func TestUpdateUserAgent(t *testing.T) { calls := 0 client := &http.Client{Transport: releaseTransport(func(req *http.Request) (*http.Response, error) { calls++ - if got := req.UserAgent(); got != tc.want { + if got := req.UserAgent(); tc.want == "" && !strings.HasPrefix(got, "Mozilla/5.0 ") { + t.Errorf("random UA missing: %q", got) + } else if tc.want != "" && got != tc.want { t.Errorf("User-Agent = %q, want %q", got, tc.want) } body := `{"tag_name":"v0.3.17"}` diff --git a/selfupdate/stage.go b/selfupdate/stage.go index 56bf939..4ab47ec 100644 --- a/selfupdate/stage.go +++ b/selfupdate/stage.go @@ -231,7 +231,7 @@ func get(ctx context.Context, c *http.Client, rawURL string) (io.ReadCloser, err if err := checkURL(req.URL); err != nil { return nil, err } - httpidentity.ApplyRequired(req) + httpidentity.Apply(req) resp, err := c.Do(req) if err != nil { return nil, err diff --git a/server/egress_probe_test.go b/server/egress_probe_test.go index af68e1e..5c84c7f 100644 --- a/server/egress_probe_test.go +++ b/server/egress_probe_test.go @@ -32,12 +32,11 @@ func TestProbeOneUserAgent(t *testing.T) { t.Run(ua, func(t *testing.T) { t.Setenv("ARTEX_HTTP_USER_AGENT", ua) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if got := r.UserAgent(); got != ua { + if got := r.UserAgent(); ua == "" && !strings.HasPrefix(got, "Mozilla/5.0 ") { + t.Errorf("random UA missing: %q", got) + } else if ua != "" && got != ua { t.Errorf("User-Agent = %q, want %q", got, ua) } - if _, present := r.Header["User-Agent"]; ua == "" && present { - t.Error("empty User-Agent present on wire") - } fmt.Fprint(w, `{"ip":"203.0.113.7"}`) })) defer server.Close() diff --git a/server/http_identity_test.go b/server/http_identity_test.go index f42ec62..013c151 100644 --- a/server/http_identity_test.go +++ b/server/http_identity_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "strings" "testing" ) @@ -23,10 +24,12 @@ func TestTargetHTTPIdentityOverrides(t *testing.T) { t.Run(route+"/"+tc.name, func(t *testing.T) { t.Setenv("ARTEX_HTTP_USER_AGENT", tc.env) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if got := r.UserAgent(); got != tc.want { + if got := r.UserAgent(); tc.name == "default" && !strings.HasPrefix(got, "Mozilla/5.0 ") { + t.Errorf("random UA missing: %q", got) + } else if tc.name != "default" && got != tc.want { t.Errorf("User-Agent = %q, want %q", got, tc.want) } - if _, present := r.Header["User-Agent"]; tc.want == "" && present { + if _, present := r.Header["User-Agent"]; tc.name == "explicit-empty" && present { t.Error("empty User-Agent present on wire") } w.Write([]byte("ok")) diff --git a/skills/api-recon/scripts/harvest_static.py b/skills/api-recon/scripts/harvest_static.py index 1dd40f7..b39ada5 100644 --- a/skills/api-recon/scripts/harvest_static.py +++ b/skills/api-recon/scripts/harvest_static.py @@ -16,16 +16,31 @@ Static SPA bundle harvester. Framework-agnostic; tuned for webpack + Vite. Stdlib only. TLS verification is disabled (recon against self-signed/internal hosts). """ -import sys, os, re, ssl, json, argparse, urllib.request, urllib.parse +import sys, os, re, ssl, secrets, json, argparse, urllib.request, urllib.parse from concurrent.futures import ThreadPoolExecutor CTX = ssl.create_default_context(); CTX.check_hostname = False; CTX.verify_mode = ssl.CERT_NONE +USER_AGENTS = ( + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36', + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) Gecko/20100101 Firefox/128.0', + 'Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.6 Safari/605.1.15', +) + + def user_agent(value=None): - """Explicit value > shared environment; empty omits User-Agent.""" + """Explicit value > environment > random sample; explicit empty omits UA.""" + explicit = value is not None if value is None: value = os.environ.get("ARTEX_HTTP_USER_AGENT", "") value = value.strip(" ") - return value if all(32 <= ord(c) <= 126 for c in value) else "" + if not all(32 <= ord(c) <= 126 for c in value): + return secrets.choice(USER_AGENTS) + if value or explicit: + return value + return secrets.choice(USER_AGENTS) def fetch(url, binary=False, user_agent_value=None): @@ -123,7 +138,7 @@ def extract_routes(js_text): def main(): ap = argparse.ArgumentParser() ap.add_argument("base"); ap.add_argument("outdir") - ap.add_argument("--user-agent", default=None, help="User-Agent override; default: ARTEX_HTTP_USER_AGENT or omitted") + ap.add_argument("--user-agent", default=None, help="User-Agent override; default: ARTEX_HTTP_USER_AGENT or random browser sample") args = ap.parse_args() base, outdir = args.base, args.outdir ua = user_agent(args.user_agent) diff --git a/skills/api-recon/scripts/spider_mpa.py b/skills/api-recon/scripts/spider_mpa.py index 928fe61..896de38 100644 --- a/skills/api-recon/scripts/spider_mpa.py +++ b/skills/api-recon/scripts/spider_mpa.py @@ -14,17 +14,32 @@ the same origin and extracts: Stdlib only. Same-origin, bounded, polite. Provide --cookie for an authed crawl. """ -import sys, os, re, ssl, argparse, urllib.request, urllib.parse, time +import sys, os, re, ssl, secrets, argparse, urllib.request, urllib.parse, time from html.parser import HTMLParser from collections import deque CTX = ssl.create_default_context(); CTX.check_hostname = False; CTX.verify_mode = ssl.CERT_NONE +USER_AGENTS = ( + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36', + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) Gecko/20100101 Firefox/128.0', + 'Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.6 Safari/605.1.15', +) + + def user_agent(value=None): - """Explicit value > shared environment; empty omits User-Agent.""" + """Explicit value > environment > random sample; explicit empty omits UA.""" + explicit = value is not None if value is None: value = os.environ.get("ARTEX_HTTP_USER_AGENT", "") value = value.strip(" ") - return value if all(32 <= ord(c) <= 126 for c in value) else "" + if not all(32 <= ord(c) <= 126 for c in value): + return secrets.choice(USER_AGENTS) + if value or explicit: + return value + return secrets.choice(USER_AGENTS) SKIP_EXT = re.compile(r'\.(css|png|jpe?g|gif|svg|ico|woff2?|ttf|pdf|zip|mp4|webp|js)(\?|$)', re.I) API_HINT = re.compile(r'/(?:api|rest|service|ajax|action|do|rpc|graphql|v\d|admin|backend)\b', re.I) @@ -79,7 +94,7 @@ def main(): ap.add_argument("--cookie", default=""); ap.add_argument("--max", type=int, default=200) ap.add_argument("--depth", type=int, default=4); ap.add_argument("--delay", type=float, default=0.1) ap.add_argument("--include", default=""); ap.add_argument("--exclude", default="logout|signout|delete|remove|destroy") - ap.add_argument("--user-agent", default=None, help="User-Agent override; default: ARTEX_HTTP_USER_AGENT or omitted") + ap.add_argument("--user-agent", default=None, help="User-Agent override; default: ARTEX_HTTP_USER_AGENT or random browser sample") args = ap.parse_args() ua = user_agent(args.user_agent) base = args.base if args.base.startswith("http") else "https://" + args.base diff --git a/skills/api-recon/scripts/test_user_agent.py b/skills/api-recon/scripts/test_user_agent.py index 278c366..a073997 100644 --- a/skills/api-recon/scripts/test_user_agent.py +++ b/skills/api-recon/scripts/test_user_agent.py @@ -17,6 +17,13 @@ SCRIPTS = Path(__file__).resolve().parent DIRECT_ENV = {"no_proxy": "*", "NO_PROXY": "*"} +def load_script(name): + spec = importlib.util.spec_from_file_location(name, SCRIPTS / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + @contextlib.contextmanager def capture_server(): seen = [] @@ -48,13 +55,32 @@ def capture_server(): class UserAgentTests(unittest.TestCase): + def assert_identity(self, seen, expected, samples): + self.assertTrue(seen) + if expected == "random": + self.assertIn(seen[0][1], samples) + expected = seen[0][1] + self.assertTrue(all(ua == expected for _, ua, _ in seen), seen) + + def test_random_selection_and_explicit_override(self): + for name in ("harvest_static", "spider_mpa"): + module = load_script(name) + for sample in module.USER_AGENTS: + with self.subTest(script=name, sample=sample): + with patch.dict(os.environ, {"ARTEX_HTTP_USER_AGENT": ""}), patch.object(module.secrets, "choice", return_value=sample) as choose: + self.assertEqual(module.user_agent(), sample) + choose.assert_called_once_with(module.USER_AGENTS) + with patch.dict(os.environ, {"ARTEX_HTTP_USER_AGENT": "Configured-UA"}), patch.object(module.secrets, "choice") as choose: + self.assertEqual(module.user_agent(), "Configured-UA") + self.assertEqual(module.user_agent("Override-UA"), "Override-UA") + self.assertEqual(module.user_agent(""), "") + choose.assert_not_called() + def test_fetch_and_redirect(self): for name in ("harvest_static", "spider_mpa"): - spec = importlib.util.spec_from_file_location(name, SCRIPTS / f"{name}.py") - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - for value, expected in (("", None), (" Captured-UA/1.0 ", "Captured-UA/1.0"), - ("ok\r\nX-Injected: yes", None), ("客户端", None)): + module = load_script(name) + for value, expected in (("", "random"), (" Captured-UA/1.0 ", "Captured-UA/1.0"), + ("ok\r\nX-Injected: yes", "random"), ("客户端", "random")): with self.subTest(script=name, value=value), capture_server() as (base, seen): with patch.dict(os.environ, {**DIRECT_ENV, "ARTEX_HTTP_USER_AGENT": value}): if name == "harvest_static": @@ -65,23 +91,25 @@ class UserAgentTests(unittest.TestCase): self.assertTrue(all(cookie == "session=sample" for _, _, cookie in seen)) self.assertIn("Page", body) self.assertEqual([path for path, _, _ in seen], ["/start", "/page"]) - self.assertTrue(all(ua == expected for _, ua, _ in seen), seen) + self.assert_identity(seen, expected, module.USER_AGENTS) def test_cli_precedence_and_outputs(self): for name in ("harvest_static", "spider_mpa"): - for flags, expected in (([], "Environment-UA/1.0"), - (["--user-agent", "Override-UA/1.0"], "Override-UA/1.0"), - (["--user-agent", ""], None)): + module = load_script(name) + for env_ua, flags, expected in (("", [], "random"), + ("Environment-UA/1.0", [], "Environment-UA/1.0"), + ("Environment-UA/1.0", ["--user-agent", "Override-UA/1.0"], "Override-UA/1.0"), + ("Environment-UA/1.0", ["--user-agent", ""], None)): with self.subTest(script=name, flags=flags), capture_server() as (base, seen): with tempfile.TemporaryDirectory() as outdir: - env = dict(os.environ, **DIRECT_ENV, ARTEX_HTTP_USER_AGENT="Environment-UA/1.0") + env = dict(os.environ, **DIRECT_ENV, ARTEX_HTTP_USER_AGENT=env_ua) command = [sys.executable, str(SCRIPTS / f"{name}.py"), base + "/start", outdir] + flags if name == "spider_mpa": command += ["--max", "1", "--delay", "0"] result = subprocess.run(command, env=env, capture_output=True, text=True, timeout=15) self.assertEqual(result.returncode, 0, result.stderr) self.assertGreaterEqual(len(seen), 2) - self.assertTrue(all(ua == expected for _, ua, _ in seen), seen) + self.assert_identity(seen, expected, module.USER_AGENTS) artifact = "api_static.txt" if name == "harvest_static" else "forms.txt" self.assertTrue((Path(outdir) / artifact).exists())