diff --git a/.env.example b/.env.example index 0dd5fec..78de041 100644 --- a/.env.example +++ b/.env.example @@ -13,3 +13,6 @@ ARTEX_LLM_PROVIDER= ARTEX_LLM_MODEL= ARTEX_LLM_BASE_URL= ARTEX_LLM_PROXY= + +# HTTP User-Agent;目标请求留空不发送,自更新留空使用通用客户端值 +ARTEX_HTTP_USER_AGENT= diff --git a/HTTP_USER_AGENT.md b/HTTP_USER_AGENT.md new file mode 100644 index 0000000..2063068 --- /dev/null +++ b/HTTP_USER_AGENT.md @@ -0,0 +1,54 @@ +# HTTP 请求标识排查 + +本次检查覆盖仓库中的 Go HTTP 请求、Python / JavaScript 脚本、安装脚本、前端示例,以及实际使用的 norma v0.4.3 的 WebFetch / WebSearch 实现。排查范围包括 User-Agent 的大小写变体、请求头构造、代理转发、重定向及代理失败后的直连重试。 + +## 已处理 + +| 通道 | 排查结果与处理 | +|---|---| +| 内置 WebFetch | SDK 将 `norma/0.4` 写死在首发及重试请求。worker、planner、main、chat 通过统一 session 入口使用本地适配器;保留 SDK 的工具 schema、权限、静态资源限制、代理、CA 验证及 HTML / RECON 输出。 | +| 资产自动补全 | 移除 `artex-enrich/1.0`,HTTP 探测使用统一策略。 | +| 出口连通性探测 | 额外发现 `artex-egress-probe/1.0`,已移除并接入统一策略。 | +| 静态采集 / 同源爬虫模板 | 移除 `spa-api-recon` / `spa-api-recon spider`,避免 urllib 自动填入自己的默认 UA;入口、下载、重定向及重试沿用同一配置。 | +| HTTP 自定义工具 | 原先没有 ARTEX 专属 UA,但未设置时会由 Go 填充默认值。现采用统一策略,模板显式请求头优先。 | +| 请求重放器 | 原先没有 ARTEX 专属 UA。现采用统一策略,用户显式请求头(包括空值)优先,保持原请求的配置能力。 | +| 记录代理 | 转发缺失 UA 的请求时,Go 会补入默认客户端标识。现保留省略语义,录制中不展示内部用于抑制默认值的空头。保留客户端已有 UA。 | +| GitHub 自更新 | 额外发现查询版本、下载校验和与发布包共用的两处 `artex-selfupdate`。均已移除;GitHub API 要求非空 UA,未配置时使用通用 `Go-http-client/1.1`,不含 ARTEX 名称。 | +| 前端 mock | `ARTEX-worker/1.0` 仅为示例 HTTP 文本,不会发出请求;已删除该行,避免误导。 | + +## 配置 + +`ARTEX_HTTP_USER_AGENT` 统一配置以上通道,默认空值使目标请求不发送此头;自更新采用表中非空兼容值。Go 服务重启后生效;Docker 在 `.env` 设置后重建容器。直接运行 Python 模板时可继承同名环境变量。 + +Python 模板的 `--user-agent VALUE` 优先于环境变量,`--user-agent ''` 强制省略。自定义 HTTP 工具 / 重放器的显式 UA 同样优先于全局值。配置仅接受可打印 ASCII,去除首尾空格;控制字符和非 ASCII 值按空值处理,防止头注入。 + +不随机轮换、不自动冒充浏览器。可以复用授权目标已有流量中的真实值。省略或修改 UA 不能保证客户端不可识别,第三方出口代理也可能再次添加或改写请求头。 + +## 检查过的其他通道与边界 + +| 通道 | 结果 | +|---|---| +| SDK 联网搜索 | norma 的 DDGS 后端含固定 Firefox 128 浏览器 UA,没有 ARTEX / norma 名称;仅发往搜索服务,属于依赖的兼容策略,未改动,也不受此环境变量控制。Brave / Tavily 及 DeepSeek 搜索未发现 ARTEX 专属 UA。 | +| 浏览器自动化 / runtime_harvest.js | 沿用浏览器请求头;运行时转发复制捕获到的请求头,没有硬编码 ARTEX UA。 | +| LLM 请求 / 模型列表 / JEV | 未发现硬编码 ARTEX UA;仍可能有 HTTP 库、SDK、服务协议和用户自定义标识,不套用目标流量配置。 | +| MCP HTTP / SSE | 未发现 ARTEX UA;显式自定义头保留。初始化协议体仍有 `clientInfo.name = artex`,这是对用户配置的 MCP 服务的协议自报身份,本次不改动。 | +| Webhook / 通知 | 未发现硬编码 ARTEX UA;保留用户自定义头及客户端默认行为。 | +| 安装脚本 | 未发现 ARTEX UA;获取 Docker 安装脚本的 curl 仍使用其原生身份。外部命令及用户自建脚本不由本次策略接管。 | +| phpfinger | 仓库未包含其源码,无法核验默认值;需要在独立工具中通过 `--user-agent` 配置。 | + +服务端响应中的 `X-Artex-Auth` 仍用于前端识别 Basic Auth 未认证状态,属于站点自身对访问者的响应协议;它不是向目标站点发送的请求头。本次不改变该协议,也不承诺消除服务端页面、API、TLS 或其他流量特征。 + +## 回归验证 + +- Go:检查真实 HTTP 线上头、配置值 / 空值、代理及失败直连重试、重定向、CA 信任、静态资源限制、HTML / RECON 解析、agent session 注册、自定义工具 / 重放器覆盖、自更新查询及下载和代理录制。 +- Python:本地 HTTP 服务验证默认 / 环境变量 / CLI 覆盖 / 显式空值、非法头值、重定向、Cookie 保留及产物生成;不访问真实目标。 + +本地命令: + +```sh +go test ./internal/httpidentity ./internal/webfetch ./enrich ./agent ./selfupdate ./traffic +go test ./server -run 'TestProbe|TestRepeater|TestRunHTTPTool|TestHTTPTool|TestTargetHTTPIdentity' -count=1 +python3 -m unittest discover -s skills/api-recon/scripts -p 'test_*.py' -v +``` + +服务端完整套件另需隔离 PostgreSQL 测试库;未配置数据库时其中的数据库集成用例会失败。 diff --git a/README.md b/README.md index 451d9c6..1b4c191 100644 --- a/README.md +++ b/README.md @@ -314,6 +314,10 @@ CGO_ENABLED=0 go build -tags embedui -o artex ./cmd/artex **LLM**:`export ANTHROPIC_API_KEY=sk-...`(或 `OPENAI_API_KEY`),也可在 UI 的「LLM 配置」页填写。 可选:`ARTEX_LLM_PROVIDER` / `ARTEX_LLM_MODEL` / `ARTEX_LLM_BASE_URL` / `ARTEX_LLM_PROXY`。 +**HTTP User-Agent**:内置 WebFetch(worker / planner / main / chat,包括代理失败后的直连重试)、资产 HTTP 自动补全、出口连通性探测、HTTP 自定义工具、请求重放器及可选的 `harvest_static.py` / `spider_mpa.py` 默认不发送 User-Agent,不再携带工具名称。通过 `ARTEX_HTTP_USER_AGENT` 设置固定值(例如复用授权目标已有流量中的值);直接运行时导出环境变量并重启服务,Docker 部署在 `.env` 配置后重建容器。两个 Python 模板还支持 `--user-agent '已有流量中的值'`,优先于环境变量;`--user-agent ''` 强制留空。值会去除首尾空格,只接受可打印 ASCII,含控制字符或非 ASCII 的配置忽略并按空值处理。不随机轮换或自动伪装浏览器;省略此请求头不能保证无法识别客户端。自定义工具和重放器中显式提供的请求头优先于全局值。GitHub 自更新查询、校验和及包下载同样移除了工具名,未配置时使用通用 `Go-http-client/1.1` 以满足 GitHub 非空请求头要求。记录代理保留请求头省略语义。此设置不影响浏览器、SDK 联网搜索、LLM / MCP / Webhook 服务通信或外部独立工具;本仓库未包含 phpfinger,其 `--user-agent` 需在该工具中单独配置。 + +完整排查范围、额外发现及未接管的服务通信见 [HTTP 请求标识排查](HTTP_USER_AGENT.md)。 + **并发**:每个任务的 work agent 数在「系统设置」里配置(默认 3)。 **常用参数**:`./start.sh -addr :8787 -proxy :8788`(`-addr` 前端+API,`-proxy` 流量录制代理)。启动脚本会把参数原样透传给 `artex`。 diff --git a/agent/capture.go b/agent/capture.go index 61eba3f..394f0e0 100644 --- a/agent/capture.go +++ b/agent/capture.go @@ -26,7 +26,7 @@ import ( // contiguous run is coalesced into a single record so the trace shows whole // messages, not dozens of fragments. func captureRun(ctx context.Context, opts agentcore.Options, input string, emit func(db.Activity)) (string, harness.TerminalReason, error) { - s := agentcore.NewSession(opts) + s := newAgentSession(opts) defer s.Close() // release the session's background-task manager (temp dir + processes) return captureRunSession(ctx, s, input, emit) } diff --git a/agent/chat.go b/agent/chat.go index 86aaac0..a897872 100644 --- a/agent/chat.go +++ b/agent/chat.go @@ -173,7 +173,7 @@ func (c *ChatAgent) Chat(ctx context.Context, agentKey, sessionID, message strin // 实验功能:开启后由 noa 接管上下文压缩(归档集中在 /noa/ 下,持久)。 enableNoa(&opts, c.noaEnabledFn, c.workDir, "chat-"+sessionID, noaWarn("chat-"+sessionID)) ctx = attachSideCapture(ctx, &opts) - s := agentcore.NewSession(opts) + s := newAgentSession(opts) defer s.Close() // reload prior conversation so the agent has context across turns (each Chat is // a fresh session). First turn: no file yet → Resume loads nothing and proceeds. diff --git a/agent/mainagent.go b/agent/mainagent.go index e59cdf5..d746df8 100644 --- a/agent/mainagent.go +++ b/agent/mainagent.go @@ -184,7 +184,7 @@ func (m *MainAgent) Chat(ctx context.Context, taskID int64, mainSeg int, as *db. } enableNoa(&opts, m.noaEnabledFn, m.workDir, noaSession, noaWarn(noaSession)) ctx = attachSideCapture(ctx, &opts) - s := agentcore.NewSession(opts) + s := newAgentSession(opts) defer s.Close() // reload the prior conversation from the transcript so the agent has context // across turns (each Chat is a fresh session; without this it can't see earlier diff --git a/agent/webfetch.go b/agent/webfetch.go new file mode 100644 index 0000000..c1411dd --- /dev/null +++ b/agent/webfetch.go @@ -0,0 +1,24 @@ +package agent + +import ( + "github.com/Autumn-27/artex/internal/webfetch" + "github.com/Autumn-27/norma/agentcore" + actool "github.com/Autumn-27/norma/tool" +) + +// newAgentSession installs the configurable WebFetch before the SDK builds the +// registry, so every agent role gets one tool and the same proxy/TLS settings. +func newAgentSession(opts agentcore.Options) *agentcore.Session { + if opts.EnableWebFetch { + opts.EnableWebFetch = false + if opts.Tools == nil { + opts.Tools = actool.DefaultTools() + } + opts.Tools = append(opts.Tools, webfetch.New(actool.WebFetchConfig{ + Proxy: opts.WebFetchProxy, + CACert: opts.WebFetchCACert, + InsecureTLS: opts.WebFetchInsecureTLS, + })) + } + return agentcore.NewSession(opts) +} diff --git a/agent/webfetch_test.go b/agent/webfetch_test.go new file mode 100644 index 0000000..dcabe00 --- /dev/null +++ b/agent/webfetch_test.go @@ -0,0 +1,67 @@ +package agent + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/Autumn-27/norma/agentcore" + "github.com/Autumn-27/norma/llm" + "github.com/Autumn-27/norma/permission" +) + +type fetchFixtureProvider struct { + autoTitleProvider + t *testing.T + url string + calls int +} + +func (p *fetchFixtureProvider) Complete(_ context.Context, req llm.CompletionRequest) (llm.Message, string, llm.Usage, error) { + p.calls++ + if p.calls == 1 { + counts := map[string]int{} + for _, tool := range req.Tools { + counts[tool.Name]++ + } + if counts["WebFetch"] != 1 || counts["Read"] != 1 || counts["Bash"] != 1 { + p.t.Errorf("session tool registration = %v", counts) + } + input, _ := json.Marshal(map[string]string{"url": p.url}) + return llm.Message{Role: llm.RoleAssistant, Content: []llm.ContentBlock{{Type: llm.BlockToolUse, ID: "fetch-fixture", Name: "WebFetch", Input: input}}}, "tool_use", llm.Usage{}, nil + } + return llm.Message{Role: llm.RoleAssistant, Content: []llm.ContentBlock{llm.TextBlock("finished")}}, "end_turn", llm.Usage{}, nil +} + +func TestAgentSessionUsesConfiguredWebFetch(t *testing.T) { + for _, ua := range []string{"", "Captured-UA/1.0"} { + t.Run(ua, func(t *testing.T) { + t.Setenv("ARTEX_HTTP_USER_AGENT", ua) + calls := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + if r.UserAgent() != ua { + t.Errorf("session sent User-Agent %q, want %q", r.UserAgent(), ua) + } + w.Write([]byte("fixture")) + })) + defer server.Close() + provider := &fetchFixtureProvider{t: t, url: server.URL} + session := newAgentSession(agentcore.Options{ + Provider: provider, NonStreaming: true, EnableWebFetch: true, + PermissionMode: permission.ModeBypass, MaxTurns: 3, + }) + defer session.Close() + for _, err := range session.Prompt(t.Context(), "fetch local fixture") { + if err != nil { + t.Fatal(err) + } + } + if calls != 1 { + t.Fatalf("WebFetch calls = %d, want 1", calls) + } + }) + } +} diff --git a/agent/worker.go b/agent/worker.go index 4aa31c2..e08b199 100644 --- a/agent/worker.go +++ b/agent/worker.go @@ -488,7 +488,7 @@ func (w *Worker) execute(ctx context.Context, name string, taskID int64, as *db. noaSession := WorkerSessionID(ts.ID(), intent.ID) enableNoa(&opts, w.noaEnabledFn, w.workDir, noaSession, noaWarn(noaSession)) ctx = attachSideCapture(ctx, &opts) - s := agentcore.NewSession(opts) + s := newAgentSession(opts) defer s.Close() // release the session's background-task manager (temp dir + processes) // Resume prior conversation if this intent was paused/blocked/exhausted and is diff --git a/docker-compose.yml b/docker-compose.yml index 4aef029..f9685c5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,6 +28,7 @@ services: ARTEX_LLM_MODEL: ${ARTEX_LLM_MODEL:-} ARTEX_LLM_BASE_URL: ${ARTEX_LLM_BASE_URL:-} ARTEX_LLM_PROXY: ${ARTEX_LLM_PROXY:-} + ARTEX_HTTP_USER_AGENT: ${ARTEX_HTTP_USER_AGENT:-} ports: - "8787:8787" - "127.0.0.1:8788:8788" diff --git a/enrich/enrich.go b/enrich/enrich.go index 7c8fc13..62d7b37 100644 --- a/enrich/enrich.go +++ b/enrich/enrich.go @@ -20,6 +20,7 @@ import ( "time" "github.com/Autumn-27/artex/db" + "github.com/Autumn-27/artex/internal/httpidentity" "github.com/miekg/dns" "github.com/projectdiscovery/dnsx/libs/dnsx" @@ -230,7 +231,7 @@ func (e *Engine) doHTTP(id int64, rawURL string) { if err != nil { return } - req.Header.Set("User-Agent", "artex-enrich/1.0") + httpidentity.Apply(req) resp, err := e.client.Do(req) if err != nil { return diff --git a/enrich/enrich_test.go b/enrich/enrich_test.go new file mode 100644 index 0000000..e4ddbd6 --- /dev/null +++ b/enrich/enrich_test.go @@ -0,0 +1,38 @@ +package enrich + +import ( + "errors" + "net/http" + "net/http/httptest" + "testing" +) + +// Stop after receiving the probe so this request regression needs no database. +func TestProbeUserAgent(t *testing.T) { + for _, ua := range []string{"", "Captured-UA/1.0"} { + t.Run(ua, func(t *testing.T) { + t.Setenv("ARTEX_HTTP_USER_AGENT", ua) + calls := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + if got := r.UserAgent(); got != ua { + t.Errorf("User-Agent = %q, want %q", got, ua) + } + if _, present := r.Header["User-Agent"]; ua == "" && present { + t.Error("empty User-Agent present on wire") + } + http.Redirect(w, r, "/stop", http.StatusFound) + })) + defer server.Close() + client := buildClient(nil) + client.CheckRedirect = func(*http.Request, []*http.Request) error { + return errors.New("probe captured") + } + e := &Engine{client: client} + e.doHTTP(1, server.URL) + if calls != 1 { + t.Fatalf("probe calls = %d, want 1", calls) + } + }) + } +} diff --git a/internal/httpidentity/useragent.go b/internal/httpidentity/useragent.go new file mode 100644 index 0000000..5c06366 --- /dev/null +++ b/internal/httpidentity/useragent.go @@ -0,0 +1,37 @@ +// Package httpidentity defines the explicit identity for target-facing HTTP requests. +package httpidentity + +import ( + "net/http" + "os" + "strings" +) + +// UserAgent returns the configured value, or an empty string to omit the header. +// Use printable ASCII for consistent behavior across Go and the Python templates. +// Ignore malformed values rather than allowing header injection. +func UserAgent() string { + value := strings.Trim(os.Getenv("ARTEX_HTTP_USER_AGENT"), " ") + for _, c := range value { + if c < 32 || c > 126 { + return "" + } + } + return value +} + +// Apply must set even an empty value: deleting the header makes net/http send +// its own Go-http-client default instead of omitting User-Agent on the wire. +func Apply(req *http.Request) { + req.Header.Set("User-Agent", UserAgent()) +} + +// ApplyRequired keeps a nonempty, generic library identity for services that +// require User-Agent (notably the GitHub API), without an ARTEX identifier. +func ApplyRequired(req *http.Request) { + value := UserAgent() + if value == "" { + value = "Go-http-client/1.1" + } + req.Header.Set("User-Agent", value) +} diff --git a/internal/httpidentity/useragent_test.go b/internal/httpidentity/useragent_test.go new file mode 100644 index 0000000..eb74fd7 --- /dev/null +++ b/internal/httpidentity/useragent_test.go @@ -0,0 +1,43 @@ +package httpidentity + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func TestApplyOnWire(t *testing.T) { + for _, tc := range []struct{ name, value, want string }{ + {"empty", "", ""}, + {"configured", " Captured-UA/1.0 ", "Captured-UA/1.0"}, + {"whitespace", " ", ""}, + {"injection", "ok\r\nX-Injected: yes", ""}, + {"non-ascii", "客户端", ""}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("ARTEX_HTTP_USER_AGENT", tc.value) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get("User-Agent"); got != tc.want { + t.Errorf("User-Agent = %q, want %q", got, tc.want) + } + if _, present := r.Header["User-Agent"]; tc.want == "" && present { + t.Error("empty User-Agent must be absent on the wire") + } + if r.Header.Get("X-Injected") != "" { + t.Error("header injection") + } + })) + defer server.Close() + req, err := http.NewRequest(http.MethodGet, server.URL, nil) + if err != nil { + t.Fatal(err) + } + Apply(req) + resp, err := server.Client().Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + }) + } +} diff --git a/internal/webfetch/webfetch.go b/internal/webfetch/webfetch.go new file mode 100644 index 0000000..0942871 --- /dev/null +++ b/internal/webfetch/webfetch.go @@ -0,0 +1,467 @@ +// Adapted from github.com/Autumn-27/norma/tool/webfetch.go at v0.4.3. +// Keep the SDK tool metadata and fetch/render behavior; use ARTEX's explicit +// HTTP identity for both the proxy attempt and its direct fallback. This local +// adapter is needed until the SDK exposes request headers in actool.WebFetchConfig. +package webfetch + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "regexp" + "strings" + "time" + + "github.com/Autumn-27/artex/internal/httpidentity" + actool "github.com/Autumn-27/norma/tool" + "golang.org/x/net/html" +) + +// New keeps the SDK's schema, permission checks and tool metadata. +func New(cfg actool.WebFetchConfig) actool.CoreTool { + base := actool.NewWebFetch(cfg) + return actool.Build(actool.Spec{ + Name: base.Name(), Description: base.Description(), Prompt: base.Prompt(), + Schema: base.InputSchema(), Permissions: base.CheckPermissions, + ReadOnly: base.IsReadOnly, Concurrent: base.IsConcurrencySafe, + Run: webFetchRunner(cfg), + }) +} + +const maxFetchBytes = 2 << 20 // 2 MiB + +// newFetchClient returns an HTTP client honoring the proxy / TLS config. +func newFetchClient(cfg actool.WebFetchConfig) *http.Client { + tr := &http.Transport{} + if cfg.Proxy != "" { + if pu, err := url.Parse(cfg.Proxy); err == nil { + tr.Proxy = http.ProxyURL(pu) + } + } + switch { + case cfg.InsecureTLS: + tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + case cfg.CACert != "": + // Trust the proxy's CA in addition to the system roots, so certs it + // re-signs verify while every other connection stays fully checked. + if pem, err := os.ReadFile(cfg.CACert); err == nil { + pool, _ := x509.SystemCertPool() + if pool == nil { + pool = x509.NewCertPool() + } + if pool.AppendCertsFromPEM(pem) { + tr.TLSClientConfig = &tls.Config{RootCAs: pool} + } + } + } + return &http.Client{Transport: tr} +} + +// proxyLikelyErr reports whether a fetch error is the kind a recording/MITM proxy +// introduces — a closed tunnel (EOF / connection reset), a failed CONNECT, or a +// TLS/HTTP2 mismatch — so a direct (no-proxy) retry may succeed. Name-resolution +// failures and caller cancellation/timeout are target- or caller-side problems a +// direct retry won't fix, so those return false (no retry). +func proxyLikelyErr(err error) bool { + if err == nil { + return false + } + s := strings.ToLower(err.Error()) + for _, n := range []string{"no such host", "context canceled", "context deadline", "timeout"} { + if strings.Contains(s, n) { + return false + } + } + for _, p := range []string{ + "eof", "connection reset", "connection refused", "proxyconnect", + "broken pipe", "forcibly closed", "malformed", "http2", "http/2", + "tls:", "protocol error", + } { + if strings.Contains(s, p) { + return true + } + } + return false +} + +// staticAssetExts are file extensions WebFetch refuses: linked resources (scripts, +// styles, source maps, images, fonts, media, wasm) that are not page content. +var staticAssetExts = map[string]bool{ + ".js": true, ".mjs": true, ".cjs": true, ".css": true, ".map": true, + ".png": true, ".jpg": true, ".jpeg": true, ".gif": true, ".svg": true, + ".webp": true, ".avif": true, ".ico": true, ".bmp": true, + ".woff": true, ".woff2": true, ".ttf": true, ".otf": true, ".eot": true, + ".wasm": true, ".mp4": true, ".webm": true, ".mp3": true, ".wav": true, ".ogg": true, +} + +// staticAssetExt returns the offending extension (with dot) if rawURL points at a +// static asset, or "" otherwise. The query string and fragment are ignored, so +// "/app.js?v=3" is still caught. +func staticAssetExt(rawURL string) string { + path := rawURL + if u, err := url.Parse(rawURL); err == nil { + path = u.Path + } + if i := strings.LastIndexByte(path, '/'); i >= 0 { + path = path[i+1:] // last segment only + } + dot := strings.LastIndexByte(path, '.') + if dot < 0 { + return "" + } + if ext := strings.ToLower(path[dot:]); staticAssetExts[ext] { + return ext + } + return "" +} + +func webFetchRunner(cfg actool.WebFetchConfig) func(context.Context, json.RawMessage, *actool.ToolContext) (actool.Result, error) { + userAgent := httpidentity.UserAgent() + client := newFetchClient(cfg) + // direct is the no-proxy fallback used when a proxied fetch fails for a + // proxy-caused reason (closed tunnel / failed CONNECT / TLS or HTTP2 mismatch): + // it bypasses the configured proxy so a recording/MITM hiccup doesn't fail an + // otherwise-reachable URL. nil when no proxy is configured (nothing to bypass). + var direct *http.Client + if cfg.Proxy != "" { + direct = newFetchClient(actool.WebFetchConfig{InsecureTLS: cfg.InsecureTLS}) + } + return func(ctx context.Context, input json.RawMessage, _ *actool.ToolContext) (actool.Result, error) { + var in struct { + URL string `json:"url"` + Extract bool `json:"extract"` + } + if err := json.Unmarshal(input, &in); err != nil { + return actool.Result{}, err + } + if !strings.HasPrefix(in.URL, "http://") && !strings.HasPrefix(in.URL, "https://") { + return actool.Errorf("Error: url must start with http:// or https://"), nil + } + if ext := staticAssetExt(in.URL); ext != "" { + return actool.Errorf("Error: WebFetch does not fetch static assets (" + ext + " file). It reads HTML pages, documentation, and API responses — not linked resources like scripts, styles, images, or fonts. Fetch the page that references this asset instead."), nil + } + cctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + req, err := http.NewRequestWithContext(cctx, http.MethodGet, in.URL, nil) + if err != nil { + return actool.Errorf("Error: " + err.Error()), nil + } + req.Header.Set("User-Agent", userAgent) + resp, err := client.Do(req) + // One direct retry when the proxied attempt failed for a proxy-caused reason. + // A non-proxy error (bad host, caller timeout/cancel) is returned as-is — a + // direct retry wouldn't fix it. The direct response is NOT captured by the proxy. + if err != nil && direct != nil && proxyLikelyErr(err) { + if r2, e2 := http.NewRequestWithContext(cctx, http.MethodGet, in.URL, nil); e2 == nil { + r2.Header.Set("User-Agent", userAgent) + if resp2, err2 := direct.Do(r2); err2 == nil { + resp, err = resp2, nil + } else { + err = err2 // surface the direct failure (more representative of the target) + } + } + } + if err != nil { + return actool.Errorf("Error fetching URL: " + err.Error()), nil + } + defer resp.Body.Close() + body, _ := io.ReadAll(io.LimitReader(resp.Body, maxFetchBytes)) + + header := fmt.Sprintf("[%d %s] %s\n", resp.StatusCode, http.StatusText(resp.StatusCode), in.URL) + // Cheap fingerprint signal that HTML→Markdown would otherwise hide. + if sv := resp.Header.Get("Server"); sv != "" { + header += "Server: " + sv + "\n" + } + if xp := resp.Header.Get("X-Powered-By"); xp != "" { + header += "X-Powered-By: " + xp + "\n" + } + + var out string + if strings.Contains(strings.ToLower(resp.Header.Get("content-type")), "html") { + doc, perr := html.Parse(strings.NewReader(string(body))) + if perr != nil { // html.Parse is lenient; this is effectively unreachable + out = string(body) + } else { + out = htmlToMarkdown(doc) + if in.Extract { + if recon := extractRecon(doc); recon != "" { + out += "\n\n" + recon + } + } + } + } else { + out = string(body) + } + return actool.Text(truncate(header+strings.TrimSpace(out), 50000)), nil + } +} + +var ( + reWS = regexp.MustCompile(`[ \t]+`) + reLineTrim = regexp.MustCompile(` *\n *`) // strip spaces hugging a newline + reBlankLines = regexp.MustCompile(`\n{2,}`) // collapse blank lines → one newline + // reEndpoint matches quoted absolute URLs and root-relative paths in inline JS. + reEndpoint = regexp.MustCompile("[\"'`](https?://[^\"'`\\s]+|/[A-Za-z0-9._~$&'()*+,;=:@%/?#\\[\\]-]{2,})[\"'`]") +) + +// skipMarkdown are elements whose subtree carries no readable prose. +var skipMarkdown = map[string]bool{"script": true, "style": true, "noscript": true, "svg": true, "template": true} + +// htmlToMarkdown renders the visible content of a parsed document as Markdown. +// Text nodes are already entity-decoded by the parser, so no unescaping is needed. +func htmlToMarkdown(doc *html.Node) string { + var b strings.Builder + renderNode(&b, doc) + s := reWS.ReplaceAllString(b.String(), " ") + s = reLineTrim.ReplaceAllString(s, "\n") // drop indentation / trailing spaces per line + s = reBlankLines.ReplaceAllString(s, "\n") // no blank lines — compact, token-lean output + return strings.TrimSpace(s) +} + +func renderNode(b *strings.Builder, n *html.Node) { + switch n.Type { + case html.TextNode: + b.WriteString(strings.ReplaceAll(n.Data, "\n", " ")) + return + case html.ElementNode: + if skipMarkdown[n.Data] { + return + } + default: + if n.Type != html.DocumentNode { + return + } + } + tag := "" + if n.Type == html.ElementNode { + tag = n.Data + } + switch tag { + case "h1", "h2", "h3", "h4", "h5", "h6": + b.WriteString("\n\n" + strings.Repeat("#", int(tag[1]-'0')) + " ") + renderChildren(b, n) + b.WriteString("\n\n") + case "title": + b.WriteString("\n\n# ") + renderChildren(b, n) + b.WriteString("\n\n") + case "p", "div", "section", "article", "header", "footer", "main", "ul", "ol", "table", "tr", "blockquote": + b.WriteString("\n\n") + renderChildren(b, n) + b.WriteString("\n\n") + case "br": + b.WriteString("\n") + case "hr": + b.WriteString("\n\n---\n\n") + case "li": + b.WriteString("\n- ") + renderChildren(b, n) + case "a": + var inner strings.Builder + renderChildren(&inner, n) + text := strings.TrimSpace(inner.String()) + if href := attr(n, "href"); href != "" && text != "" { + fmt.Fprintf(b, "[%s](%s)", text, href) + } else { + b.WriteString(text) + } + case "img": + if src := attr(n, "src"); src != "" { + fmt.Fprintf(b, "![%s](%s)", attr(n, "alt"), src) + } + case "strong", "b": + b.WriteString("**") + renderChildren(b, n) + b.WriteString("**") + case "em", "i": + b.WriteString("*") + renderChildren(b, n) + b.WriteString("*") + case "code": + b.WriteString("`") + renderChildren(b, n) + b.WriteString("`") + case "pre": + var inner strings.Builder + textOnly(&inner, n) + b.WriteString("\n\n```\n" + strings.TrimSpace(inner.String()) + "\n```\n\n") + default: + renderChildren(b, n) + } +} + +func renderChildren(b *strings.Builder, n *html.Node) { + for c := n.FirstChild; c != nil; c = c.NextSibling { + renderNode(b, c) + } +} + +// extractRecon walks the document for security-relevant signal that Markdown +// rendering drops: comments, script sources, inline endpoints, forms, meta, links. +func extractRecon(doc *html.Node) string { + var comments, scriptSrcs, inlineEndpoints, forms, metas, links []string + seenLink := map[string]bool{} + + var walk func(n *html.Node) + walk = func(n *html.Node) { + switch n.Type { + case html.CommentNode: + if c := oneLine(n.Data, 300); c != "" { + comments = append(comments, c) + } + case html.ElementNode: + switch n.Data { + case "script": + if src := attr(n, "src"); src != "" { + scriptSrcs = append(scriptSrcs, src) + } else { + var body strings.Builder + textOnly(&body, n) + for _, m := range reEndpoint.FindAllStringSubmatch(body.String(), 40) { + inlineEndpoints = append(inlineEndpoints, m[1]) + } + } + case "form": + forms = append(forms, describeForm(n)) + case "meta": + name := attr(n, "name") + if name == "" { + name = attr(n, "property") + } + if content := attr(n, "content"); name != "" && content != "" { + metas = append(metas, name+"="+oneLine(content, 120)) + } + case "a", "link": + if h := attr(n, "href"); h != "" && !seenLink[h] { + seenLink[h] = true + links = append(links, h) + } + } + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + walk(c) + } + } + walk(doc) + + var b strings.Builder + b.WriteString("=== RECON ===") + reconSection(&b, "COMMENTS", comments) + reconSection(&b, "SCRIPTS (src)", dedup(scriptSrcs)) + reconSection(&b, "INLINE ENDPOINTS", dedup(inlineEndpoints)) + reconSection(&b, "FORMS", forms) + reconSection(&b, "META", metas) + reconSection(&b, "LINKS", capSlice(links, 100)) + if b.Len() == len("=== RECON ===") { + return "" // nothing worth reporting + } + return b.String() +} + +// describeForm summarizes a
as "METHOD action → name(type[=value]) …". +// Hidden inputs keep their value (CSRF tokens, parameter defaults matter in recon). +func describeForm(form *html.Node) string { + method := strings.ToUpper(attr(form, "method")) + if method == "" { + method = "GET" + } + var inputs []string + var walk func(n *html.Node) + walk = func(n *html.Node) { + if n.Type == html.ElementNode { + switch n.Data { + case "input", "textarea", "select": + typ := attr(n, "type") + if typ == "" { + typ = n.Data + } + desc := attr(n, "name") + "(" + typ + if typ == "hidden" { + if v := attr(n, "value"); v != "" { + desc += "=" + oneLine(v, 60) + } + } + inputs = append(inputs, desc+")") + } + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + walk(c) + } + } + walk(form) + return fmt.Sprintf("%s %s → %s", method, attr(form, "action"), strings.Join(inputs, " ")) +} + +// --- small helpers --- + +func attr(n *html.Node, key string) string { + for _, a := range n.Attr { + if a.Key == key { + return a.Val + } + } + return "" +} + +func textOnly(b *strings.Builder, n *html.Node) { + if n.Type == html.TextNode { + b.WriteString(n.Data) + return + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + textOnly(b, c) + } +} + +func oneLine(s string, max int) string { + s = strings.Join(strings.Fields(s), " ") + if len(s) > max { + return s[:max] + "…" + } + return s +} + +func reconSection(b *strings.Builder, title string, items []string) { + if len(items) == 0 { + return + } + b.WriteString("\n\n## " + title + "\n") + for _, it := range items { + b.WriteString("- " + it + "\n") + } +} + +func dedup(in []string) []string { + seen := map[string]bool{} + out := in[:0:0] + for _, s := range in { + if s == "" || seen[s] { + continue + } + seen[s] = true + out = append(out, s) + } + return out +} + +func capSlice(in []string, n int) []string { + if len(in) <= n { + return in + } + return append(in[:n:n], fmt.Sprintf("… (%d more)", len(in)-n)) +} + +// truncate retains the SDK's head/tail output limit. +func truncate(s string, max int) string { + if max <= 0 || len(s) <= max { + return s + } + half := max / 2 + return s[:half] + fmt.Sprintf("\n\n... [%d characters truncated] ...\n\n", len(s)-max) + s[len(s)-half:] +} diff --git a/internal/webfetch/webfetch_test.go b/internal/webfetch/webfetch_test.go new file mode 100644 index 0000000..1760163 --- /dev/null +++ b/internal/webfetch/webfetch_test.go @@ -0,0 +1,146 @@ +package webfetch + +import ( + "context" + "encoding/json" + "encoding/pem" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "reflect" + "strings" + "sync/atomic" + "testing" + + actool "github.com/Autumn-27/norma/tool" +) + +func callFetch(t *testing.T, tool actool.CoreTool, target string, extract bool) actool.Result { + t.Helper() + input, err := json.Marshal(map[string]any{"url": target, "extract": extract}) + if err != nil { + t.Fatal(err) + } + result, err := tool.Call(context.Background(), input, nil) + if err != nil { + t.Fatal(err) + } + return result +} + +func TestIdentityAcrossRoutes(t *testing.T) { + for _, ua := range []string{"", "Captured-UA/1.0"} { + for _, route := range []string{"direct", "proxy", "fallback"} { + t.Run(route+"/"+ua, func(t *testing.T) { + t.Setenv("ARTEX_HTTP_USER_AGENT", ua) + var targetCalls, proxyCalls atomic.Int32 + checkUA := func(r *http.Request) { + if got := r.Header.Get("User-Agent"); got != ua { + t.Errorf("User-Agent = %q, want %q", got, ua) + } + if _, present := r.Header["User-Agent"]; ua == "" && present { + t.Error("empty User-Agent present on wire") + } + } + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + targetCalls.Add(1) + checkUA(r) + if r.URL.Path == "/start" { + http.Redirect(w, r, "/page", http.StatusFound) + return + } + w.Header().Set("Content-Type", "text/html") + fmt.Fprint(w, `

Page

`) + })) + defer target.Close() + proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + proxyCalls.Add(1) + checkUA(r) + if route == "fallback" { + conn, _, err := w.(http.Hijacker).Hijack() + if err != nil { + t.Error(err) + return + } + conn.Close() // Deterministic EOF, causing the SDK-compatible retry. + return + } + req := r.Clone(r.Context()) + req.RequestURI = "" + // A forwarding proxy must preserve absence, like ARTEX's sink. + if _, present := req.Header["User-Agent"]; !present { + req.Header.Set("User-Agent", "") + } + resp, err := target.Client().Do(req) + if err != nil { + t.Error(err) + w.WriteHeader(http.StatusBadGateway) + return + } + defer resp.Body.Close() + for key, values := range resp.Header { + w.Header()[key] = values + } + w.WriteHeader(resp.StatusCode) + io.Copy(w, resp.Body) + })) + defer proxy.Close() + cfg := actool.WebFetchConfig{} + if route != "direct" { + cfg.Proxy = proxy.URL + } + result := callFetch(t, New(cfg), target.URL+"/start", true) + for _, want := range []string{"[200 OK]", "# Page", "=== RECON ===", "keep", "/app.js", "csrf(hidden=token)"} { + if result.IsError || !strings.Contains(result.Flatten(), want) { + t.Fatalf("missing %q in result: %+v", want, result) + } + } + if targetCalls.Load() != 2 { + t.Errorf("target calls = %d, want 2", targetCalls.Load()) + } + if route != "direct" && proxyCalls.Load() == 0 { + t.Error("configured proxy was bypassed") + } + }) + } + } +} + +func TestTLSAndStaticAssetGuard(t *testing.T) { + t.Setenv("ARTEX_HTTP_USER_AGENT", "Captured-UA/1.0") + var calls atomic.Int32 + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + if r.UserAgent() != "Captured-UA/1.0" { + t.Errorf("User-Agent = %q", r.UserAgent()) + } + fmt.Fprint(w, "verified") + })) + defer server.Close() + certPath := filepath.Join(t.TempDir(), "ca.pem") + cert := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw}) + if err := os.WriteFile(certPath, cert, 0600); err != nil { + t.Fatal(err) + } + tool := New(actool.WebFetchConfig{CACert: certPath}) + if result := callFetch(t, tool, server.URL, false); result.IsError || !strings.Contains(result.Flatten(), "verified") { + t.Fatalf("CA trust lost: %+v", result) + } + if result := callFetch(t, tool, server.URL+"/app.js?v=1", false); !result.IsError { + t.Fatalf("static asset guard lost: %+v", result) + } + if calls.Load() != 1 { + t.Errorf("static asset triggered network request: calls = %d", calls.Load()) + } +} + +func TestSDKMetadataPreserved(t *testing.T) { + base := actool.NewWebFetch(actool.WebFetchConfig{}) + local := New(actool.WebFetchConfig{}) + if local.Name() != base.Name() || local.Description() != base.Description() || !reflect.DeepEqual(local.InputSchema(), base.InputSchema()) { + t.Fatal("SDK tool contract changed") + } +} diff --git a/selfupdate/github.go b/selfupdate/github.go index 00503d5..4dd79e0 100644 --- a/selfupdate/github.go +++ b/selfupdate/github.go @@ -9,6 +9,8 @@ import ( "strconv" "strings" "time" + + "github.com/Autumn-27/artex/internal/httpidentity" ) // Repo 是发布源。写死而不是做成配置项:更新源可配等于给任何能改配置的人一条 @@ -125,7 +127,7 @@ func FetchLatest(ctx context.Context, c *http.Client) (*Release, error) { return nil, err } req.Header.Set("Accept", "application/vnd.github+json") - req.Header.Set("User-Agent", "artex-selfupdate") + httpidentity.ApplyRequired(req) resp, err := c.Do(req) if err != nil { diff --git a/selfupdate/github_test.go b/selfupdate/github_test.go index a4e1376..1a2e07b 100644 --- a/selfupdate/github_test.go +++ b/selfupdate/github_test.go @@ -56,3 +56,36 @@ func TestFetchLatestMissingRelease(t *testing.T) { t.Fatalf("unexpected missing-release error: %v", err) } } + +func TestUpdateUserAgent(t *testing.T) { + for _, tc := range []struct{ env, want string }{ + {"", "Go-http-client/1.1"}, + {"Captured-UA/1.0", "Captured-UA/1.0"}, + } { + t.Run(tc.env, func(t *testing.T) { + t.Setenv("ARTEX_HTTP_USER_AGENT", tc.env) + calls := 0 + client := &http.Client{Transport: releaseTransport(func(req *http.Request) (*http.Response, error) { + calls++ + if got := req.UserAgent(); got != tc.want { + t.Errorf("User-Agent = %q, want %q", got, tc.want) + } + body := `{"tag_name":"v0.3.17"}` + return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(body))}, nil + })} + if _, err := FetchLatest(context.Background(), client); err != nil { + t.Fatal(err) + } + for _, name := range []string{"SHA256SUMS", "artex.zip"} { + body, err := get(context.Background(), client, "https://github.com/RuoJi6/ARTEX/releases/download/v0.3.17/"+name) + if err != nil { + t.Fatal(err) + } + body.Close() + } + if calls != 3 { + t.Fatalf("calls = %d, want 3", calls) + } + }) + } +} diff --git a/selfupdate/stage.go b/selfupdate/stage.go index 63a689c..56bf939 100644 --- a/selfupdate/stage.go +++ b/selfupdate/stage.go @@ -13,6 +13,8 @@ import ( "runtime" "strings" "time" + + "github.com/Autumn-27/artex/internal/httpidentity" ) // sumsAsset 是 release.yml 生成的校验和清单,覆盖 Release 里全部 zip。 @@ -229,7 +231,7 @@ func get(ctx context.Context, c *http.Client, rawURL string) (io.ReadCloser, err if err := checkURL(req.URL); err != nil { return nil, err } - req.Header.Set("User-Agent", "artex-selfupdate") + httpidentity.ApplyRequired(req) resp, err := c.Do(req) if err != nil { return nil, err diff --git a/server/customtool.go b/server/customtool.go index 63f87bb..5822b4c 100644 --- a/server/customtool.go +++ b/server/customtool.go @@ -18,6 +18,7 @@ import ( "time" "github.com/Autumn-27/artex/db" + "github.com/Autumn-27/artex/internal/httpidentity" "github.com/Autumn-27/norma/permission" actool "github.com/Autumn-27/norma/tool" ) @@ -438,6 +439,7 @@ func (s *Server) runHTTPTool(ctx context.Context, execRaw json.RawMessage, param if err != nil { return actool.Errorf(maskSecrets(err.Error())), nil } + httpidentity.Apply(req) for k, v := range spec.Headers { req.Header.Set(k, renderTemplate(v, params, identity)) } diff --git a/server/egress_probe.go b/server/egress_probe.go index b5f6648..bfde1e9 100644 --- a/server/egress_probe.go +++ b/server/egress_probe.go @@ -23,6 +23,7 @@ import ( "syscall" "time" + "github.com/Autumn-27/artex/internal/httpidentity" "github.com/Autumn-27/artex/traffic" ) @@ -305,7 +306,7 @@ func probeOne(ctx context.Context, client *http.Client, target string) (string, if err != nil { return "", err } - req.Header.Set("User-Agent", "artex-egress-probe/1.0") + httpidentity.Apply(req) resp, err := client.Do(req) if err != nil { diff --git a/server/egress_probe_test.go b/server/egress_probe_test.go index f7aed78..af68e1e 100644 --- a/server/egress_probe_test.go +++ b/server/egress_probe_test.go @@ -27,6 +27,28 @@ func fakeIPServer(ip string) *httptest.Server { })) } +func TestProbeOneUserAgent(t *testing.T) { + for _, ua := range []string{"", "Captured-UA/1.0"} { + t.Run(ua, func(t *testing.T) { + t.Setenv("ARTEX_HTTP_USER_AGENT", ua) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.UserAgent(); got != ua { + t.Errorf("User-Agent = %q, want %q", got, ua) + } + if _, present := r.Header["User-Agent"]; ua == "" && present { + t.Error("empty User-Agent present on wire") + } + fmt.Fprint(w, `{"ip":"203.0.113.7"}`) + })) + defer server.Close() + ip, err := probeOne(context.Background(), server.Client(), server.URL) + if err != nil || ip != "203.0.113.7" { + t.Fatalf("ip = %q, err = %v", ip, err) + } + }) + } +} + // TestProbeEgress 覆盖 probeEgress 的输入/响应形态:成功、非法协议、死代理、坏响应、回落。 // 每条用例自带服务端与断言,失败时 t.Run 的名字直接指出是哪一条。 func TestProbeEgress(t *testing.T) { diff --git a/server/http_identity_test.go b/server/http_identity_test.go new file mode 100644 index 0000000..f42ec62 --- /dev/null +++ b/server/http_identity_test.go @@ -0,0 +1,51 @@ +package server + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" +) + +func TestTargetHTTPIdentityOverrides(t *testing.T) { + t.Setenv(repeaterAllowLocalEnv, "1") + for _, route := range []string{"custom", "repeater"} { + for _, tc := range []struct { + name, env, want string + headers map[string]string + }{ + {name: "default"}, + {name: "environment", env: "Captured-UA/1.0", want: "Captured-UA/1.0"}, + {name: "override", env: "Environment-UA/1.0", want: "Explicit-UA/1.0", headers: map[string]string{"user-agent": "Explicit-UA/1.0"}}, + {name: "explicit-empty", env: "Environment-UA/1.0", headers: map[string]string{"User-Agent": ""}}, + } { + t.Run(route+"/"+tc.name, func(t *testing.T) { + t.Setenv("ARTEX_HTTP_USER_AGENT", tc.env) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.UserAgent(); got != tc.want { + t.Errorf("User-Agent = %q, want %q", got, tc.want) + } + if _, present := r.Header["User-Agent"]; tc.want == "" && present { + t.Error("empty User-Agent present on wire") + } + w.Write([]byte("ok")) + })) + defer server.Close() + payload := map[string]any{"url": server.URL, "headers": tc.headers, "use_recording_proxy": false} + if route == "custom" { + raw, _ := json.Marshal(payload) + result, err := (&Server{}).runHTTPTool(context.Background(), raw, nil, nil) + if err != nil || result.IsError { + t.Fatalf("custom tool: %v %+v", err, result) + } + } else { + status, result, body := repeaterTestSend(t, &Server{}, payload) + if status != http.StatusOK || result.Status != http.StatusOK { + t.Fatalf("repeater: %d %s", status, body) + } + } + }) + } + } +} diff --git a/server/repeater.go b/server/repeater.go index 435f53f..3d682a6 100644 --- a/server/repeater.go +++ b/server/repeater.go @@ -16,6 +16,8 @@ import ( "strings" "syscall" "time" + + "github.com/Autumn-27/artex/internal/httpidentity" ) const ( @@ -141,6 +143,7 @@ func (s *Server) sendRepeater(w http.ResponseWriter, r *http.Request) { writeErr(w, http.StatusBadRequest, "构造请求失败") return } + httpidentity.Apply(req) for key, value := range in.Headers { name := strings.TrimSpace(key) if name == "" { diff --git a/skills/api-recon/scripts/harvest_static.py b/skills/api-recon/scripts/harvest_static.py index d1974c5..1dd40f7 100644 --- a/skills/api-recon/scripts/harvest_static.py +++ b/skills/api-recon/scripts/harvest_static.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 """ -harvest_static.py +harvest_static.py [--user-agent VALUE] 参考模板 — 非通用成品。执行前须按目标站点调整,常见改动: - extract_endpoints 正则(endpoint 方言) @@ -16,16 +16,25 @@ Static SPA bundle harvester. Framework-agnostic; tuned for webpack + Vite. Stdlib only. TLS verification is disabled (recon against self-signed/internal hosts). """ -import sys, os, re, ssl, json, urllib.request, urllib.parse +import sys, os, re, ssl, json, argparse, urllib.request, urllib.parse from concurrent.futures import ThreadPoolExecutor CTX = ssl.create_default_context(); CTX.check_hostname = False; CTX.verify_mode = ssl.CERT_NONE -UA = "Mozilla/5.0 (spa-api-recon)" +def user_agent(value=None): + """Explicit value > shared environment; empty omits User-Agent.""" + if value is None: + value = os.environ.get("ARTEX_HTTP_USER_AGENT", "") + value = value.strip(" ") + return value if all(32 <= ord(c) <= 126 for c in value) else "" -def fetch(url, binary=False): + +def fetch(url, binary=False, user_agent_value=None): try: - req = urllib.request.Request(url, headers={"User-Agent": UA}) - with urllib.request.urlopen(req, context=CTX, timeout=25) as r: + ua = user_agent(user_agent_value) + req = urllib.request.Request(url, headers={"User-Agent": ua} if ua else {}) + opener = urllib.request.build_opener(urllib.request.HTTPSHandler(context=CTX)) + opener.addheaders = [] # Do not reintroduce Python-urllib's default UA. + with opener.open(req, timeout=25) as r: data = r.read() return (data if binary else data.decode("utf-8", "ignore")), r.status except Exception as e: @@ -112,14 +121,17 @@ def extract_routes(js_text): # ---- main ------------------------------------------------------------------- def main(): - if len(sys.argv) < 3: - print("usage: harvest_static.py "); sys.exit(1) - base, outdir = sys.argv[1], sys.argv[2] + ap = argparse.ArgumentParser() + ap.add_argument("base"); ap.add_argument("outdir") + ap.add_argument("--user-agent", default=None, help="User-Agent override; default: ARTEX_HTTP_USER_AGENT or omitted") + args = ap.parse_args() + base, outdir = args.base, args.outdir + ua = user_agent(args.user_agent) if not base.startswith("http"): base = "https://" + base jsdir = os.path.join(outdir, "js"); os.makedirs(jsdir, exist_ok=True) print(f"[*] entry: {base}") - html, status = fetch(base) + html, status = fetch(base, user_agent_value=ua) open(os.path.join(outdir, "index.html"), "w").write(html) origin = origin_of(base) @@ -137,7 +149,7 @@ def main(): if not fn.endswith(".js"): return None lp = os.path.join(jsdir, fn) if url in have: return have[url] - data, st = fetch(url, binary=True) + data, st = fetch(url, binary=True, user_agent_value=ua) if st == 200 and data and not data[:15].lstrip().startswith(b"<"): open(lp, "wb").write(data); have[url] = lp; return lp return None diff --git a/skills/api-recon/scripts/spider_mpa.py b/skills/api-recon/scripts/spider_mpa.py index d91f2f7..928fe61 100644 --- a/skills/api-recon/scripts/spider_mpa.py +++ b/skills/api-recon/scripts/spider_mpa.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 """ -spider_mpa.py [--cookie "k=v; k2=v2"] [--max 200] [--depth 4] +spider_mpa.py [--cookie "k=v; k2=v2"] [--max 200] [--depth 4] [--user-agent VALUE] 参考模板 — 非通用成品。执行前须按目标调整 --exclude、cookie、depth/max 等同域策略。 @@ -19,15 +19,24 @@ from html.parser import HTMLParser from collections import deque CTX = ssl.create_default_context(); CTX.check_hostname = False; CTX.verify_mode = ssl.CERT_NONE -UA = "Mozilla/5.0 (spa-api-recon spider)" +def user_agent(value=None): + """Explicit value > shared environment; empty omits User-Agent.""" + if value is None: + value = os.environ.get("ARTEX_HTTP_USER_AGENT", "") + value = value.strip(" ") + return value if all(32 <= ord(c) <= 126 for c in value) else "" + SKIP_EXT = re.compile(r'\.(css|png|jpe?g|gif|svg|ico|woff2?|ttf|pdf|zip|mp4|webp|js)(\?|$)', re.I) API_HINT = re.compile(r'/(?:api|rest|service|ajax|action|do|rpc|graphql|v\d|admin|backend)\b', re.I) -def fetch(url, cookie): - h = {"User-Agent": UA} +def fetch(url, cookie, user_agent_value=None): + ua = user_agent(user_agent_value) + h = {"User-Agent": ua} if ua else {} if cookie: h["Cookie"] = cookie try: - with urllib.request.urlopen(urllib.request.Request(url, headers=h), context=CTX, timeout=20) as r: + opener = urllib.request.build_opener(urllib.request.HTTPSHandler(context=CTX)) + opener.addheaders = [] # Do not reintroduce Python-urllib's default UA. + with opener.open(urllib.request.Request(url, headers=h), timeout=20) as r: ct = r.headers.get("Content-Type", "") if "html" not in ct and "xml" not in ct: return "", ct return r.read().decode("utf-8", "ignore"), ct @@ -70,7 +79,9 @@ def main(): ap.add_argument("--cookie", default=""); ap.add_argument("--max", type=int, default=200) ap.add_argument("--depth", type=int, default=4); ap.add_argument("--delay", type=float, default=0.1) ap.add_argument("--include", default=""); ap.add_argument("--exclude", default="logout|signout|delete|remove|destroy") + ap.add_argument("--user-agent", default=None, help="User-Agent override; default: ARTEX_HTTP_USER_AGENT or omitted") args = ap.parse_args() + ua = user_agent(args.user_agent) base = args.base if args.base.startswith("http") else "https://" + args.base os.makedirs(args.outdir, exist_ok=True) origin = "{0.scheme}://{0.netloc}".format(urllib.parse.urlparse(base)) @@ -82,7 +93,7 @@ def main(): n = 0 while q and n < args.max: url, d = q.popleft(); n += 1 - html, ct = fetch(url, args.cookie) + html, ct = fetch(url, args.cookie, user_agent_value=ua) if args.delay: time.sleep(args.delay) if not html: continue p = Page() diff --git a/skills/api-recon/scripts/test_user_agent.py b/skills/api-recon/scripts/test_user_agent.py new file mode 100644 index 0000000..278c366 --- /dev/null +++ b/skills/api-recon/scripts/test_user_agent.py @@ -0,0 +1,90 @@ +"""Offline regressions: python3 -m unittest discover -s skills/api-recon/scripts.""" +import contextlib +import http.server +import importlib.util +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import threading +import unittest +from unittest.mock import patch + + +SCRIPTS = Path(__file__).resolve().parent +# Keep the offline fixtures independent of the runner's outbound/recording proxy. +DIRECT_ENV = {"no_proxy": "*", "NO_PROXY": "*"} + + +@contextlib.contextmanager +def capture_server(): + seen = [] + + class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + seen.append((self.path, self.headers.get("User-Agent"), self.headers.get("Cookie"))) + if self.path == "/start": + self.send_response(302) + self.send_header("Location", "/page") + self.end_headers() + return + self.send_response(200) + self.send_header("Content-Type", "text/html") + self.end_headers() + self.wfile.write(b'

Page

') + + def log_message(self, *args): + pass + + with http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) as server: + thread = threading.Thread(target=server.serve_forever) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_port}", seen + finally: + server.shutdown() + thread.join() + + +class UserAgentTests(unittest.TestCase): + def test_fetch_and_redirect(self): + for name in ("harvest_static", "spider_mpa"): + spec = importlib.util.spec_from_file_location(name, SCRIPTS / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + for value, expected in (("", None), (" Captured-UA/1.0 ", "Captured-UA/1.0"), + ("ok\r\nX-Injected: yes", None), ("客户端", None)): + with self.subTest(script=name, value=value), capture_server() as (base, seen): + with patch.dict(os.environ, {**DIRECT_ENV, "ARTEX_HTTP_USER_AGENT": value}): + if name == "harvest_static": + body, status = module.fetch(base + "/start") + self.assertEqual(status, 200) + else: + body, _ = module.fetch(base + "/start", "session=sample") + self.assertTrue(all(cookie == "session=sample" for _, _, cookie in seen)) + self.assertIn("Page", body) + self.assertEqual([path for path, _, _ in seen], ["/start", "/page"]) + self.assertTrue(all(ua == expected for _, ua, _ in seen), seen) + + def test_cli_precedence_and_outputs(self): + for name in ("harvest_static", "spider_mpa"): + for flags, expected in (([], "Environment-UA/1.0"), + (["--user-agent", "Override-UA/1.0"], "Override-UA/1.0"), + (["--user-agent", ""], None)): + with self.subTest(script=name, flags=flags), capture_server() as (base, seen): + with tempfile.TemporaryDirectory() as outdir: + env = dict(os.environ, **DIRECT_ENV, ARTEX_HTTP_USER_AGENT="Environment-UA/1.0") + command = [sys.executable, str(SCRIPTS / f"{name}.py"), base + "/start", outdir] + flags + if name == "spider_mpa": + command += ["--max", "1", "--delay", "0"] + result = subprocess.run(command, env=env, capture_output=True, text=True, timeout=15) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertGreaterEqual(len(seen), 2) + self.assertTrue(all(ua == expected for _, ua, _ in seen), seen) + artifact = "api_static.txt" if name == "harvest_static" else "forms.txt" + self.assertTrue((Path(outdir) / artifact).exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/traffic/traffic.go b/traffic/traffic.go index b3968f8..d9baf42 100644 --- a/traffic/traffic.go +++ b/traffic/traffic.go @@ -412,6 +412,20 @@ type sink struct { t *Traffic } +// Preserve an absent User-Agent when go-mitmproxy creates its outbound Go +// request. An explicit empty value suppresses net/http's automatic default. +func (s *sink) Requestheaders(f *mproxy.Flow) { + if f.Request == nil { + return + } + if f.Request.Header == nil { + f.Request.Header = make(http.Header) + } + if _, present := f.Request.Header["User-Agent"]; !present { + f.Request.Header.Set("User-Agent", "") + } +} + func (s *sink) Response(f *mproxy.Flow) { if f.Request == nil || f.Response == nil { return @@ -683,6 +697,9 @@ func headerLines(h map[string][]string) string { func requestHeaderLines(req *mproxy.Request) string { headers := req.Header.Clone() headers.Del("Host") + if headers.Get("User-Agent") == "" { + headers.Del("User-Agent") // Empty suppression value is not sent on the wire. + } host := req.URL.Host if raw := req.Raw(); raw != nil && strings.TrimSpace(raw.Host) != "" { host = raw.Host diff --git a/traffic/useragent_test.go b/traffic/useragent_test.go new file mode 100644 index 0000000..28408dc --- /dev/null +++ b/traffic/useragent_test.go @@ -0,0 +1,48 @@ +package traffic + +import ( + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + + mproxy "github.com/lqqyt2423/go-mitmproxy/proxy" +) + +func TestProxyPreservesUserAgent(t *testing.T) { + for _, ua := range []string{"", "Captured-UA/1.0"} { + t.Run(ua, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.UserAgent(); got != ua { + t.Errorf("forwarded User-Agent = %q, want %q", got, ua) + } + if _, present := r.Header["User-Agent"]; ua == "" && present { + t.Error("proxy reintroduced a User-Agent") + } + })) + defer server.Close() + u, _ := url.Parse(server.URL) + flow := &mproxy.Flow{Request: &mproxy.Request{URL: u, Header: make(http.Header)}} + if ua != "" { + flow.Request.Header.Set("User-Agent", ua) + } + (&sink{}).Requestheaders(flow) + // go-mitmproxy rebuilds the outbound request by adding every value. + req, _ := http.NewRequest(http.MethodGet, server.URL, nil) + for key, values := range flow.Request.Header { + for _, value := range values { + req.Header.Add(key, value) + } + } + resp, err := server.Client().Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if ua == "" && strings.Contains(requestHeaderLines(flow.Request), "User-Agent:") { + t.Error("recording contains a synthetic header that was not sent") + } + }) + } +} diff --git a/web/src/lib/mock/data.ts b/web/src/lib/mock/data.ts index 687d044..924c02a 100644 --- a/web/src/lib/mock/data.ts +++ b/web/src/lib/mock/data.ts @@ -2921,7 +2921,6 @@ export const trafficDetail: TrafficDetail = { req: `GET /v1/orders?id=1002 HTTP/1.1 Host: api.acme.com Authorization: Bearer eyJhbGciOiJIUzI1NiJ9... -User-Agent: ARTEX-worker/1.0 Accept: application/json `,