fix: make HTTP user agents configurable and remove tool identifiers

.env.example

@@ -13,3 +13,6 @@ ARTEX_LLM_PROVIDER=
 ARTEX_LLM_MODEL=
 ARTEX_LLM_BASE_URL=
 ARTEX_LLM_PROXY=
+
+# HTTP User-Agent；目标请求留空不发送，自更新留空使用通用客户端值
+ARTEX_HTTP_USER_AGENT=

HTTP_USER_AGENT.md

@@ -0,0 +1,54 @@
+# HTTP 请求标识排查
+
+本次检查覆盖仓库中的 Go HTTP 请求、Python / JavaScript 脚本、安装脚本、前端示例，以及实际使用的 norma v0.4.3 的 WebFetch / WebSearch 实现。排查范围包括 User-Agent 的大小写变体、请求头构造、代理转发、重定向及代理失败后的直连重试。
+
+## 已处理
+
+| 通道 | 排查结果与处理 |
+|---|---|
+| 内置 WebFetch | SDK 将 `norma/0.4` 写死在首发及重试请求。worker、planner、main、chat 通过统一 session 入口使用本地适配器；保留 SDK 的工具 schema、权限、静态资源限制、代理、CA 验证及 HTML / RECON 输出。 |
+| 资产自动补全 | 移除 `artex-enrich/1.0`，HTTP 探测使用统一策略。 |
+| 出口连通性探测 | 额外发现 `artex-egress-probe/1.0`，已移除并接入统一策略。 |
+| 静态采集 / 同源爬虫模板 | 移除 `spa-api-recon` / `spa-api-recon spider`，避免 urllib 自动填入自己的默认 UA；入口、下载、重定向及重试沿用同一配置。 |
+| HTTP 自定义工具 | 原先没有 ARTEX 专属 UA，但未设置时会由 Go 填充默认值。现采用统一策略，模板显式请求头优先。 |
+| 请求重放器 | 原先没有 ARTEX 专属 UA。现采用统一策略，用户显式请求头（包括空值）优先，保持原请求的配置能力。 |
+| 记录代理 | 转发缺失 UA 的请求时，Go 会补入默认客户端标识。现保留省略语义，录制中不展示内部用于抑制默认值的空头。保留客户端已有 UA。 |
+| GitHub 自更新 | 额外发现查询版本、下载校验和与发布包共用的两处 `artex-selfupdate`。均已移除；GitHub API 要求非空 UA，未配置时使用通用 `Go-http-client/1.1`，不含 ARTEX 名称。 |
+| 前端 mock | `ARTEX-worker/1.0` 仅为示例 HTTP 文本，不会发出请求；已删除该行，避免误导。 |
+
+## 配置
+
+`ARTEX_HTTP_USER_AGENT` 统一配置以上通道，默认空值使目标请求不发送此头；自更新采用表中非空兼容值。Go 服务重启后生效；Docker 在 `.env` 设置后重建容器。直接运行 Python 模板时可继承同名环境变量。
+
+Python 模板的 `--user-agent VALUE` 优先于环境变量，`--user-agent ''` 强制省略。自定义 HTTP 工具 / 重放器的显式 UA 同样优先于全局值。配置仅接受可打印 ASCII，去除首尾空格；控制字符和非 ASCII 值按空值处理，防止头注入。
+
+不随机轮换、不自动冒充浏览器。可以复用授权目标已有流量中的真实值。省略或修改 UA 不能保证客户端不可识别，第三方出口代理也可能再次添加或改写请求头。
+
+## 检查过的其他通道与边界
+
+| 通道 | 结果 |
+|---|---|
+| SDK 联网搜索 | norma 的 DDGS 后端含固定 Firefox 128 浏览器 UA，没有 ARTEX / norma 名称；仅发往搜索服务，属于依赖的兼容策略，未改动，也不受此环境变量控制。Brave / Tavily 及 DeepSeek 搜索未发现 ARTEX 专属 UA。 |
+| 浏览器自动化 / runtime_harvest.js | 沿用浏览器请求头；运行时转发复制捕获到的请求头，没有硬编码 ARTEX UA。 |
+| LLM 请求 / 模型列表 / JEV | 未发现硬编码 ARTEX UA；仍可能有 HTTP 库、SDK、服务协议和用户自定义标识，不套用目标流量配置。 |
+| MCP HTTP / SSE | 未发现 ARTEX UA；显式自定义头保留。初始化协议体仍有 `clientInfo.name = artex`，这是对用户配置的 MCP 服务的协议自报身份，本次不改动。 |
+| Webhook / 通知 | 未发现硬编码 ARTEX UA；保留用户自定义头及客户端默认行为。 |
+| 安装脚本 | 未发现 ARTEX UA；获取 Docker 安装脚本的 curl 仍使用其原生身份。外部命令及用户自建脚本不由本次策略接管。 |
+| phpfinger | 仓库未包含其源码，无法核验默认值；需要在独立工具中通过 `--user-agent` 配置。 |
+
+服务端响应中的 `X-Artex-Auth` 仍用于前端识别 Basic Auth 未认证状态，属于站点自身对访问者的响应协议；它不是向目标站点发送的请求头。本次不改变该协议，也不承诺消除服务端页面、API、TLS 或其他流量特征。
+
+## 回归验证
+
+- Go：检查真实 HTTP 线上头、配置值 / 空值、代理及失败直连重试、重定向、CA 信任、静态资源限制、HTML / RECON 解析、agent session 注册、自定义工具 / 重放器覆盖、自更新查询及下载和代理录制。
+- Python：本地 HTTP 服务验证默认 / 环境变量 / CLI 覆盖 / 显式空值、非法头值、重定向、Cookie 保留及产物生成；不访问真实目标。
+
+本地命令：
+
+```sh
+go test ./internal/httpidentity ./internal/webfetch ./enrich ./agent ./selfupdate ./traffic
+go test ./server -run 'TestProbe|TestRepeater|TestRunHTTPTool|TestHTTPTool|TestTargetHTTPIdentity' -count=1
+python3 -m unittest discover -s skills/api-recon/scripts -p 'test_*.py' -v
+```
+
+服务端完整套件另需隔离 PostgreSQL 测试库；未配置数据库时其中的数据库集成用例会失败。

README.md

@@ -314,6 +314,10 @@ CGO_ENABLED=0 go build -tags embedui -o artex ./cmd/artex
 **LLM**：`export ANTHROPIC_API_KEY=sk-...`（或 `OPENAI_API_KEY`），也可在 UI 的「LLM 配置」页填写。
 可选：`ARTEX_LLM_PROVIDER` / `ARTEX_LLM_MODEL` / `ARTEX_LLM_BASE_URL` / `ARTEX_LLM_PROXY`。
 
+**HTTP User-Agent**：内置 WebFetch（worker / planner / main / chat，包括代理失败后的直连重试）、资产 HTTP 自动补全、出口连通性探测、HTTP 自定义工具、请求重放器及可选的 `harvest_static.py` / `spider_mpa.py` 默认不发送 User-Agent，不再携带工具名称。通过 `ARTEX_HTTP_USER_AGENT` 设置固定值（例如复用授权目标已有流量中的值）；直接运行时导出环境变量并重启服务，Docker 部署在 `.env` 配置后重建容器。两个 Python 模板还支持 `--user-agent '已有流量中的值'`，优先于环境变量；`--user-agent ''` 强制留空。值会去除首尾空格，只接受可打印 ASCII，含控制字符或非 ASCII 的配置忽略并按空值处理。不随机轮换或自动伪装浏览器；省略此请求头不能保证无法识别客户端。自定义工具和重放器中显式提供的请求头优先于全局值。GitHub 自更新查询、校验和及包下载同样移除了工具名，未配置时使用通用 `Go-http-client/1.1` 以满足 GitHub 非空请求头要求。记录代理保留请求头省略语义。此设置不影响浏览器、SDK 联网搜索、LLM / MCP / Webhook 服务通信或外部独立工具；本仓库未包含 phpfinger，其 `--user-agent` 需在该工具中单独配置。
+
+完整排查范围、额外发现及未接管的服务通信见 [HTTP 请求标识排查](HTTP_USER_AGENT.md)。
+
 **并发**：每个任务的 work agent 数在「系统设置」里配置（默认 3）。
 
 **常用参数**：`./start.sh -addr :8787 -proxy :8788`（`-addr` 前端+API，`-proxy` 流量录制代理）。启动脚本会把参数原样透传给 `artex`。

agent/capture.go

@@ -26,7 +26,7 @@ import (
 // contiguous run is coalesced into a single record so the trace shows whole
 // messages, not dozens of fragments.
 func captureRun(ctx context.Context, opts agentcore.Options, input string, emit func(db.Activity)) (string, harness.TerminalReason, error) {
-	s := agentcore.NewSession(opts)
+	s := newAgentSession(opts)
 	defer s.Close() // release the session's background-task manager (temp dir + processes)
 	return captureRunSession(ctx, s, input, emit)
 }

agent/chat.go

@@ -173,7 +173,7 @@ func (c *ChatAgent) Chat(ctx context.Context, agentKey, sessionID, message strin
 	// 实验功能:开启后由 noa 接管上下文压缩(归档集中在 <workDir>/noa/<SessionID> 下,持久)。
 	enableNoa(&opts, c.noaEnabledFn, c.workDir, "chat-"+sessionID, noaWarn("chat-"+sessionID))
 	ctx = attachSideCapture(ctx, &opts)
-	s := agentcore.NewSession(opts)
+	s := newAgentSession(opts)
 	defer s.Close()
 	// reload prior conversation so the agent has context across turns (each Chat is
 	// a fresh session). First turn: no file yet → Resume loads nothing and proceeds.

agent/mainagent.go

@@ -184,7 +184,7 @@ func (m *MainAgent) Chat(ctx context.Context, taskID int64, mainSeg int, as *db.
 	}
 	enableNoa(&opts, m.noaEnabledFn, m.workDir, noaSession, noaWarn(noaSession))
 	ctx = attachSideCapture(ctx, &opts)
-	s := agentcore.NewSession(opts)
+	s := newAgentSession(opts)
 	defer s.Close()
 	// reload the prior conversation from the transcript so the agent has context
 	// across turns (each Chat is a fresh session; without this it can't see earlier

agent/webfetch.go

@@ -0,0 +1,24 @@
+package agent
+
+import (
+	"github.com/Autumn-27/artex/internal/webfetch"
+	"github.com/Autumn-27/norma/agentcore"
+	actool "github.com/Autumn-27/norma/tool"
+)
+
+// newAgentSession installs the configurable WebFetch before the SDK builds the
+// registry, so every agent role gets one tool and the same proxy/TLS settings.
+func newAgentSession(opts agentcore.Options) *agentcore.Session {
+	if opts.EnableWebFetch {
+		opts.EnableWebFetch = false
+		if opts.Tools == nil {
+			opts.Tools = actool.DefaultTools()
+		}
+		opts.Tools = append(opts.Tools, webfetch.New(actool.WebFetchConfig{
+			Proxy:       opts.WebFetchProxy,
+			CACert:      opts.WebFetchCACert,
+			InsecureTLS: opts.WebFetchInsecureTLS,
+		}))
+	}
+	return agentcore.NewSession(opts)
+}

agent/webfetch_test.go

@@ -0,0 +1,67 @@
+package agent
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/Autumn-27/norma/agentcore"
+	"github.com/Autumn-27/norma/llm"
+	"github.com/Autumn-27/norma/permission"
+)
+
+type fetchFixtureProvider struct {
+	autoTitleProvider
+	t     *testing.T
+	url   string
+	calls int
+}
+
+func (p *fetchFixtureProvider) Complete(_ context.Context, req llm.CompletionRequest) (llm.Message, string, llm.Usage, error) {
+	p.calls++
+	if p.calls == 1 {
+		counts := map[string]int{}
+		for _, tool := range req.Tools {
+			counts[tool.Name]++
+		}
+		if counts["WebFetch"] != 1 || counts["Read"] != 1 || counts["Bash"] != 1 {
+			p.t.Errorf("session tool registration = %v", counts)
+		}
+		input, _ := json.Marshal(map[string]string{"url": p.url})
+		return llm.Message{Role: llm.RoleAssistant, Content: []llm.ContentBlock{{Type: llm.BlockToolUse, ID: "fetch-fixture", Name: "WebFetch", Input: input}}}, "tool_use", llm.Usage{}, nil
+	}
+	return llm.Message{Role: llm.RoleAssistant, Content: []llm.ContentBlock{llm.TextBlock("finished")}}, "end_turn", llm.Usage{}, nil
+}
+
+func TestAgentSessionUsesConfiguredWebFetch(t *testing.T) {
+	for _, ua := range []string{"", "Captured-UA/1.0"} {
+		t.Run(ua, func(t *testing.T) {
+			t.Setenv("ARTEX_HTTP_USER_AGENT", ua)
+			calls := 0
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				calls++
+				if r.UserAgent() != ua {
+					t.Errorf("session sent User-Agent %q, want %q", r.UserAgent(), ua)
+				}
+				w.Write([]byte("fixture"))
+			}))
+			defer server.Close()
+			provider := &fetchFixtureProvider{t: t, url: server.URL}
+			session := newAgentSession(agentcore.Options{
+				Provider: provider, NonStreaming: true, EnableWebFetch: true,
+				PermissionMode: permission.ModeBypass, MaxTurns: 3,
+			})
+			defer session.Close()
+			for _, err := range session.Prompt(t.Context(), "fetch local fixture") {
+				if err != nil {
+					t.Fatal(err)
+				}
+			}
+			if calls != 1 {
+				t.Fatalf("WebFetch calls = %d, want 1", calls)
+			}
+		})
+	}
+}

agent/worker.go

@@ -488,7 +488,7 @@ func (w *Worker) execute(ctx context.Context, name string, taskID int64, as *db.
 	noaSession := WorkerSessionID(ts.ID(), intent.ID)
 	enableNoa(&opts, w.noaEnabledFn, w.workDir, noaSession, noaWarn(noaSession))
 	ctx = attachSideCapture(ctx, &opts)
-	s := agentcore.NewSession(opts)
+	s := newAgentSession(opts)
 	defer s.Close() // release the session's background-task manager (temp dir + processes)
 
 	// Resume prior conversation if this intent was paused/blocked/exhausted and is

docker-compose.yml

@@ -28,6 +28,7 @@ services:
       ARTEX_LLM_MODEL: ${ARTEX_LLM_MODEL:-}
       ARTEX_LLM_BASE_URL: ${ARTEX_LLM_BASE_URL:-}
       ARTEX_LLM_PROXY: ${ARTEX_LLM_PROXY:-}
+      ARTEX_HTTP_USER_AGENT: ${ARTEX_HTTP_USER_AGENT:-}
     ports:
       - "8787:8787"
       - "127.0.0.1:8788:8788"

enrich/enrich.go

@@ -20,6 +20,7 @@ import (
 	"time"
 
 	"github.com/Autumn-27/artex/db"
+	"github.com/Autumn-27/artex/internal/httpidentity"
 
 	"github.com/miekg/dns"
 	"github.com/projectdiscovery/dnsx/libs/dnsx"
@@ -230,7 +231,7 @@ func (e *Engine) doHTTP(id int64, rawURL string) {
 	if err != nil {
 		return
 	}
-	req.Header.Set("User-Agent", "artex-enrich/1.0")
+	httpidentity.Apply(req)
 	resp, err := e.client.Do(req)
 	if err != nil {
 		return

enrich/enrich_test.go

@@ -0,0 +1,38 @@
+package enrich
+
+import (
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+// Stop after receiving the probe so this request regression needs no database.
+func TestProbeUserAgent(t *testing.T) {
+	for _, ua := range []string{"", "Captured-UA/1.0"} {
+		t.Run(ua, func(t *testing.T) {
+			t.Setenv("ARTEX_HTTP_USER_AGENT", ua)
+			calls := 0
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				calls++
+				if got := r.UserAgent(); got != ua {
+					t.Errorf("User-Agent = %q, want %q", got, ua)
+				}
+				if _, present := r.Header["User-Agent"]; ua == "" && present {
+					t.Error("empty User-Agent present on wire")
+				}
+				http.Redirect(w, r, "/stop", http.StatusFound)
+			}))
+			defer server.Close()
+			client := buildClient(nil)
+			client.CheckRedirect = func(*http.Request, []*http.Request) error {
+				return errors.New("probe captured")
+			}
+			e := &Engine{client: client}
+			e.doHTTP(1, server.URL)
+			if calls != 1 {
+				t.Fatalf("probe calls = %d, want 1", calls)
+			}
+		})
+	}
+}

internal/httpidentity/useragent.go

@@ -0,0 +1,37 @@
+// Package httpidentity defines the explicit identity for target-facing HTTP requests.
+package httpidentity
+
+import (
+	"net/http"
+	"os"
+	"strings"
+)
+
+// UserAgent returns the configured value, or an empty string to omit the header.
+// Use printable ASCII for consistent behavior across Go and the Python templates.
+// Ignore malformed values rather than allowing header injection.
+func UserAgent() string {
+	value := strings.Trim(os.Getenv("ARTEX_HTTP_USER_AGENT"), " ")
+	for _, c := range value {
+		if c < 32 || c > 126 {
+			return ""
+		}
+	}
+	return value
+}
+
+// Apply must set even an empty value: deleting the header makes net/http send
+// its own Go-http-client default instead of omitting User-Agent on the wire.
+func Apply(req *http.Request) {
+	req.Header.Set("User-Agent", UserAgent())
+}
+
+// ApplyRequired keeps a nonempty, generic library identity for services that
+// require User-Agent (notably the GitHub API), without an ARTEX identifier.
+func ApplyRequired(req *http.Request) {
+	value := UserAgent()
+	if value == "" {
+		value = "Go-http-client/1.1"
+	}
+	req.Header.Set("User-Agent", value)
+}

internal/httpidentity/useragent_test.go

@@ -0,0 +1,43 @@
+package httpidentity
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func TestApplyOnWire(t *testing.T) {
+	for _, tc := range []struct{ name, value, want string }{
+		{"empty", "", ""},
+		{"configured", " Captured-UA/1.0 ", "Captured-UA/1.0"},
+		{"whitespace", "   ", ""},
+		{"injection", "ok\r\nX-Injected: yes", ""},
+		{"non-ascii", "客户端", ""},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Setenv("ARTEX_HTTP_USER_AGENT", tc.value)
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				if got := r.Header.Get("User-Agent"); got != tc.want {
+					t.Errorf("User-Agent = %q, want %q", got, tc.want)
+				}
+				if _, present := r.Header["User-Agent"]; tc.want == "" && present {
+					t.Error("empty User-Agent must be absent on the wire")
+				}
+				if r.Header.Get("X-Injected") != "" {
+					t.Error("header injection")
+				}
+			}))
+			defer server.Close()
+			req, err := http.NewRequest(http.MethodGet, server.URL, nil)
+			if err != nil {
+				t.Fatal(err)
+			}
+			Apply(req)
+			resp, err := server.Client().Do(req)
+			if err != nil {
+				t.Fatal(err)
+			}
+			resp.Body.Close()
+		})
+	}
+}

internal/webfetch/webfetch.go

@@ -0,0 +1,467 @@
+// Adapted from github.com/Autumn-27/norma/tool/webfetch.go at v0.4.3.
+// Keep the SDK tool metadata and fetch/render behavior; use ARTEX's explicit
+// HTTP identity for both the proxy attempt and its direct fallback. This local
+// adapter is needed until the SDK exposes request headers in actool.WebFetchConfig.
+package webfetch
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/Autumn-27/artex/internal/httpidentity"
+	actool "github.com/Autumn-27/norma/tool"
+	"golang.org/x/net/html"
+)
+
+// New keeps the SDK's schema, permission checks and tool metadata.
+func New(cfg actool.WebFetchConfig) actool.CoreTool {
+	base := actool.NewWebFetch(cfg)
+	return actool.Build(actool.Spec{
+		Name: base.Name(), Description: base.Description(), Prompt: base.Prompt(),
+		Schema: base.InputSchema(), Permissions: base.CheckPermissions,
+		ReadOnly: base.IsReadOnly, Concurrent: base.IsConcurrencySafe,
+		Run: webFetchRunner(cfg),
+	})
+}
+
+const maxFetchBytes = 2 << 20 // 2 MiB
+
+// newFetchClient returns an HTTP client honoring the proxy / TLS config.
+func newFetchClient(cfg actool.WebFetchConfig) *http.Client {
+	tr := &http.Transport{}
+	if cfg.Proxy != "" {
+		if pu, err := url.Parse(cfg.Proxy); err == nil {
+			tr.Proxy = http.ProxyURL(pu)
+		}
+	}
+	switch {
+	case cfg.InsecureTLS:
+		tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+	case cfg.CACert != "":
+		// Trust the proxy's CA in addition to the system roots, so certs it
+		// re-signs verify while every other connection stays fully checked.
+		if pem, err := os.ReadFile(cfg.CACert); err == nil {
+			pool, _ := x509.SystemCertPool()
+			if pool == nil {
+				pool = x509.NewCertPool()
+			}
+			if pool.AppendCertsFromPEM(pem) {
+				tr.TLSClientConfig = &tls.Config{RootCAs: pool}
+			}
+		}
+	}
+	return &http.Client{Transport: tr}
+}
+
+// proxyLikelyErr reports whether a fetch error is the kind a recording/MITM proxy
+// introduces — a closed tunnel (EOF / connection reset), a failed CONNECT, or a
+// TLS/HTTP2 mismatch — so a direct (no-proxy) retry may succeed. Name-resolution
+// failures and caller cancellation/timeout are target- or caller-side problems a
+// direct retry won't fix, so those return false (no retry).
+func proxyLikelyErr(err error) bool {
+	if err == nil {
+		return false
+	}
+	s := strings.ToLower(err.Error())
+	for _, n := range []string{"no such host", "context canceled", "context deadline", "timeout"} {
+		if strings.Contains(s, n) {
+			return false
+		}
+	}
+	for _, p := range []string{
+		"eof", "connection reset", "connection refused", "proxyconnect",
+		"broken pipe", "forcibly closed", "malformed", "http2", "http/2",
+		"tls:", "protocol error",
+	} {
+		if strings.Contains(s, p) {
+			return true
+		}
+	}
+	return false
+}
+
+// staticAssetExts are file extensions WebFetch refuses: linked resources (scripts,
+// styles, source maps, images, fonts, media, wasm) that are not page content.
+var staticAssetExts = map[string]bool{
+	".js": true, ".mjs": true, ".cjs": true, ".css": true, ".map": true,
+	".png": true, ".jpg": true, ".jpeg": true, ".gif": true, ".svg": true,
+	".webp": true, ".avif": true, ".ico": true, ".bmp": true,
+	".woff": true, ".woff2": true, ".ttf": true, ".otf": true, ".eot": true,
+	".wasm": true, ".mp4": true, ".webm": true, ".mp3": true, ".wav": true, ".ogg": true,
+}
+
+// staticAssetExt returns the offending extension (with dot) if rawURL points at a
+// static asset, or "" otherwise. The query string and fragment are ignored, so
+// "/app.js?v=3" is still caught.
+func staticAssetExt(rawURL string) string {
+	path := rawURL
+	if u, err := url.Parse(rawURL); err == nil {
+		path = u.Path
+	}
+	if i := strings.LastIndexByte(path, '/'); i >= 0 {
+		path = path[i+1:] // last segment only
+	}
+	dot := strings.LastIndexByte(path, '.')
+	if dot < 0 {
+		return ""
+	}
+	if ext := strings.ToLower(path[dot:]); staticAssetExts[ext] {
+		return ext
+	}
+	return ""
+}
+
+func webFetchRunner(cfg actool.WebFetchConfig) func(context.Context, json.RawMessage, *actool.ToolContext) (actool.Result, error) {
+	userAgent := httpidentity.UserAgent()
+	client := newFetchClient(cfg)
+	// direct is the no-proxy fallback used when a proxied fetch fails for a
+	// proxy-caused reason (closed tunnel / failed CONNECT / TLS or HTTP2 mismatch):
+	// it bypasses the configured proxy so a recording/MITM hiccup doesn't fail an
+	// otherwise-reachable URL. nil when no proxy is configured (nothing to bypass).
+	var direct *http.Client
+	if cfg.Proxy != "" {
+		direct = newFetchClient(actool.WebFetchConfig{InsecureTLS: cfg.InsecureTLS})
+	}
+	return func(ctx context.Context, input json.RawMessage, _ *actool.ToolContext) (actool.Result, error) {
+		var in struct {
+			URL     string `json:"url"`
+			Extract bool   `json:"extract"`
+		}
+		if err := json.Unmarshal(input, &in); err != nil {
+			return actool.Result{}, err
+		}
+		if !strings.HasPrefix(in.URL, "http://") && !strings.HasPrefix(in.URL, "https://") {
+			return actool.Errorf("Error: url must start with http:// or https://"), nil
+		}
+		if ext := staticAssetExt(in.URL); ext != "" {
+			return actool.Errorf("Error: WebFetch does not fetch static assets (" + ext + " file). It reads HTML pages, documentation, and API responses — not linked resources like scripts, styles, images, or fonts. Fetch the page that references this asset instead."), nil
+		}
+		cctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+		defer cancel()
+		req, err := http.NewRequestWithContext(cctx, http.MethodGet, in.URL, nil)
+		if err != nil {
+			return actool.Errorf("Error: " + err.Error()), nil
+		}
+		req.Header.Set("User-Agent", userAgent)
+		resp, err := client.Do(req)
+		// One direct retry when the proxied attempt failed for a proxy-caused reason.
+		// A non-proxy error (bad host, caller timeout/cancel) is returned as-is — a
+		// direct retry wouldn't fix it. The direct response is NOT captured by the proxy.
+		if err != nil && direct != nil && proxyLikelyErr(err) {
+			if r2, e2 := http.NewRequestWithContext(cctx, http.MethodGet, in.URL, nil); e2 == nil {
+				r2.Header.Set("User-Agent", userAgent)
+				if resp2, err2 := direct.Do(r2); err2 == nil {
+					resp, err = resp2, nil
+				} else {
+					err = err2 // surface the direct failure (more representative of the target)
+				}
+			}
+		}
+		if err != nil {
+			return actool.Errorf("Error fetching URL: " + err.Error()), nil
+		}
+		defer resp.Body.Close()
+		body, _ := io.ReadAll(io.LimitReader(resp.Body, maxFetchBytes))
+
+		header := fmt.Sprintf("[%d %s] %s\n", resp.StatusCode, http.StatusText(resp.StatusCode), in.URL)
+		// Cheap fingerprint signal that HTML→Markdown would otherwise hide.
+		if sv := resp.Header.Get("Server"); sv != "" {
+			header += "Server: " + sv + "\n"
+		}
+		if xp := resp.Header.Get("X-Powered-By"); xp != "" {
+			header += "X-Powered-By: " + xp + "\n"
+		}
+
+		var out string
+		if strings.Contains(strings.ToLower(resp.Header.Get("content-type")), "html") {
+			doc, perr := html.Parse(strings.NewReader(string(body)))
+			if perr != nil { // html.Parse is lenient; this is effectively unreachable
+				out = string(body)
+			} else {
+				out = htmlToMarkdown(doc)
+				if in.Extract {
+					if recon := extractRecon(doc); recon != "" {
+						out += "\n\n" + recon
+					}
+				}
+			}
+		} else {
+			out = string(body)
+		}
+		return actool.Text(truncate(header+strings.TrimSpace(out), 50000)), nil
+	}
+}
+
+var (
+	reWS         = regexp.MustCompile(`[ \t]+`)
+	reLineTrim   = regexp.MustCompile(` *\n *`) // strip spaces hugging a newline
+	reBlankLines = regexp.MustCompile(`\n{2,}`) // collapse blank lines → one newline
+	// reEndpoint matches quoted absolute URLs and root-relative paths in inline JS.
+	reEndpoint = regexp.MustCompile("[\"'`](https?://[^\"'`\\s]+|/[A-Za-z0-9._~$&'()*+,;=:@%/?#\\[\\]-]{2,})[\"'`]")
+)
+
+// skipMarkdown are elements whose subtree carries no readable prose.
+var skipMarkdown = map[string]bool{"script": true, "style": true, "noscript": true, "svg": true, "template": true}
+
+// htmlToMarkdown renders the visible content of a parsed document as Markdown.
+// Text nodes are already entity-decoded by the parser, so no unescaping is needed.
+func htmlToMarkdown(doc *html.Node) string {
+	var b strings.Builder
+	renderNode(&b, doc)
+	s := reWS.ReplaceAllString(b.String(), " ")
+	s = reLineTrim.ReplaceAllString(s, "\n")   // drop indentation / trailing spaces per line
+	s = reBlankLines.ReplaceAllString(s, "\n") // no blank lines — compact, token-lean output
+	return strings.TrimSpace(s)
+}
+
+func renderNode(b *strings.Builder, n *html.Node) {
+	switch n.Type {
+	case html.TextNode:
+		b.WriteString(strings.ReplaceAll(n.Data, "\n", " "))
+		return
+	case html.ElementNode:
+		if skipMarkdown[n.Data] {
+			return
+		}
+	default:
+		if n.Type != html.DocumentNode {
+			return
+		}
+	}
+	tag := ""
+	if n.Type == html.ElementNode {
+		tag = n.Data
+	}
+	switch tag {
+	case "h1", "h2", "h3", "h4", "h5", "h6":
+		b.WriteString("\n\n" + strings.Repeat("#", int(tag[1]-'0')) + " ")
+		renderChildren(b, n)
+		b.WriteString("\n\n")
+	case "title":
+		b.WriteString("\n\n# ")
+		renderChildren(b, n)
+		b.WriteString("\n\n")
+	case "p", "div", "section", "article", "header", "footer", "main", "ul", "ol", "table", "tr", "blockquote":
+		b.WriteString("\n\n")
+		renderChildren(b, n)
+		b.WriteString("\n\n")
+	case "br":
+		b.WriteString("\n")
+	case "hr":
+		b.WriteString("\n\n---\n\n")
+	case "li":
+		b.WriteString("\n- ")
+		renderChildren(b, n)
+	case "a":
+		var inner strings.Builder
+		renderChildren(&inner, n)
+		text := strings.TrimSpace(inner.String())
+		if href := attr(n, "href"); href != "" && text != "" {
+			fmt.Fprintf(b, "[%s](%s)", text, href)
+		} else {
+			b.WriteString(text)
+		}
+	case "img":
+		if src := attr(n, "src"); src != "" {
+			fmt.Fprintf(b, "![%s](%s)", attr(n, "alt"), src)
+		}
+	case "strong", "b":
+		b.WriteString("**")
+		renderChildren(b, n)
+		b.WriteString("**")
+	case "em", "i":
+		b.WriteString("*")
+		renderChildren(b, n)
+		b.WriteString("*")
+	case "code":
+		b.WriteString("`")
+		renderChildren(b, n)
+		b.WriteString("`")
+	case "pre":
+		var inner strings.Builder
+		textOnly(&inner, n)
+		b.WriteString("\n\n```\n" + strings.TrimSpace(inner.String()) + "\n```\n\n")
+	default:
+		renderChildren(b, n)
+	}
+}
+
+func renderChildren(b *strings.Builder, n *html.Node) {
+	for c := n.FirstChild; c != nil; c = c.NextSibling {
+		renderNode(b, c)
+	}
+}
+
+// extractRecon walks the document for security-relevant signal that Markdown
+// rendering drops: comments, script sources, inline endpoints, forms, meta, links.
+func extractRecon(doc *html.Node) string {
+	var comments, scriptSrcs, inlineEndpoints, forms, metas, links []string
+	seenLink := map[string]bool{}
+
+	var walk func(n *html.Node)
+	walk = func(n *html.Node) {
+		switch n.Type {
+		case html.CommentNode:
+			if c := oneLine(n.Data, 300); c != "" {
+				comments = append(comments, c)
+			}
+		case html.ElementNode:
+			switch n.Data {
+			case "script":
+				if src := attr(n, "src"); src != "" {
+					scriptSrcs = append(scriptSrcs, src)
+				} else {
+					var body strings.Builder
+					textOnly(&body, n)
+					for _, m := range reEndpoint.FindAllStringSubmatch(body.String(), 40) {
+						inlineEndpoints = append(inlineEndpoints, m[1])
+					}
+				}
+			case "form":
+				forms = append(forms, describeForm(n))
+			case "meta":
+				name := attr(n, "name")
+				if name == "" {
+					name = attr(n, "property")
+				}
+				if content := attr(n, "content"); name != "" && content != "" {
+					metas = append(metas, name+"="+oneLine(content, 120))
+				}
+			case "a", "link":
+				if h := attr(n, "href"); h != "" && !seenLink[h] {
+					seenLink[h] = true
+					links = append(links, h)
+				}
+			}
+		}
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			walk(c)
+		}
+	}
+	walk(doc)
+
+	var b strings.Builder
+	b.WriteString("=== RECON ===")
+	reconSection(&b, "COMMENTS", comments)
+	reconSection(&b, "SCRIPTS (src)", dedup(scriptSrcs))
+	reconSection(&b, "INLINE ENDPOINTS", dedup(inlineEndpoints))
+	reconSection(&b, "FORMS", forms)
+	reconSection(&b, "META", metas)
+	reconSection(&b, "LINKS", capSlice(links, 100))
+	if b.Len() == len("=== RECON ===") {
+		return "" // nothing worth reporting
+	}
+	return b.String()
+}
+
+// describeForm summarizes a <form> as "METHOD action → name(type[=value]) …".
+// Hidden inputs keep their value (CSRF tokens, parameter defaults matter in recon).
+func describeForm(form *html.Node) string {
+	method := strings.ToUpper(attr(form, "method"))
+	if method == "" {
+		method = "GET"
+	}
+	var inputs []string
+	var walk func(n *html.Node)
+	walk = func(n *html.Node) {
+		if n.Type == html.ElementNode {
+			switch n.Data {
+			case "input", "textarea", "select":
+				typ := attr(n, "type")
+				if typ == "" {
+					typ = n.Data
+				}
+				desc := attr(n, "name") + "(" + typ
+				if typ == "hidden" {
+					if v := attr(n, "value"); v != "" {
+						desc += "=" + oneLine(v, 60)
+					}
+				}
+				inputs = append(inputs, desc+")")
+			}
+		}
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			walk(c)
+		}
+	}
+	walk(form)
+	return fmt.Sprintf("%s %s → %s", method, attr(form, "action"), strings.Join(inputs, " "))
+}
+
+// --- small helpers ---
+
+func attr(n *html.Node, key string) string {
+	for _, a := range n.Attr {
+		if a.Key == key {
+			return a.Val
+		}
+	}
+	return ""
+}
+
+func textOnly(b *strings.Builder, n *html.Node) {
+	if n.Type == html.TextNode {
+		b.WriteString(n.Data)
+		return
+	}
+	for c := n.FirstChild; c != nil; c = c.NextSibling {
+		textOnly(b, c)
+	}
+}
+
+func oneLine(s string, max int) string {
+	s = strings.Join(strings.Fields(s), " ")
+	if len(s) > max {
+		return s[:max] + "…"
+	}
+	return s
+}
+
+func reconSection(b *strings.Builder, title string, items []string) {
+	if len(items) == 0 {
+		return
+	}
+	b.WriteString("\n\n## " + title + "\n")
+	for _, it := range items {
+		b.WriteString("- " + it + "\n")
+	}
+}
+
+func dedup(in []string) []string {
+	seen := map[string]bool{}
+	out := in[:0:0]
+	for _, s := range in {
+		if s == "" || seen[s] {
+			continue
+		}
+		seen[s] = true
+		out = append(out, s)
+	}
+	return out
+}
+
+func capSlice(in []string, n int) []string {
+	if len(in) <= n {
+		return in
+	}
+	return append(in[:n:n], fmt.Sprintf("… (%d more)", len(in)-n))
+}
+
+// truncate retains the SDK's head/tail output limit.
+func truncate(s string, max int) string {
+	if max <= 0 || len(s) <= max {
+		return s
+	}
+	half := max / 2
+	return s[:half] + fmt.Sprintf("\n\n... [%d characters truncated] ...\n\n", len(s)-max) + s[len(s)-half:]
+}

internal/webfetch/webfetch_test.go

@@ -0,0 +1,146 @@
+package webfetch
+
+import (
+	"context"
+	"encoding/json"
+	"encoding/pem"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strings"
+	"sync/atomic"
+	"testing"
+
+	actool "github.com/Autumn-27/norma/tool"
+)
+
+func callFetch(t *testing.T, tool actool.CoreTool, target string, extract bool) actool.Result {
+	t.Helper()
+	input, err := json.Marshal(map[string]any{"url": target, "extract": extract})
+	if err != nil {
+		t.Fatal(err)
+	}
+	result, err := tool.Call(context.Background(), input, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return result
+}
+
+func TestIdentityAcrossRoutes(t *testing.T) {
+	for _, ua := range []string{"", "Captured-UA/1.0"} {
+		for _, route := range []string{"direct", "proxy", "fallback"} {
+			t.Run(route+"/"+ua, func(t *testing.T) {
+				t.Setenv("ARTEX_HTTP_USER_AGENT", ua)
+				var targetCalls, proxyCalls atomic.Int32
+				checkUA := func(r *http.Request) {
+					if got := r.Header.Get("User-Agent"); got != ua {
+						t.Errorf("User-Agent = %q, want %q", got, ua)
+					}
+					if _, present := r.Header["User-Agent"]; ua == "" && present {
+						t.Error("empty User-Agent present on wire")
+					}
+				}
+				target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+					targetCalls.Add(1)
+					checkUA(r)
+					if r.URL.Path == "/start" {
+						http.Redirect(w, r, "/page", http.StatusFound)
+						return
+					}
+					w.Header().Set("Content-Type", "text/html")
+					fmt.Fprint(w, `<h1>Page</h1><!--keep--><script src="/app.js"></script><form action="/submit"><input name="csrf" type="hidden" value="token"></form>`)
+				}))
+				defer target.Close()
+				proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+					proxyCalls.Add(1)
+					checkUA(r)
+					if route == "fallback" {
+						conn, _, err := w.(http.Hijacker).Hijack()
+						if err != nil {
+							t.Error(err)
+							return
+						}
+						conn.Close() // Deterministic EOF, causing the SDK-compatible retry.
+						return
+					}
+					req := r.Clone(r.Context())
+					req.RequestURI = ""
+					// A forwarding proxy must preserve absence, like ARTEX's sink.
+					if _, present := req.Header["User-Agent"]; !present {
+						req.Header.Set("User-Agent", "")
+					}
+					resp, err := target.Client().Do(req)
+					if err != nil {
+						t.Error(err)
+						w.WriteHeader(http.StatusBadGateway)
+						return
+					}
+					defer resp.Body.Close()
+					for key, values := range resp.Header {
+						w.Header()[key] = values
+					}
+					w.WriteHeader(resp.StatusCode)
+					io.Copy(w, resp.Body)
+				}))
+				defer proxy.Close()
+				cfg := actool.WebFetchConfig{}
+				if route != "direct" {
+					cfg.Proxy = proxy.URL
+				}
+				result := callFetch(t, New(cfg), target.URL+"/start", true)
+				for _, want := range []string{"[200 OK]", "# Page", "=== RECON ===", "keep", "/app.js", "csrf(hidden=token)"} {
+					if result.IsError || !strings.Contains(result.Flatten(), want) {
+						t.Fatalf("missing %q in result: %+v", want, result)
+					}
+				}
+				if targetCalls.Load() != 2 {
+					t.Errorf("target calls = %d, want 2", targetCalls.Load())
+				}
+				if route != "direct" && proxyCalls.Load() == 0 {
+					t.Error("configured proxy was bypassed")
+				}
+			})
+		}
+	}
+}
+
+func TestTLSAndStaticAssetGuard(t *testing.T) {
+	t.Setenv("ARTEX_HTTP_USER_AGENT", "Captured-UA/1.0")
+	var calls atomic.Int32
+	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		calls.Add(1)
+		if r.UserAgent() != "Captured-UA/1.0" {
+			t.Errorf("User-Agent = %q", r.UserAgent())
+		}
+		fmt.Fprint(w, "verified")
+	}))
+	defer server.Close()
+	certPath := filepath.Join(t.TempDir(), "ca.pem")
+	cert := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw})
+	if err := os.WriteFile(certPath, cert, 0600); err != nil {
+		t.Fatal(err)
+	}
+	tool := New(actool.WebFetchConfig{CACert: certPath})
+	if result := callFetch(t, tool, server.URL, false); result.IsError || !strings.Contains(result.Flatten(), "verified") {
+		t.Fatalf("CA trust lost: %+v", result)
+	}
+	if result := callFetch(t, tool, server.URL+"/app.js?v=1", false); !result.IsError {
+		t.Fatalf("static asset guard lost: %+v", result)
+	}
+	if calls.Load() != 1 {
+		t.Errorf("static asset triggered network request: calls = %d", calls.Load())
+	}
+}
+
+func TestSDKMetadataPreserved(t *testing.T) {
+	base := actool.NewWebFetch(actool.WebFetchConfig{})
+	local := New(actool.WebFetchConfig{})
+	if local.Name() != base.Name() || local.Description() != base.Description() || !reflect.DeepEqual(local.InputSchema(), base.InputSchema()) {
+		t.Fatal("SDK tool contract changed")
+	}
+}

selfupdate/github.go

@@ -9,6 +9,8 @@ import (
 	"strconv"
 	"strings"
 	"time"
+
+	"github.com/Autumn-27/artex/internal/httpidentity"
 )
 
 // Repo 是发布源。写死而不是做成配置项：更新源可配等于给任何能改配置的人一条
@@ -125,7 +127,7 @@ func FetchLatest(ctx context.Context, c *http.Client) (*Release, error) {
 		return nil, err
 	}
 	req.Header.Set("Accept", "application/vnd.github+json")
-	req.Header.Set("User-Agent", "artex-selfupdate")
+	httpidentity.ApplyRequired(req)
 
 	resp, err := c.Do(req)
 	if err != nil {

selfupdate/github_test.go

@@ -56,3 +56,36 @@ func TestFetchLatestMissingRelease(t *testing.T) {
 		t.Fatalf("unexpected missing-release error: %v", err)
 	}
 }
+
+func TestUpdateUserAgent(t *testing.T) {
+	for _, tc := range []struct{ env, want string }{
+		{"", "Go-http-client/1.1"},
+		{"Captured-UA/1.0", "Captured-UA/1.0"},
+	} {
+		t.Run(tc.env, func(t *testing.T) {
+			t.Setenv("ARTEX_HTTP_USER_AGENT", tc.env)
+			calls := 0
+			client := &http.Client{Transport: releaseTransport(func(req *http.Request) (*http.Response, error) {
+				calls++
+				if got := req.UserAgent(); got != tc.want {
+					t.Errorf("User-Agent = %q, want %q", got, tc.want)
+				}
+				body := `{"tag_name":"v0.3.17"}`
+				return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(body))}, nil
+			})}
+			if _, err := FetchLatest(context.Background(), client); err != nil {
+				t.Fatal(err)
+			}
+			for _, name := range []string{"SHA256SUMS", "artex.zip"} {
+				body, err := get(context.Background(), client, "https://github.com/RuoJi6/ARTEX/releases/download/v0.3.17/"+name)
+				if err != nil {
+					t.Fatal(err)
+				}
+				body.Close()
+			}
+			if calls != 3 {
+				t.Fatalf("calls = %d, want 3", calls)
+			}
+		})
+	}
+}

selfupdate/stage.go

@@ -13,6 +13,8 @@ import (
 	"runtime"
 	"strings"
 	"time"
+
+	"github.com/Autumn-27/artex/internal/httpidentity"
 )
 
 // sumsAsset 是 release.yml 生成的校验和清单，覆盖 Release 里全部 zip。
@@ -229,7 +231,7 @@ func get(ctx context.Context, c *http.Client, rawURL string) (io.ReadCloser, err
 	if err := checkURL(req.URL); err != nil {
 		return nil, err
 	}
-	req.Header.Set("User-Agent", "artex-selfupdate")
+	httpidentity.ApplyRequired(req)
 	resp, err := c.Do(req)
 	if err != nil {
 		return nil, err

server/customtool.go

@@ -18,6 +18,7 @@ import (
 	"time"
 
 	"github.com/Autumn-27/artex/db"
+	"github.com/Autumn-27/artex/internal/httpidentity"
 	"github.com/Autumn-27/norma/permission"
 	actool "github.com/Autumn-27/norma/tool"
 )
@@ -438,6 +439,7 @@ func (s *Server) runHTTPTool(ctx context.Context, execRaw json.RawMessage, param
 	if err != nil {
 		return actool.Errorf(maskSecrets(err.Error())), nil
 	}
+	httpidentity.Apply(req)
 	for k, v := range spec.Headers {
 		req.Header.Set(k, renderTemplate(v, params, identity))
 	}

server/egress_probe.go

@@ -23,6 +23,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/Autumn-27/artex/internal/httpidentity"
 	"github.com/Autumn-27/artex/traffic"
 )
 
@@ -305,7 +306,7 @@ func probeOne(ctx context.Context, client *http.Client, target string) (string,
 	if err != nil {
 		return "", err
 	}
-	req.Header.Set("User-Agent", "artex-egress-probe/1.0")
+	httpidentity.Apply(req)
 
 	resp, err := client.Do(req)
 	if err != nil {

server/egress_probe_test.go

@@ -27,6 +27,28 @@ func fakeIPServer(ip string) *httptest.Server {
 	}))
 }
 
+func TestProbeOneUserAgent(t *testing.T) {
+	for _, ua := range []string{"", "Captured-UA/1.0"} {
+		t.Run(ua, func(t *testing.T) {
+			t.Setenv("ARTEX_HTTP_USER_AGENT", ua)
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				if got := r.UserAgent(); got != ua {
+					t.Errorf("User-Agent = %q, want %q", got, ua)
+				}
+				if _, present := r.Header["User-Agent"]; ua == "" && present {
+					t.Error("empty User-Agent present on wire")
+				}
+				fmt.Fprint(w, `{"ip":"203.0.113.7"}`)
+			}))
+			defer server.Close()
+			ip, err := probeOne(context.Background(), server.Client(), server.URL)
+			if err != nil || ip != "203.0.113.7" {
+				t.Fatalf("ip = %q, err = %v", ip, err)
+			}
+		})
+	}
+}
+
 // TestProbeEgress 覆盖 probeEgress 的输入/响应形态：成功、非法协议、死代理、坏响应、回落。
 // 每条用例自带服务端与断言，失败时 t.Run 的名字直接指出是哪一条。
 func TestProbeEgress(t *testing.T) {

server/http_identity_test.go

@@ -0,0 +1,51 @@
+package server
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func TestTargetHTTPIdentityOverrides(t *testing.T) {
+	t.Setenv(repeaterAllowLocalEnv, "1")
+	for _, route := range []string{"custom", "repeater"} {
+		for _, tc := range []struct {
+			name, env, want string
+			headers         map[string]string
+		}{
+			{name: "default"},
+			{name: "environment", env: "Captured-UA/1.0", want: "Captured-UA/1.0"},
+			{name: "override", env: "Environment-UA/1.0", want: "Explicit-UA/1.0", headers: map[string]string{"user-agent": "Explicit-UA/1.0"}},
+			{name: "explicit-empty", env: "Environment-UA/1.0", headers: map[string]string{"User-Agent": ""}},
+		} {
+			t.Run(route+"/"+tc.name, func(t *testing.T) {
+				t.Setenv("ARTEX_HTTP_USER_AGENT", tc.env)
+				server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+					if got := r.UserAgent(); got != tc.want {
+						t.Errorf("User-Agent = %q, want %q", got, tc.want)
+					}
+					if _, present := r.Header["User-Agent"]; tc.want == "" && present {
+						t.Error("empty User-Agent present on wire")
+					}
+					w.Write([]byte("ok"))
+				}))
+				defer server.Close()
+				payload := map[string]any{"url": server.URL, "headers": tc.headers, "use_recording_proxy": false}
+				if route == "custom" {
+					raw, _ := json.Marshal(payload)
+					result, err := (&Server{}).runHTTPTool(context.Background(), raw, nil, nil)
+					if err != nil || result.IsError {
+						t.Fatalf("custom tool: %v %+v", err, result)
+					}
+				} else {
+					status, result, body := repeaterTestSend(t, &Server{}, payload)
+					if status != http.StatusOK || result.Status != http.StatusOK {
+						t.Fatalf("repeater: %d %s", status, body)
+					}
+				}
+			})
+		}
+	}
+}

server/repeater.go

@@ -16,6 +16,8 @@ import (
 	"strings"
 	"syscall"
 	"time"
+
+	"github.com/Autumn-27/artex/internal/httpidentity"
 )
 
 const (
@@ -141,6 +143,7 @@ func (s *Server) sendRepeater(w http.ResponseWriter, r *http.Request) {
 		writeErr(w, http.StatusBadRequest, "构造请求失败")
 		return
 	}
+	httpidentity.Apply(req)
 	for key, value := range in.Headers {
 		name := strings.TrimSpace(key)
 		if name == "" {

skills/api-recon/scripts/harvest_static.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 """
-harvest_static.py <BASE_URL> <OUTDIR>
+harvest_static.py <BASE_URL> <OUTDIR> [--user-agent VALUE]
 
 参考模板 — 非通用成品。执行前须按目标站点调整，常见改动：
   - extract_endpoints 正则（endpoint 方言）
@@ -16,16 +16,25 @@ Static SPA bundle harvester. Framework-agnostic; tuned for webpack + Vite.
 
 Stdlib only. TLS verification is disabled (recon against self-signed/internal hosts).
 """
-import sys, os, re, ssl, json, urllib.request, urllib.parse
+import sys, os, re, ssl, json, argparse, urllib.request, urllib.parse
 from concurrent.futures import ThreadPoolExecutor
 
 CTX = ssl.create_default_context(); CTX.check_hostname = False; CTX.verify_mode = ssl.CERT_NONE
-UA = "Mozilla/5.0 (spa-api-recon)"
+def user_agent(value=None):
+    """Explicit value > shared environment; empty omits User-Agent."""
+    if value is None:
+        value = os.environ.get("ARTEX_HTTP_USER_AGENT", "")
+    value = value.strip(" ")
+    return value if all(32 <= ord(c) <= 126 for c in value) else ""
 
-def fetch(url, binary=False):
+
+def fetch(url, binary=False, user_agent_value=None):
     try:
-        req = urllib.request.Request(url, headers={"User-Agent": UA})
-        with urllib.request.urlopen(req, context=CTX, timeout=25) as r:
+        ua = user_agent(user_agent_value)
+        req = urllib.request.Request(url, headers={"User-Agent": ua} if ua else {})
+        opener = urllib.request.build_opener(urllib.request.HTTPSHandler(context=CTX))
+        opener.addheaders = []  # Do not reintroduce Python-urllib's default UA.
+        with opener.open(req, timeout=25) as r:
             data = r.read()
             return (data if binary else data.decode("utf-8", "ignore")), r.status
     except Exception as e:
@@ -112,14 +121,17 @@ def extract_routes(js_text):
 
 # ---- main -------------------------------------------------------------------
 def main():
-    if len(sys.argv) < 3:
-        print("usage: harvest_static.py <BASE_URL> <OUTDIR>"); sys.exit(1)
-    base, outdir = sys.argv[1], sys.argv[2]
+    ap = argparse.ArgumentParser()
+    ap.add_argument("base"); ap.add_argument("outdir")
+    ap.add_argument("--user-agent", default=None, help="User-Agent override; default: ARTEX_HTTP_USER_AGENT or omitted")
+    args = ap.parse_args()
+    base, outdir = args.base, args.outdir
+    ua = user_agent(args.user_agent)
     if not base.startswith("http"): base = "https://" + base
     jsdir = os.path.join(outdir, "js"); os.makedirs(jsdir, exist_ok=True)
 
     print(f"[*] entry: {base}")
-    html, status = fetch(base)
+    html, status = fetch(base, user_agent_value=ua)
     open(os.path.join(outdir, "index.html"), "w").write(html)
     origin = origin_of(base)
 
@@ -137,7 +149,7 @@ def main():
         if not fn.endswith(".js"): return None
         lp = os.path.join(jsdir, fn)
         if url in have: return have[url]
-        data, st = fetch(url, binary=True)
+        data, st = fetch(url, binary=True, user_agent_value=ua)
         if st == 200 and data and not data[:15].lstrip().startswith(b"<"):
             open(lp, "wb").write(data); have[url] = lp; return lp
         return None

skills/api-recon/scripts/spider_mpa.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 """
-spider_mpa.py <BASE_URL> <OUTDIR> [--cookie "k=v; k2=v2"] [--max 200] [--depth 4]
+spider_mpa.py <BASE_URL> <OUTDIR> [--cookie "k=v; k2=v2"] [--max 200] [--depth 4] [--user-agent VALUE]
 
 参考模板 — 非通用成品。执行前须按目标调整 --exclude、cookie、depth/max 等同域策略。
 
@@ -19,15 +19,24 @@ from html.parser import HTMLParser
 from collections import deque
 
 CTX = ssl.create_default_context(); CTX.check_hostname = False; CTX.verify_mode = ssl.CERT_NONE
-UA = "Mozilla/5.0 (spa-api-recon spider)"
+def user_agent(value=None):
+    """Explicit value > shared environment; empty omits User-Agent."""
+    if value is None:
+        value = os.environ.get("ARTEX_HTTP_USER_AGENT", "")
+    value = value.strip(" ")
+    return value if all(32 <= ord(c) <= 126 for c in value) else ""
+
 SKIP_EXT = re.compile(r'\.(css|png|jpe?g|gif|svg|ico|woff2?|ttf|pdf|zip|mp4|webp|js)(\?|$)', re.I)
 API_HINT = re.compile(r'/(?:api|rest|service|ajax|action|do|rpc|graphql|v\d|admin|backend)\b', re.I)
 
-def fetch(url, cookie):
-    h = {"User-Agent": UA}
+def fetch(url, cookie, user_agent_value=None):
+    ua = user_agent(user_agent_value)
+    h = {"User-Agent": ua} if ua else {}
     if cookie: h["Cookie"] = cookie
     try:
-        with urllib.request.urlopen(urllib.request.Request(url, headers=h), context=CTX, timeout=20) as r:
+        opener = urllib.request.build_opener(urllib.request.HTTPSHandler(context=CTX))
+        opener.addheaders = []  # Do not reintroduce Python-urllib's default UA.
+        with opener.open(urllib.request.Request(url, headers=h), timeout=20) as r:
             ct = r.headers.get("Content-Type", "")
             if "html" not in ct and "xml" not in ct: return "", ct
             return r.read().decode("utf-8", "ignore"), ct
@@ -70,7 +79,9 @@ def main():
     ap.add_argument("--cookie", default=""); ap.add_argument("--max", type=int, default=200)
     ap.add_argument("--depth", type=int, default=4); ap.add_argument("--delay", type=float, default=0.1)
     ap.add_argument("--include", default=""); ap.add_argument("--exclude", default="logout|signout|delete|remove|destroy")
+    ap.add_argument("--user-agent", default=None, help="User-Agent override; default: ARTEX_HTTP_USER_AGENT or omitted")
     args = ap.parse_args()
+    ua = user_agent(args.user_agent)
     base = args.base if args.base.startswith("http") else "https://" + args.base
     os.makedirs(args.outdir, exist_ok=True)
     origin = "{0.scheme}://{0.netloc}".format(urllib.parse.urlparse(base))
@@ -82,7 +93,7 @@ def main():
     n = 0
     while q and n < args.max:
         url, d = q.popleft(); n += 1
-        html, ct = fetch(url, args.cookie)
+        html, ct = fetch(url, args.cookie, user_agent_value=ua)
         if args.delay: time.sleep(args.delay)
         if not html: continue
         p = Page()

skills/api-recon/scripts/test_user_agent.py

@@ -0,0 +1,90 @@
+"""Offline regressions: python3 -m unittest discover -s skills/api-recon/scripts."""
+import contextlib
+import http.server
+import importlib.util
+import os
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import threading
+import unittest
+from unittest.mock import patch
+
+
+SCRIPTS = Path(__file__).resolve().parent
+# Keep the offline fixtures independent of the runner's outbound/recording proxy.
+DIRECT_ENV = {"no_proxy": "*", "NO_PROXY": "*"}
+
+
+@contextlib.contextmanager
+def capture_server():
+    seen = []
+
+    class Handler(http.server.BaseHTTPRequestHandler):
+        def do_GET(self):
+            seen.append((self.path, self.headers.get("User-Agent"), self.headers.get("Cookie")))
+            if self.path == "/start":
+                self.send_response(302)
+                self.send_header("Location", "/page")
+                self.end_headers()
+                return
+            self.send_response(200)
+            self.send_header("Content-Type", "text/html")
+            self.end_headers()
+            self.wfile.write(b'<h1>Page</h1><script src="/app.js"></script><form action="/submit"></form>')
+
+        def log_message(self, *args):
+            pass
+
+    with http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) as server:
+        thread = threading.Thread(target=server.serve_forever)
+        thread.start()
+        try:
+            yield f"http://127.0.0.1:{server.server_port}", seen
+        finally:
+            server.shutdown()
+            thread.join()
+
+
+class UserAgentTests(unittest.TestCase):
+    def test_fetch_and_redirect(self):
+        for name in ("harvest_static", "spider_mpa"):
+            spec = importlib.util.spec_from_file_location(name, SCRIPTS / f"{name}.py")
+            module = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(module)
+            for value, expected in (("", None), (" Captured-UA/1.0 ", "Captured-UA/1.0"),
+                                    ("ok\r\nX-Injected: yes", None), ("客户端", None)):
+                with self.subTest(script=name, value=value), capture_server() as (base, seen):
+                    with patch.dict(os.environ, {**DIRECT_ENV, "ARTEX_HTTP_USER_AGENT": value}):
+                        if name == "harvest_static":
+                            body, status = module.fetch(base + "/start")
+                            self.assertEqual(status, 200)
+                        else:
+                            body, _ = module.fetch(base + "/start", "session=sample")
+                            self.assertTrue(all(cookie == "session=sample" for _, _, cookie in seen))
+                    self.assertIn("Page", body)
+                    self.assertEqual([path for path, _, _ in seen], ["/start", "/page"])
+                    self.assertTrue(all(ua == expected for _, ua, _ in seen), seen)
+
+    def test_cli_precedence_and_outputs(self):
+        for name in ("harvest_static", "spider_mpa"):
+            for flags, expected in (([], "Environment-UA/1.0"),
+                                    (["--user-agent", "Override-UA/1.0"], "Override-UA/1.0"),
+                                    (["--user-agent", ""], None)):
+                with self.subTest(script=name, flags=flags), capture_server() as (base, seen):
+                    with tempfile.TemporaryDirectory() as outdir:
+                        env = dict(os.environ, **DIRECT_ENV, ARTEX_HTTP_USER_AGENT="Environment-UA/1.0")
+                        command = [sys.executable, str(SCRIPTS / f"{name}.py"), base + "/start", outdir] + flags
+                        if name == "spider_mpa":
+                            command += ["--max", "1", "--delay", "0"]
+                        result = subprocess.run(command, env=env, capture_output=True, text=True, timeout=15)
+                        self.assertEqual(result.returncode, 0, result.stderr)
+                        self.assertGreaterEqual(len(seen), 2)
+                        self.assertTrue(all(ua == expected for _, ua, _ in seen), seen)
+                        artifact = "api_static.txt" if name == "harvest_static" else "forms.txt"
+                        self.assertTrue((Path(outdir) / artifact).exists())
+
+
+if __name__ == "__main__":
+    unittest.main()

traffic/traffic.go

@@ -412,6 +412,20 @@ type sink struct {
 	t *Traffic
 }
 
+// Preserve an absent User-Agent when go-mitmproxy creates its outbound Go
+// request. An explicit empty value suppresses net/http's automatic default.
+func (s *sink) Requestheaders(f *mproxy.Flow) {
+	if f.Request == nil {
+		return
+	}
+	if f.Request.Header == nil {
+		f.Request.Header = make(http.Header)
+	}
+	if _, present := f.Request.Header["User-Agent"]; !present {
+		f.Request.Header.Set("User-Agent", "")
+	}
+}
+
 func (s *sink) Response(f *mproxy.Flow) {
 	if f.Request == nil || f.Response == nil {
 		return
@@ -683,6 +697,9 @@ func headerLines(h map[string][]string) string {
 func requestHeaderLines(req *mproxy.Request) string {
 	headers := req.Header.Clone()
 	headers.Del("Host")
+	if headers.Get("User-Agent") == "" {
+		headers.Del("User-Agent") // Empty suppression value is not sent on the wire.
+	}
 	host := req.URL.Host
 	if raw := req.Raw(); raw != nil && strings.TrimSpace(raw.Host) != "" {
 		host = raw.Host

traffic/useragent_test.go

@@ -0,0 +1,48 @@
+package traffic
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strings"
+	"testing"
+
+	mproxy "github.com/lqqyt2423/go-mitmproxy/proxy"
+)
+
+func TestProxyPreservesUserAgent(t *testing.T) {
+	for _, ua := range []string{"", "Captured-UA/1.0"} {
+		t.Run(ua, func(t *testing.T) {
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				if got := r.UserAgent(); got != ua {
+					t.Errorf("forwarded User-Agent = %q, want %q", got, ua)
+				}
+				if _, present := r.Header["User-Agent"]; ua == "" && present {
+					t.Error("proxy reintroduced a User-Agent")
+				}
+			}))
+			defer server.Close()
+			u, _ := url.Parse(server.URL)
+			flow := &mproxy.Flow{Request: &mproxy.Request{URL: u, Header: make(http.Header)}}
+			if ua != "" {
+				flow.Request.Header.Set("User-Agent", ua)
+			}
+			(&sink{}).Requestheaders(flow)
+			// go-mitmproxy rebuilds the outbound request by adding every value.
+			req, _ := http.NewRequest(http.MethodGet, server.URL, nil)
+			for key, values := range flow.Request.Header {
+				for _, value := range values {
+					req.Header.Add(key, value)
+				}
+			}
+			resp, err := server.Client().Do(req)
+			if err != nil {
+				t.Fatal(err)
+			}
+			resp.Body.Close()
+			if ua == "" && strings.Contains(requestHeaderLines(flow.Request), "User-Agent:") {
+				t.Error("recording contains a synthetic header that was not sent")
+			}
+		})
+	}
+}

web/src/lib/mock/data.ts

@@ -2921,7 +2921,6 @@ export const trafficDetail: TrafficDetail = {
   req: `GET /v1/orders?id=1002 HTTP/1.1
 Host: api.acme.com
 Authorization: Bearer eyJhbGciOiJIUzI1NiJ9...
-User-Agent: ARTEX-worker/1.0
 Accept: application/json
 
 `,