chore: ARTEX 初始化

ARTEX — AI 自主渗透测试系统(Go 后端 + Next.js 前端,单二进制内嵌)。

Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
This commit is contained in:
Autumn-27andClaude Opus 4.8 committed 2026-07-26 04:27:45 -04:00
commit 0ef8561411
287 files changed
+70178

No files matched your search

+68
View File
@@ -0,0 +1,68 @@
package traffic
import (
"errors"
"net/url"
"testing"
mproxy "github.com/lqqyt2423/go-mitmproxy/proxy"
)
func TestHostOnly(t *testing.T) {
cases := map[string]string{
"example.com:443": "example.com",
"example.com": "example.com",
"10.0.0.1:8080": "10.0.0.1",
}
for in, want := range cases {
if got := hostOnly(in); got != want {
t.Errorf("hostOnly(%q)=%q want %q", in, got, want)
}
}
}
func TestProxyCausedErr(t *testing.T) {
proxy := []string{
"protocol error: received DATA on a HEAD request",
"http2: server sent GOAWAY",
"malformed HTTP response",
}
target := []string{ // target-side failures must NOT trigger passthrough
"dial tcp 1.2.3.4:443: connect: connection refused",
"read: connection reset by peer",
"context deadline exceeded",
}
for _, s := range proxy {
if !proxyCausedErr(errors.New(s)) {
t.Errorf("expected proxy-caused: %q", s)
}
}
for _, s := range target {
if proxyCausedErr(errors.New(s)) {
t.Errorf("expected NOT proxy-caused: %q", s)
}
}
}
func TestMaybePassthroughFlagsHostOnce(t *testing.T) {
tr := &Traffic{}
f := &mproxy.Flow{Request: &mproxy.Request{URL: &url.URL{Host: "target.test:443"}}}
// Target-caused error → do NOT flag (keep MITM + recording).
tr.maybePassthrough(f, errors.New("connection refused"))
if _, ok := tr.pass.Load("target.test"); ok {
t.Fatal("target-caused error must not flag passthrough")
}
// Proxy-caused error → flag the host for transparent passthrough.
tr.maybePassthrough(f, errors.New("protocol error: received DATA on a HEAD request"))
if _, ok := tr.pass.Load("target.test"); !ok {
t.Fatal("proxy-caused error must flag passthrough")
}
// The shouldIntercept rule uses hostOnly(req.Host); the CONNECT host carries a
// port, so it must resolve to the same flagged key → intercept=false (tunnel).
if _, tunnel := tr.pass.Load(hostOnly("target.test:443")); !tunnel {
t.Fatal("flagged host must be recognized for the CONNECT form with port")
}
}
+525
View File
@@ -0,0 +1,525 @@
// Package traffic implements the request-recording subsystem (docs §10): an
// embedded go-mitmproxy proxy whose addon writes every target HTTP exchange into
// a human-browsable file tree, with a sidecar SQLite index for paged queries.
// Full capture, plaintext (no redaction), target HTTP only.
package traffic
import (
"context"
"crypto/sha256"
"database/sql"
"encoding/hex"
"encoding/json"
"fmt"
"log"
"net"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/Autumn-27/norma/permission"
actool "github.com/Autumn-27/norma/tool"
"github.com/Autumn-27/artex/db"
mproxy "github.com/lqqyt2423/go-mitmproxy/proxy"
_ "modernc.org/sqlite"
)
const indexSchema = `
CREATE TABLE IF NOT EXISTS exchanges (
id TEXT PRIMARY KEY,
ts INTEGER,
host TEXT,
method TEXT,
url_template TEXT,
url TEXT,
status INTEGER,
content_type TEXT,
req_len INTEGER,
resp_len INTEGER,
path TEXT
);
CREATE INDEX IF NOT EXISTS idx_ex_host ON exchanges(host);
CREATE INDEX IF NOT EXISTS idx_ex_tmpl ON exchanges(host, url_template);
CREATE INDEX IF NOT EXISTS idx_ex_ts ON exchanges(ts);
`
const maxInlineBody = 256 * 1024
// Traffic runs the recording proxy and owns the file tree + index.
type Traffic struct {
dir string
addr string
db *sql.DB
wmu sync.Mutex
seq atomic.Int64
proxy *mproxy.Proxy
// pass is the set of hosts whose MITM interception failed for a proxy/protocol
// reason; connections to them are tunneled transparently (fail-open) so the
// request still reaches the target — unrecorded — instead of being killed.
pass sync.Map // hostname(string) -> struct{}
}
// Open initializes the traffic tree, blob store and SQLite index under dir.
func Open(dir, addr string) (*Traffic, error) {
for _, d := range []string{dir, filepath.Join(dir, "_index"), filepath.Join(dir, "_blobs"), filepath.Join(dir, "_ca")} {
if err := os.MkdirAll(d, 0o755); err != nil {
return nil, err
}
}
db, err := sql.Open("sqlite", filepath.Join(dir, "_index", "index.sqlite"))
if err != nil {
return nil, err
}
for _, p := range []string{"PRAGMA journal_mode=WAL", "PRAGMA busy_timeout=5000"} {
if _, err := db.Exec(p); err != nil {
db.Close()
return nil, err
}
}
if _, err := db.Exec(indexSchema); err != nil {
db.Close()
return nil, err
}
t := &Traffic{dir: dir, addr: addr, db: db}
p, err := mproxy.NewProxy(&mproxy.Options{
Addr: addr,
SslInsecure: true,
CaRootPath: filepath.Join(dir, "_ca"),
})
if err != nil {
db.Close()
return nil, err
}
// Dial targets DIRECTLY. go-mitmproxy's default upstream uses
// http.ProxyFromEnvironment, so an HTTP_PROXY/HTTPS_PROXY in the environment
// (a VPN/system proxy) would make it forward target requests through that
// external proxy — which can't reach the target → 502. We capture target
// traffic directly, never via the host's proxy.
p.SetUpstreamProxy(func(*http.Request) (*url.URL, error) { return nil, nil })
// Fail-open: MITM every host by default, EXCEPT ones a prior request proved we
// can't intercept without breaking (see maybePassthrough). Those are tunneled
// transparently so the request still reaches the target instead of being killed.
p.SetShouldInterceptRule(func(req *http.Request) bool {
_, tunnel := t.pass.Load(hostOnly(req.Host))
return !tunnel
})
p.AddAddon(&sink{t: t})
t.proxy = p
return t, nil
}
// hostOnly strips an optional :port, so passthrough keys match whether the host
// arrives as "example.com:443" (CONNECT) or "example.com" (request URL).
func hostOnly(hostport string) string {
if h, _, err := net.SplitHostPort(hostport); err == nil {
return h
}
return hostport
}
// ProxyAddr returns the address workers should set as HTTP(S)_PROXY.
func (t *Traffic) ProxyAddr() string { return "http://127.0.0.1" + t.addr }
// CACertPath returns the PEM CA cert clients must trust to verify HTTPS through
// the MITM proxy (go-mitmproxy writes it here on first start).
func (t *Traffic) CACertPath() string {
return filepath.Join(t.dir, "_ca", "mitmproxy-ca-cert.pem")
}
// Start runs the proxy (blocking); run in a goroutine.
func (t *Traffic) Start() error { return t.proxy.Start() }
func (t *Traffic) Close() error { return t.db.Close() }
func (t *Traffic) DB() *sql.DB { return t.db }
// sink is the go-mitmproxy addon that records completed exchanges.
type sink struct {
mproxy.BaseAddon
t *Traffic
}
func (s *sink) Response(f *mproxy.Flow) {
if f.Request == nil || f.Response == nil {
return
}
s.t.record(f)
}
// RequestError fires when a request through an established MITM tunnel fails. If
// the failure looks proxy/protocol-caused (h2 quirks, HEAD-with-body, protocol
// errors) — not a plain target-unreachable error — we flag the host for
// transparent passthrough so future requests to it succeed instead of dying.
func (s *sink) RequestError(f *mproxy.Flow, err error) { s.t.maybePassthrough(f, err) }
// maybePassthrough marks a host to be tunneled transparently on the next
// connection, but only for errors the proxy itself caused — a target that is
// simply down/filtered would fail without us too, and must stay MITM'd+recorded.
func (t *Traffic) maybePassthrough(f *mproxy.Flow, err error) {
if err == nil || f == nil || f.Request == nil || f.Request.URL == nil || !proxyCausedErr(err) {
return
}
host := f.Request.URL.Hostname()
if host == "" {
return
}
if _, loaded := t.pass.LoadOrStore(host, struct{}{}); !loaded {
log.Printf("[traffic] 与 %s 的 MITM 出错,改为透传(该 host 后续直连目标、不再记录,但请求照常):%v", host, err)
}
}
// proxyCausedErr reports whether err indicates the interception layer (not the
// target) is at fault — HTTP/2 handling, HEAD-with-body, or a protocol violation.
func proxyCausedErr(err error) bool {
s := strings.ToLower(err.Error())
for _, p := range []string{"head request", "http2", "http/2", "protocol error", "protocol_error", "malformed"} {
if strings.Contains(s, p) {
return true
}
}
return false
}
func (t *Traffic) record(f *mproxy.Flow) {
host := f.Request.URL.Hostname()
method := f.Request.Method
tmpl := db.TemplatePath(f.Request.URL.EscapedPath())
n := t.seq.Add(1)
now := time.Now()
id := fmt.Sprintf("%d-%04d", now.Unix(), n%10000)
// dir mirrors the URL path as a browsable file tree:
// <host>/<seg1>/<seg2>/.../<METHOD>/<id>/{request.http,response.http,meta.json}
// e.g. http://h/api/v1/login → traffic/h/api/v1/login/GET/<id>/
parts := []string{t.dir, sanitize(host)}
for _, seg := range strings.Split(strings.Trim(tmpl, "/"), "/") {
if seg != "" {
parts = append(parts, sanitize(seg))
}
}
parts = append(parts, method, id)
exDir := filepath.Join(parts...)
if err := os.MkdirAll(exDir, 0o755); err != nil {
return
}
reqBody := t.bodyOrBlob(f.Request.Body)
respBody := t.bodyOrBlob(f.Response.Body)
ct := f.Response.Header.Get("Content-Type")
reqTxt := fmt.Sprintf("%s %s %s\n%s\n%s", method, f.Request.URL.RequestURI(), f.Request.Proto, headerLines(f.Request.Header), reqBody)
respTxt := fmt.Sprintf("HTTP %d\n%s\n%s", f.Response.StatusCode, headerLines(f.Response.Header), respBody)
_ = os.WriteFile(filepath.Join(exDir, "request.http"), []byte(reqTxt), 0o644)
_ = os.WriteFile(filepath.Join(exDir, "response.http"), []byte(respTxt), 0o644)
meta := fmt.Sprintf(`{"id":%q,"host":%q,"method":%q,"url":%q,"template":%q,"status":%d,"content_type":%q}`,
id, host, method, f.Request.URL.String(), tmpl, f.Response.StatusCode, ct)
_ = os.WriteFile(filepath.Join(exDir, "meta.json"), []byte(meta), 0o644)
rel, _ := filepath.Rel(t.dir, exDir)
t.wmu.Lock()
_, _ = t.db.Exec(`INSERT OR REPLACE INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
VALUES(?,?,?,?,?,?,?,?,?,?,?)`,
id, now.Unix(), host, method, tmpl, f.Request.URL.String(), f.Response.StatusCode, ct,
len(f.Request.Body), len(f.Response.Body), rel)
t.wmu.Unlock()
}
// bodyOrBlob returns the body inline if small, else stores it content-addressed
// and returns a human-readable pointer.
func (t *Traffic) bodyOrBlob(body []byte) string {
if len(body) == 0 {
return ""
}
if len(body) <= maxInlineBody {
return string(body)
}
sum := sha256.Sum256(body)
h := hex.EncodeToString(sum[:])
blobDir := filepath.Join(t.dir, "_blobs", "sha256", h[:2], h[2:4])
_ = os.MkdirAll(blobDir, 0o755)
blobPath := filepath.Join(blobDir, h+".bin")
if _, err := os.Stat(blobPath); os.IsNotExist(err) {
_ = os.WriteFile(blobPath, body, 0o644)
}
return fmt.Sprintf("@blob sha256:%s (len=%d)", h, len(body))
}
func headerLines(h map[string][]string) string {
var b strings.Builder
for k, vs := range h {
for _, v := range vs {
b.WriteString(k)
b.WriteString(": ")
b.WriteString(v)
b.WriteByte('\n')
}
}
return b.String()
}
func sanitize(s string) string {
r := strings.NewReplacer("/", "_", "\\", "_", ":", "_", "?", "_", "*", "_", "\"", "_", "<", "_", ">", "_", "|", "_")
out := r.Replace(s)
if out == "" || out == "_" {
return "root"
}
if len(out) > 120 {
out = out[:120]
}
return out
}
// ExchangeMeta is one row of the index (returned by Search).
type ExchangeMeta struct {
ID string `json:"id"`
TS int64 `json:"ts"`
Host string `json:"host"`
Method string `json:"method"`
URLTemplate string `json:"url_template"`
URL string `json:"url"`
Status int `json:"status"`
ContentType string `json:"content_type"`
RespLen int `json:"resp_len"`
Path string `json:"path"`
}
// Search returns paged exchange metadata (never bodies).
func (t *Traffic) Search(host string, page, size int) ([]ExchangeMeta, error) {
if size <= 0 || size > 500 {
size = 100
}
q := `SELECT id,ts,host,method,url_template,url,status,content_type,resp_len,path FROM exchanges`
args := []any{}
if host != "" {
q += ` WHERE host=?`
args = append(args, host)
}
q += ` ORDER BY ts DESC LIMIT ? OFFSET ?`
args = append(args, size, page*size)
rows, err := t.db.Query(q, args...)
if err != nil {
return nil, err
}
defer rows.Close()
var out []ExchangeMeta
for rows.Next() {
var m ExchangeMeta
if err := rows.Scan(&m.ID, &m.TS, &m.Host, &m.Method, &m.URLTemplate, &m.URL, &m.Status, &m.ContentType, &m.RespLen, &m.Path); err != nil {
return nil, err
}
out = append(out, m)
}
return out, rows.Err()
}
// Page returns one page of exchange metadata filtered by an optional host
// substring, exact method, and a free-text query q that fuzzy-matches across all
// indexed metadata columns (host/url/method/content-type/status), together with
// the total number of rows matching that filter (for the UI's pagination).
// Newest first. Bodies are never included.
func (t *Traffic) Page(host, method, q string, page, size int) (rows []ExchangeMeta, total int, err error) {
if size <= 0 || size > 500 {
size = 100
}
if page < 0 {
page = 0
}
where := ""
var args []any
add := func(cond string, vs ...any) {
if where == "" {
where = " WHERE "
} else {
where += " AND "
}
where += cond
args = append(args, vs...)
}
if h := strings.TrimSpace(host); h != "" {
add("host LIKE ?", "%"+h+"%")
}
if m := strings.TrimSpace(method); m != "" {
add("method=?", strings.ToUpper(m))
}
if s := strings.TrimSpace(q); s != "" {
like := "%" + s + "%"
// Fuzzy match across every indexed column (bodies aren't indexed).
add("(host LIKE ? OR url LIKE ? OR url_template LIKE ? OR method LIKE ? OR content_type LIKE ? OR CAST(status AS TEXT) LIKE ?)",
like, like, like, like, like, like)
}
if err = t.db.QueryRow(`SELECT COUNT(*) FROM exchanges`+where, args...).Scan(&total); err != nil {
return nil, 0, err
}
sel := `SELECT id,ts,host,method,url_template,url,status,content_type,resp_len,path FROM exchanges` +
where + ` ORDER BY ts DESC LIMIT ? OFFSET ?`
qargs := append(append([]any{}, args...), size, page*size)
rs, err := t.db.Query(sel, qargs...)
if err != nil {
return nil, 0, err
}
defer rs.Close()
for rs.Next() {
var m ExchangeMeta
if err := rs.Scan(&m.ID, &m.TS, &m.Host, &m.Method, &m.URLTemplate, &m.URL, &m.Status, &m.ContentType, &m.RespLen, &m.Path); err != nil {
return nil, 0, err
}
rows = append(rows, m)
}
return rows, total, rs.Err()
}
// Get returns the full request/response of one exchange (reads from the tree).
func (t *Traffic) Get(id string) (req, resp string, err error) {
var rel string
if err = t.db.QueryRow(`SELECT path FROM exchanges WHERE id=?`, id).Scan(&rel); err != nil {
return "", "", err
}
rb, _ := os.ReadFile(filepath.Join(t.dir, rel, "request.http"))
pb, _ := os.ReadFile(filepath.Join(t.dir, rel, "response.http"))
return string(rb), string(pb), nil
}
// Count returns total recorded exchanges.
func (t *Traffic) Count() (int, error) {
var n int
err := t.db.QueryRow(`SELECT COUNT(*) FROM exchanges`).Scan(&n)
return n, err
}
// query returns one page of exchange metadata filtered by host and/or a url
// substring. Default page size is intentionally small (3) to keep tool results
// lightweight and capped at 10; page is 0-based (page*limit offset).
func (t *Traffic) query(host, contains string, page, limit int) ([]ExchangeMeta, error) {
if limit <= 0 {
limit = 3
}
if limit > 10 {
limit = 10
}
if page < 0 {
page = 0
}
q := `SELECT id,ts,host,method,url_template,url,status,content_type,resp_len,path FROM exchanges WHERE 1=1`
args := []any{}
if host != "" {
q += ` AND host=?`
args = append(args, host)
}
if contains != "" {
q += ` AND (url LIKE ? OR url_template LIKE ?)`
args = append(args, "%"+contains+"%", "%"+contains+"%")
}
q += ` ORDER BY ts DESC LIMIT ? OFFSET ?`
args = append(args, limit, page*limit)
rows, err := t.db.Query(q, args...)
if err != nil {
return nil, err
}
defer rows.Close()
var out []ExchangeMeta
for rows.Next() {
var m ExchangeMeta
if err := rows.Scan(&m.ID, &m.TS, &m.Host, &m.Method, &m.URLTemplate, &m.URL, &m.Status, &m.ContentType, &m.RespLen, &m.Path); err != nil {
return nil, err
}
out = append(out, m)
}
return out, rows.Err()
}
// Tools exposes traffic lookup to work agents so they query already-captured
// traffic instead of re-curling the same resource (token + dedup win).
func (t *Traffic) Tools() []actool.CoreTool {
allow := func(context.Context, json.RawMessage, permission.Context) permission.Decision {
return permission.Allowed()
}
ro := func(json.RawMessage) bool { return true }
search := actool.Build(actool.Spec{
Name: "traffic_search",
Description: "查询记录代理已抓取的目标流量(必须指定 host,可再按 URL 子串过滤)。仅返回极轻量索引(id/method/url/status/resp_len),不含任何响应内容。默认只返回 3 条、每页最多 10 条;结果多时用 page 翻页(page=0 起);要看某条的请求/响应原文用 traffic_get(id)。回看已访问资源、找端点先用它,避免重复 curl 同一 URL。",
Schema: map[string]any{
"type": "object",
"properties": map[string]any{
"host": map[string]any{"type": "string", "description": "按主机过滤(必填,如 '107.172.96.177:8082')"},
"contains": map[string]any{"type": "string", "description": "URL 子串过滤(可选,如 'api' / 'login')"},
"limit": map[string]any{"type": "integer", "description": "每页条数,默认 3,最大 10"},
"page": map[string]any{"type": "integer", "description": "页码,从 0 开始,默认 0(按 ts 倒序分页)"},
},
"required": []any{"host"},
},
ReadOnly: ro,
Permissions: allow,
Run: func(_ context.Context, in json.RawMessage, _ *actool.ToolContext) (actool.Result, error) {
var a struct {
Host, Contains string
Limit int
Page int
}
_ = json.Unmarshal(in, &a)
if strings.TrimSpace(a.Host) == "" {
return actool.Errorf("host 为必填参数:请指定要查询的主机(如 '107.172.96.177:8082'),避免全库扫描。"), nil
}
rows, err := t.query(a.Host, a.Contains, a.Page, a.Limit)
if err != nil {
return actool.Errorf(err.Error()), nil
}
if len(rows) == 0 {
return actool.Text("无匹配流量。"), nil
}
// 精简为最小索引:仅保留定位所需字段 + 响应码/长度,不带任何响应内容。
type liteRow struct {
ID string `json:"id"`
Method string `json:"method"`
URL string `json:"url"`
Status int `json:"status"`
RespLen int `json:"resp_len"`
}
lite := make([]liteRow, 0, len(rows))
for _, r := range rows {
lite = append(lite, liteRow{ID: r.ID, Method: r.Method, URL: r.URL, Status: r.Status, RespLen: r.RespLen})
}
b, _ := json.Marshal(lite)
return actool.Text(string(b)), nil
},
})
get := actool.Build(actool.Spec{
Name: "traffic_get",
Description: "按 id 取一条已抓流量的请求/响应原文(过大会截断)。配合 traffic_search 用,避免重复 curl。",
Schema: map[string]any{
"type": "object",
"properties": map[string]any{"id": map[string]any{"type": "string", "description": "traffic_search 返回的 id"}},
"required": []any{"id"},
},
ReadOnly: ro,
Permissions: allow,
Run: func(_ context.Context, in json.RawMessage, _ *actool.ToolContext) (actool.Result, error) {
var a struct{ ID string }
_ = json.Unmarshal(in, &a)
req, resp, err := t.Get(a.ID)
if err != nil {
return actool.Errorf(err.Error()), nil
}
return actool.Text("=== REQUEST ===\n" + clip(req, 2500) + "\n\n=== RESPONSE ===\n" + clip(resp, 4000)), nil
},
})
return []actool.CoreTool{search, get}
}
// SeedToolMetas returns the traffic tools built on a ZERO receiver, for seeding the
// tools catalog (metadata only — Name/Description/InputSchema). The handlers close
// over the nil receiver but are never invoked on this instance, so it is safe.
func SeedToolMetas() []actool.CoreTool { return (&Traffic{}).Tools() }
func clip(s string, max int) string {
if len(s) <= max {
return s
}
return s[:max] + fmt.Sprintf("\n... [截断,共 %d 字节;完整在流量文件树] ...", len(s))
}