admin/ARTEX-chixuweihu
1
0 Fork 0
mirror of https://github.com/RuoJi6/ARTEX.git synced 2026-11-15 06:33:50 +08:00
Code Issues Packages Projects Releases Wiki Activity
Files
6997a4661921395072290becc50380db8dacb04e
ARTEX-chixuweihu/skills/api-recon/scripts/spider_mpa.py
T

164 lines
8.9 KiB
Python
Raw Blame History

#!/usr/bin/env python3
"""
spider_mpa.py <BASE_URL> <OUTDIR> [--cookie "k=v; k2=v2"] [--max 200] [--depth 4] [--user-agent VALUE]
参考模板 — 非通用成品。执行前须按目标调整 --exclude、cookie、depth/max 等同域策略。
Fallback for NON-SPA targets (traditional server-rendered MPAs: Django/Rails/PHP/
JSP, classic admin panels). When there is no JS endpoint bundle, the API surface
lives in HTML <form action>, <a href>, and inline-JS ajax urls. This BFS-crawls
the same origin and extracts:
- forms.txt : METHOD action [param1, param2, ...] (the real "endpoints")
- links.txt : every same-origin URL reached
- api_inline.txt : url-ish strings found in inline <script> / onclick (fetch/ajax)
Stdlib only. Same-origin, bounded, polite. Provide --cookie for an authed crawl.
"""
import sys, os, re, ssl, secrets, argparse, urllib.request, urllib.parse, time
from html.parser import HTMLParser
from collections import deque
CTX = ssl.create_default_context(); CTX.check_hostname = False; CTX.verify_mode = ssl.CERT_NONE
USER_AGENTS = (
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/157.0.0.0 Safari/537.36',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/157.0.0.0 Safari/537.36',
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/157.0.0.0 Safari/537.36',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/156.0.0.0 Safari/537.36',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/156.0.0.0 Safari/537.36',
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/156.0.0.0 Safari/537.36',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/155.0.0.0 Safari/537.36',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/155.0.0.0 Safari/537.36',
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/155.0.0.0 Safari/537.36',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/154.0.0.0 Safari/537.36',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/154.0.0.0 Safari/537.36',
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/154.0.0.0 Safari/537.36',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/157.0.0.0 Safari/537.36 Edg/157.0.0.0',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/157.0.0.0 Safari/537.36 Edg/157.0.0.0',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/156.0.0.0 Safari/537.36 Edg/156.0.0.0',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/156.0.0.0 Safari/537.36 Edg/156.0.0.0',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:159.0) Gecko/20100101 Firefox/159.0',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:159.0) Gecko/20100101 Firefox/159.0',
'Mozilla/5.0 (X11; Linux x86_64; rv:159.0) Gecko/20100101 Firefox/159.0',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/27.1 Safari/605.1.15',
)
def user_agent(value=None):
"""Explicit value > environment > random sample; explicit empty omits UA."""
explicit = value is not None
if value is None:
value = os.environ.get("ARTEX_HTTP_USER_AGENT", "")
value = value.strip(" ")
if not all(32 <= ord(c) <= 126 for c in value):
return secrets.choice(USER_AGENTS)
if value or explicit:
return value
return secrets.choice(USER_AGENTS)
SKIP_EXT = re.compile(r'\.(css|png|jpe?g|gif|svg|ico|woff2?|ttf|pdf|zip|mp4|webp|js)(\?|$)', re.I)
API_HINT = re.compile(r'/(?:api|rest|service|ajax|action|do|rpc|graphql|v\d|admin|backend)\b', re.I)
def fetch(url, cookie, user_agent_value=None):
ua = user_agent(user_agent_value)
h = {"User-Agent": ua} if ua else {}
if cookie: h["Cookie"] = cookie
try:
opener = urllib.request.build_opener(urllib.request.HTTPSHandler(context=CTX))
opener.addheaders = [] # Do not reintroduce Python-urllib's default UA.
with opener.open(urllib.request.Request(url, headers=h), timeout=20) as r:
ct = r.headers.get("Content-Type", "")
if "html" not in ct and "xml" not in ct: return "", ct
return r.read().decode("utf-8", "ignore"), ct
except Exception:
return "", ""
class Page(HTMLParser):
def __init__(self):
super().__init__()
self.links, self.scripts, self.inline = [], [], []
self.forms, self._cur = [], None
self._in_script = False
def handle_starttag(self, tag, attrs):
a = dict(attrs)
if tag == "a" and a.get("href"): self.links.append(a["href"])
elif tag == "script":
self._in_script = True
if a.get("src"): self.scripts.append(a["src"])
elif tag == "form":
self._cur = {"action": a.get("action", ""), "method": (a.get("method") or "GET").upper(), "params": []}
elif tag in ("input", "select", "textarea", "button") and self._cur is not None:
n = a.get("name")
if n: self._cur["params"].append(n)
# ajax-ish handlers
for v in a.values():
for m in re.findall(r'''["'](/[^"']{2,120})["']''', v or ""):
if API_HINT.search(m): self.inline.append(m)
def handle_endtag(self, tag):
if tag == "script": self._in_script = False
elif tag == "form" and self._cur is not None:
self.forms.append(self._cur); self._cur = None
def handle_data(self, data):
if self._in_script and data:
for m in re.findall(r'''["'`](/[A-Za-z0-9_\-./{}$:?=&]{2,120})["'`]''', data):
if API_HINT.search(m): self.inline.append(m)
def main():
ap = argparse.ArgumentParser()
ap.add_argument("base"); ap.add_argument("outdir")
ap.add_argument("--cookie", default=""); ap.add_argument("--max", type=int, default=200)
ap.add_argument("--depth", type=int, default=4); ap.add_argument("--delay", type=float, default=0.1)
ap.add_argument("--include", default=""); ap.add_argument("--exclude", default="logout|signout|delete|remove|destroy")
ap.add_argument("--user-agent", default=None, help="User-Agent override; default: ARTEX_HTTP_USER_AGENT or random browser sample")
args = ap.parse_args()
ua = user_agent(args.user_agent)
base = args.base if args.base.startswith("http") else "https://" + args.base
os.makedirs(args.outdir, exist_ok=True)
origin = "{0.scheme}://{0.netloc}".format(urllib.parse.urlparse(base))
inc = re.compile(args.include) if args.include else None
exc = re.compile(args.exclude, re.I) if args.exclude else None
seen, forms, inline, links = set(), {}, set(), set()
q = deque([(base, 0)]); seen.add(base.split("#")[0])
n = 0
while q and n < args.max:
url, d = q.popleft(); n += 1
html, ct = fetch(url, args.cookie, user_agent_value=ua)
if args.delay: time.sleep(args.delay)
if not html: continue
p = Page()
try: p.feed(html)
except Exception: pass
links.add(url.replace(origin, "") or "/")
for f in p.forms:
act = urllib.parse.urljoin(url, f["action"] or url)
key = f["method"] + " " + act.replace(origin, "")
forms.setdefault(key, set()).update(f["params"])
for s in p.inline: inline.add(s)
if d < args.depth:
for href in p.links:
if href.startswith(("mailto:", "tel:", "javascript:", "#")): continue
nxt = urllib.parse.urljoin(url, href).split("#")[0]
if not nxt.startswith(origin): continue
if SKIP_EXT.search(nxt): continue
if exc and exc.search(nxt): continue
if inc and not inc.search(nxt): continue
if nxt not in seen:
seen.add(nxt); q.append((nxt, d + 1))
with open(os.path.join(args.outdir, "forms.txt"), "w") as fh:
for k in sorted(forms):
ps = ", ".join(sorted(forms[k]))
fh.write(f"{k} [{ps}]\n")
open(os.path.join(args.outdir, "links.txt"), "w").write("\n".join(sorted(links)))
open(os.path.join(args.outdir, "api_inline.txt"), "w").write("\n".join(sorted(inline)))
print(f"[+] crawled {n} pages (cap {args.max}, depth {args.depth})")
print(f"[+] forms (endpoints): {len(forms)} -> forms.txt")
print(f"[+] inline ajax urls: {len(inline)} -> api_inline.txt")
print(f"[+] pages reached: {len(links)} -> links.txt")
if not args.cookie:
print("[i] no --cookie: only public pages crawled. Pass an authed session cookie for the full surface.")
if __name__ == "__main__":
main()
Reference in new issue View Git Blame Copy Permalink