1
2
3
4
5
6
7
8

反爬虫技术
├── 被动检测（静默收集信息）
│   ├── 浏览器指纹采集 / 行为轨迹记录 / TLS·HTTP 协议指纹
├── 主动挑战（要求客户端完成任务）
│   ├── JavaScript 计算挑战 / CAPTCHA 验证码 / Proof-of-Work
└── 响应策略（检测异常后处理）
    ├── 直接封禁（403/429）/ 静默降级（返回假数据）
    └── 蜜罐诱捕 / 延迟响应

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

import requests

headers = {
    "sec-ch-ua": '"Google Chrome";v="123", "Not:A-Brand";v="8", "Chromium";v="123"',
    "sec-ch-ua-mobile": "?0",
    "sec-ch-ua-platform": '"Windows"',
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
                  "(KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Sec-Fetch-Site": "none",
    "Sec-Fetch-Mode": "navigate",
    "Accept-Encoding": "gzip, deflate, br",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}
resp = requests.get("https://example.com", headers=headers)

1
2

from curl_cffi import requests as cffi_requests
resp = cffi_requests.get("https://example.com", impersonate="chrome123")

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

import random, time, requests

class ProxyRotator:
    def __init__(self, proxies: list[str]):
        self.proxies = proxies
        self.failed = {}

    def get_proxy(self) -> dict:
        available = [p for p in self.proxies if self.failed.get(p, 0) < 3]
        proxy = random.choice(available)
        return {"http": f"http://{proxy}", "https": f"http://{proxy}"}

    def mark_failed(self, proxy: str):
        self.failed[proxy] = self.failed.get(proxy, 0) + 1

rotator = ProxyRotator(["1.2.3.4:8080", "5.6.7.8:3128"])
for url in target_urls:
    proxy = rotator.get_proxy()
    try:
        resp = requests.get(url, proxies=proxy, timeout=10)
    except Exception:
        rotator.mark_failed(list(proxy.values())[0])
    time.sleep(random.uniform(1.5, 4.0))  # 随机延迟

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

import random

def generate_human_track(distance: int) -> list[dict]:
    """生成仿人类滑动轨迹：先加速后减速 + Y 轴抖动"""
    track, current, t, v = [], 0, 0, 0
    mid = distance * 0.7
    while current < distance:
        a = random.uniform(2.5, 3.5) if current < mid else random.uniform(-3.0, -1.5)
        dt = random.uniform(10, 20)
        v = max(v + a * (dt / 1000), 0.5)
        current = min(current + v * (dt / 1000) * 100, distance)
        t += dt
        track.append({"x": round(current, 2), "y": round(random.uniform(-2, 2), 2), "t": round(t)})
    return track

1
2
3
4
5
6
7
8

// 网站端常见检测点
const botSignals = {
    webdriver: navigator.webdriver === true,
    noPlugins: navigator.plugins.length === 0,
    noLanguages: !navigator.languages || navigator.languages.length === 0,
    noChromeRuntime: !window.chrome || !window.chrome.runtime,
    automationAPIs: !!(window.__puppeteer_evaluation_script__),
};

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

from playwright.sync_api import sync_playwright
from playwright_stealth import stealth_sync

with sync_playwright() as p:
    browser = p.chromium.launch(headless=False)
    context = browser.new_context(
        viewport={"width": 1920, "height": 1080},
        locale="zh-CN", timezone_id="Asia/Shanghai",
    )
    page = context.new_page()
    stealth_sync(page)  # 修补 navigator.webdriver 等检测点
    page.goto("https://bot.sannysoft.com/")
    browser.close()

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

// Frida 脚本：覆盖 WebView 中的指纹检测点
Java.perform(function () {
    var WebView = Java.use("android.webkit.WebView");
    WebView.loadUrl.overload("java.lang.String").implementation = function (url) {
        this.evaluateJavascript(
            `Object.defineProperty(navigator, 'webdriver', {get: () => false});` +
            `Object.defineProperty(navigator, 'plugins', {get: () => [1,2,3,4,5]});`,
            null
        );
        this.loadUrl.overload("java.lang.String").call(this, url);
    };
});

1
2
3
4
5
6
7
8

客户端                      Cloudflare                   源站
  │── 首次请求 ──────────>│                              │
  │<── 返回 JS 挑战 ─────│  (HTTP 503 + 混淆 JS)       │
  │   [执行 JS + 采集指纹]│                              │
  │── 提交答案 ──────────>│── 验证 ──>                  │
  │<── 302 + Set-Cookie ──│  (cf_clearance)              │
  │── 携带 Cookie 访问 ──>│── 代理转发 ────────────────>│
  │<── 真实内容 ──────────│<── 源站响应 ────────────────│

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

from playwright.sync_api import sync_playwright

def extract_cf_cookies(url: str) -> dict:
    """用 Playwright 通过挑战后提取 Cookie，再用 requests 高效请求"""
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=False)
        page = browser.new_page()
        page.goto(url)
        page.wait_for_load_state("networkidle")
        page.wait_for_timeout(5000)
        cookies = {c["name"]: c["value"] for c in page.context.cookies()}
        browser.close()
    return cookies

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

from playwright.sync_api import sync_playwright

with sync_playwright() as p:
    page = p.chromium.launch().new_page()
    page.goto("https://example.com/spa")
    page.wait_for_selector(".product-list", timeout=15000)
    for _ in range(5):  # 处理无限滚动
        page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
        page.wait_for_timeout(1500)
    html = page.content()

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

from playwright.sync_api import sync_playwright

api_data = []
def on_response(resp):
    if "/api/" in resp.url and resp.status == 200:
        try: api_data.append(resp.json())
        except: pass

with sync_playwright() as p:
    page = p.chromium.launch().new_page()
    page.on("response", on_response)
    page.goto("https://example.com/products")
    page.wait_for_load_state("networkidle")
# api_data 中包含所有拦截到的 JSON 响应

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

from bs4 import BeautifulSoup

def is_honeypot(element) -> bool:
    style = element.get("style", "").lower()
    traps = ["display:none", "display: none", "visibility:hidden",
             "opacity:0", "font-size:0", "left:-9999"]
    if any(t in style for t in traps):
        return True
    classes = " ".join(element.get("class", [])).lower()
    return any(k in classes for k in ["trap", "honey", "hidden", "invisible"])

soup = BeautifulSoup(html, "html.parser")
safe_links = [a["href"] for a in soup.find_all("a", href=True) if not is_honeypot(a)]

1
2
3
4
5

sensor_data:
├── 设备信息: UA / 屏幕分辨率 / 时区
├── 浏览器指纹: Canvas hash / WebGL 渲染器 / AudioContext
├── 行为数据: 鼠标轨迹 / 键盘事件 / 触摸事件 / 滚动
└── 环境检测: 自动化工具 / DevTools / 虚拟机检测

1
2
3
4
5
6
7

1. 有公开 API / RSS？       → 是 → 直接调用（最优方案）
2. 服务端渲染（SSR）？       → 是 → requests + 正确 Headers
3. 有 JS 挑战 / WAF？       → 否 → Playwright 渲染即可
4. 哪家 WAF？               → Cloudflare: Playwright+stealth+住宅代理
                             → Akamai: 真实浏览器+行为模拟+高质量代理
5. 有 CAPTCHA？             → 是 → 打码平台 API
6. 有 IP 限制？             → 是 → 代理池轮转 + 频率控制

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

from playwright.sync_api import sync_playwright
from playwright_stealth import stealth_sync

def bypass_cloudflare(target_url: str) -> tuple[str, dict]:
    with sync_playwright() as p:
        browser = p.chromium.launch(
            headless=False,
            args=["--disable-blink-features=AutomationControlled"]
        )
        context = browser.new_context(viewport={"width": 1920, "height": 1080}, locale="zh-CN")
        page = context.new_page()
        stealth_sync(page)
        page.goto(target_url)

        # 等待 cf_clearance Cookie 出现
        for _ in range(30):
            cookies = {c["name"]: c["value"] for c in context.cookies()}
            if "cf_clearance" in cookies:
                break
            page.wait_for_timeout(1000)

        content = page.content()
        browser.close()
    return content, cookies

1
2
3
4
5
6

import requests

# cf_clearance 绑定 IP + User-Agent，必须保持一致
cookies = {"cf_clearance": "从浏览器获取的值", "__cf_bm": "对应的值"}
headers = {"User-Agent": "获取 Cookie 时使用的 UA 字符串"}
resp = requests.get("https://protected-site.com/data", cookies=cookies, headers=headers)