9
异步爬虫 aiohttp
aiohttp: 10x Faster
requests 串行爬 100 个网页要 100 秒，aiohttp 并发爬只要 1-2 秒。asyncio + aiohttp 是爬虫提速的终极武器。
基础：ClientSession
pip install aiohttp
import aiohttp, asyncio
async def fetch(url):
async with aiohttp.ClientSession() as session:
async with session.get(url) as resp:
return resp.status, await resp.text()
async def main():
status, text = await fetch("https://example.com")
print(status, len(text))
asyncio.run(main())
并发：gather + Semaphore 限流
async def fetch_limited(session, url, sem):
async with sem: # 同时最多10个
async with session.get(url) as resp:
return url, resp.status
async def main(urls):
sem = asyncio.Semaphore(10)
async with aiohttp.ClientSession() as session:
tasks = [fetch_limited(session, u, sem) for u in urls]
results = await asyncio.gather(*tasks)
return results
urls = ["https://example.com"] * 100
results = asyncio.run(main(urls))
连接池与 Session 复用
# 别每个请求建一个 ClientSession！一个 Session 复用连接池
import aiohttp
connector = aiohttp.TCPConnector(limit=20) # 最大连接数
async def main(urls):
# 整个爬取过程用一个 Session
async with aiohttp.ClientSession(connector=connector) as session:
# ... 用这个 session 发所有请求
pass
请求头、Cookie 与错误处理
# aiohttp 完整请求：headers / cookies / 超时 / 重试
import aiohttp, asyncio
headers = {"User-Agent": "Mozilla/5.0 ..."}
cookies = {"sessionid": "abc123"}
timeout = aiohttp.ClientTimeout(total=10)
async def fetch(session, url):
try:
async with session.get(url, headers=headers,
cookies=cookies, timeout=timeout) as resp:
if resp.status == 200:
return await resp.text()
else:
print(f"{url} 状态 {resp.status}")
except Exception as e:
print(f"{url} 失败：{e}")
return None
完整案例：同步 vs 异步速度对比
# speed_test.py —— 10个请求，对比 requests 和 aiohttp
import time, requests, asyncio, aiohttp
URL = "https://httpbin.org/delay/1" # 服务器延迟1秒返回
URLS = [URL] * 10
# 同步：串行等，10秒
start = time.time()
[requests.get(u) for u in URLS]
print(f"同步：{time.time()-start:.2f}s")
# 异步：并发，约1秒
async def async_main():
sem = asyncio.Semaphore(10)
async with aiohttp.ClientSession() as s:
async def one(u):
async with sem:
async with s.get(u) as r:
return r.status
await asyncio.gather(*[one(u) for u in URLS])
start = time.time()
asyncio.run(async_main())
print(f"异步：{time.time()-start:.2f}s")
输出
# 同步：10.23s
# 异步：1.15s
# 10倍提速，请求越多越夸张
异步解析：aiohttp + BeautifulSoup
# aiohttp 拿 HTML，BS4 解析（解析是 CPU 操作，同步）
async def parse_page(session, url, sem):
async with sem:
async with session.get(url) as resp:
html = await resp.text()
# 解析（同步操作，但很快）
soup = BeautifulSoup(html, "lxml")
title = soup.select_one("h1").text
return title
完整案例：异步爬 100 个网页
# async_crawl.py —— aiohttp 爬 100 个 URL
import aiohttp, asyncio, time
URLS = [f"https://httpbin.org/delay/1"] * 100
async def fetch(session, url, sem):
async with sem:
try:
async with session.get(url, timeout=10) as resp:
return resp.status
except:
return 0
async def main():
sem = asyncio.Semaphore(20)
async with aiohttp.ClientSession() as session:
tasks = [fetch(session, u, sem) for u in URLS]
results = await asyncio.gather(*tasks)
print(f"成功 {results.count(200)}/100")
start = time.time()
asyncio.run(main())
print(f"耗时 {time.time()-start:.2f}s")
输出
# 成功 100/100
# 耗时 5.32s
# 同步 requests 串行要 100s，异步 20 并发只要 5s
aiohttp 和 requests 怎么选
| requests | aiohttp | |
|---|---|---|
| 速度 | 串行慢 | 并发快 10 倍+ |
| 复杂度 | 简单 | 要懂 async/await |
| 适合 | 少量请求、脚本 | 几百上千个请求 |
| 错误处理 | try/except | try/except + gather |
别无限制并发
不加 Semaphore，1000 个请求同时飞出去——对方服务器直接 DOS，你也立刻被封 IP。Semaphore 限并发（一般 5-20），既快又礼貌。