#为什么选择 Scrapy？：同步 vs 异步、Twisted 引擎、2026 爬虫生态最佳实践

# 同步爬虫示例（requests + BeautifulSoup）
import requests
from bs4 import BeautifulSoup
import time

def sync_crawler(urls):
    """同步爬虫 - 串行执行"""
    results = []
    start_time = time.time()
    
    for i, url in enumerate(urls):
        print(f"正在爬取第 {i+1} 个URL: {url}")
        
        # 阻塞等待 - 这是性能瓶颈所在
        response = requests.get(url)
        soup = BeautifulSoup(response.text, 'html.parser')
        
        # 模拟数据处理
        data = {
            'url': url,
            'title': soup.find('title').text if soup.find('title') else '',
            'status_code': response.status_code
        }
        results.append(data)
    
    end_time = time.time()
    print(f"同步爬虫完成，总耗时: {end_time - start_time:.2f}秒")
    return results

# 性能测试
urls = ['https://httpbin.org/delay/1'] * 5  # 每个请求延迟1秒
sync_results = sync_crawler(urls)
# 串行执行：5个请求 × 1秒 = 5秒（实际上可能更长）

# Scrapy异步爬虫架构模拟
import asyncio
import aiohttp
from bs4 import BeautifulSoup

class AsyncCrawler:
    """异步爬虫模拟 - 展示Scrapy核心理念"""
    
    def __init__(self, max_concurrent=16):
        self.max_concurrent = max_concurrent
        self.semaphore = asyncio.Semaphore(max_concurrent)
        self.session = None
    
    async def fetch_page(self, url):
        """异步获取单个页面"""
        async with self.semaphore:  # 限制并发数
            try:
                async with self.session.get(url) as response:
                    html = await response.text()
                    soup = BeautifulSoup(html, 'html.parser')
                    
                    return {
                        'url': url,
                        'title': soup.find('title').text if soup.find('title') else '',
                        'status_code': response.status,
                        'size': len(html)
                    }
            except Exception as e:
                print(f"请求失败 {url}: {str(e)}")
                return {'url': url, 'error': str(e)}
    
    async def crawl_multiple(self, urls):
        """异步爬取多个URL"""
        connector = aiohttp.TCPConnector(limit=100, limit_per_host=30)
        timeout = aiohttp.ClientTimeout(total=30)
        
        async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session:
            self.session = session
            start_time = asyncio.get_event_loop().time()
            
            # 并发执行所有请求
            tasks = [self.fetch_page(url) for url in urls]
            results = await asyncio.gather(*tasks, return_exceptions=True)
            
            end_time = asyncio.get_event_loop().time()
            print(f"异步爬虫完成，总耗时: {end_time - start_time:.2f}秒")
            
            return results

# 性能对比测试
async def performance_comparison():
    urls = ['https://httpbin.org/delay/1'] * 5  # 每个请求延迟1秒
    
    # 异步爬虫（模拟Scrapy效果）
    crawler = AsyncCrawler(max_concurrent=16)
    async_results = await crawler.crawl_multiple(urls)
    
    # 结果分析：5个请求几乎同时完成，总耗时接近1秒
    print(f"异步爬虫处理了 {len([r for r in async_results if isinstance(r, dict)])} 个URL")

# 运行异步爬虫
# asyncio.run(performance_comparison())

# Scrapy爬虫示例 - 真实框架使用
import scrapy
from scrapy.crawler import CrawlerProcess

class ExampleSpider(scrapy.Spider):
    name = 'example'
    start_urls = ['https://httpbin.org/delay/1'] * 100  # 100个URL
    
    custom_settings = {
        'CONCURRENT_REQUESTS': 16,  # 并发请求数
        'DOWNLOAD_DELAY': 0.1,     # 下载延迟
        'RANDOMIZE_DOWNLOAD_DELAY': 0.5,  # 随机延迟
    }
    
    def parse(self, response):
        yield {
            'url': response.url,
            'status': response.status,
            'size': len(response.body),
        }

# 运行Scrapy爬虫
# process = CrawlerProcess()
# process.crawl(ExampleSpider)
# process.start()

"""
Twisted = 事件驱动的异步网络框架

核心组件：
  - Reactor：事件循环，驱动整个异步系统
  - Deferred：异步操作的承诺对象
  - Protocol：网络协议实现
  - Factory：协议工厂
  - Transport：底层网络传输

Scrapy基于Twisted构建，天然具备高性能异步能力
"""

# Twisted Reactor机制示意（简化版）
import select
import socket
import time
from collections import deque

class SimpleReactor:
    """简化版Reactor - 展示事件循环原理"""
    
    def __init__(self):
        self.readers = {}  # 读事件监听者
        self.writers = {}  # 写事件监听者
        self.deferreds = deque()  # 延迟对象队列
        self.running = True
    
    def add_reader(self, fd, callback):
        """添加读事件监听"""
        self.readers[fd] = callback
    
    def remove_reader(self, fd):
        """移除读事件监听"""
        return self.readers.pop(fd, None) is not None
    
    def add_writer(self, fd, callback):
        """添加写事件监听"""
        self.writers[fd] = callback
    
    def remove_writer(self, fd):
        """移除写事件监听"""
        return self.writers.pop(fd, None) is not None
    
    def call_later(self, delay, callback, *args):
        """延迟调用"""
        # 在实际Twisted中，这是核心功能
        pass
    
    def run_once(self):
        """执行一次事件循环"""
        # 使用select等待IO事件
        if self.readers or self.writers:
            r_list, w_list, _ = select.select(
                list(self.readers.keys()),
                list(self.writers.keys()),
                [],
                0.1  # 0.1秒超时
            )
            
            # 处理可读事件
            for fd in r_list:
                callback = self.readers.get(fd)
                if callback:
                    callback(fd)
            
            # 处理可写事件
            for fd in w_list:
                callback = self.writers.get(fd)
                if callback:
                    callback(fd)
    
    def run(self):
        """运行事件循环"""
        while self.running:
            self.run_once()
            time.sleep(0.001)  # 防止CPU占用过高
    
    def stop(self):
        """停止事件循环"""
        self.running = False

"""
Scrapy异步处理流程：

1. Engine（引擎）启动
   ↓
2. Engine从Scheduler获取Request
   ↓
3. Engine将Request发送给Downloader
   ↓
4. Downloader处理Request，返回Response
   ↓
5. Engine将Response发送给Spider
   ↓
6. Spider解析Response，产生Items和新Requests
   ↓
7. Items进入Item Pipeline进行处理
   ↓
8. 新Requests返回Scheduler（如果有）
   ↓
9. 重复步骤2-8直到完成

整个流程异步非阻塞，多个请求并发处理
"""

# 2026年企业级爬虫技术栈示例
"""
基础框架层：
├── Scrapy 2.11+          # 核心爬虫框架
├── Scrapy-Redis 0.7+     # 分布式支持
├── Scrapy-Splash 0.9+    # JS渲染（可选）
└── Scrapy-Playwright     # 现代浏览器自动化

数据处理层：
├── Pandas 2.0+           # 数据处理
├── PyArrow 12.0+         # 高性能数据处理
├── SQLAlchemy 2.0+       # 数据库ORM
└── PyMongo 4.0+          # MongoDB驱动

部署运维层：
├── Docker 24.0+          # 容器化
├── Kubernetes 1.28+      # 容器编排
├── Helm 3.12+            # 包管理
└── Prometheus 2.45+      # 监控

基础设施层：
├── Redis 7.2+            # 缓存/队列/去重
├── PostgreSQL 16+        # 结构化数据存储
├── MongoDB 7.0+          # 非结构化数据存储
└── Elasticsearch 8.10+   # 搜索引擎
"""

# 企业级分布式爬虫架构
"""
                    ┌─────────────────┐
                    │   控制中心      │
                    │ • 任务调度      │
                    │ • 监控告警      │
                    │ • 配置管理      │
                    └─────────┬───────┘
                              │
                    ┌─────────▼───────┐
                    │   Redis集群     │
                    │ • 请求队列      │
                    │ • 去重集合      │
                    │ • 状态管理      │
                    │ • 配置存储      │
                    └──────┬──────────┘
                           │
        ┌──────────────────┼──────────────────┐
        │                  │                  │
        ▼                  ▼                  ▼
┌───────────────┐  ┌───────────────┐  ┌───────────────┐
│  爬虫节点1    │  │  爬虫节点2    │  │  爬虫节点N    │
│               │  │               │  │               │
│ • Scrapy      │  │ • Scrapy      │  │ • Scrapy      │
│ • Middleware  │  │ • Middleware  │  │ • Middleware  │
│ • Pipelines   │  │ • Pipelines   │  │ • Pipelines   │
│ • Monitoring  │  │ • Monitoring  │  │ • Monitoring  │
└───────────────┘  └───────────────┘  └───────────────┘
        │                  │                  │
        └──────────────────┼──────────────────┘
                           ▼
                    ┌─────────────────┐
                    │   数据中心      │
                    │ • 数据存储      │
                    │ • 数据清洗      │
                    │ • 数据分析      │
                    │ • API服务       │
                    └─────────────────┘
"""

# Scrapy性能优化配置示例
"""
# settings.py 高性能配置
CONCURRENT_REQUESTS = 32                    # 并发请求数
CONCURRENT_REQUESTS_PER_DOMAIN = 8          # 单域名并发数
DOWNLOAD_DELAY = 0.5                        # 下载延迟
RANDOMIZE_DOWNLOAD_DELAY = 0.5              # 随机延迟
AUTOTHROTTLE_ENABLED = True                 # 自动限速
AUTOTHROTTLE_START_DELAY = 0.5              # 自动限速起始延迟
AUTOTHROTTLE_MAX_DELAY = 10                 # 自动限速最大延迟
AUTOTHROTTLE_TARGET_CONCURRENCY = 8.0       # 目标并发数
DNSCACHE_ENABLED = True                     # DNS缓存
COOKIES_ENABLED = False                     # 禁用cookies（如不需要）
TELNETCONSOLE_ENABLED = False               # 禁用telnet控制台
LOG_LEVEL = 'INFO'                          # 日志级别
"""

# Scrapy中间件架构
"""
请求处理链：
Downloader Middleware → Spider → Spider Middleware → Item Pipeline

响应处理链：
Downloader → Spider Middleware → Spider → Downloader Middleware
"""

# Scrapy扩展点示例
import scrapy
from scrapy import signals
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings

class MyExtension:
    """自定义扩展示例"""
    
    def __init__(self, crawler):
        self.crawler = crawler
        self.stats = crawler.stats
    
    @classmethod
    def from_crawler(cls, crawler):
        ext = cls(crawler)
        
        # 监听信号
        crawler.signals.connect(ext.spider_opened, signal=signals.spider_opened)
        crawler.signals.connect(ext.spider_closed, signal=signals.spider_closed)
        
        return ext
    
    def spider_opened(self, spider):
        print(f"Spider {spider.name} opened")
    
    def spider_closed(self, spider):
        print(f"Spider {spider.name} closed, scraped {self.stats.get_value('item_scraped_count')} items")

# Scrapy压力测试工具
import time
import asyncio
from scrapy.crawler import CrawlerRunner
from scrapy.utils.project import get_project_settings
from twisted.internet import reactor

class PerformanceTestSpider(scrapy.Spider):
    name = 'performance_test'
    
    def __init__(self, url_count=1000, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.start_urls = [f'https://httpbin.org/delay/0.1'] * int(url_count)
        self.start_time = time.time()
    
    def parse(self, response):
        elapsed = time.time() - self.start_time
        yield {
            'url': response.url,
            'status': response.status,
            'response_time': response.meta.get('download_latency'),
            'total_elapsed': elapsed
        }

def run_performance_test(url_count=1000):
    """运行性能测试"""
    runner = CrawlerRunner(get_project_settings())
    d = runner.crawl(PerformanceTestSpider, url_count=url_count)
    
    # 测试完成后打印结果
    d.addBoth(lambda _: reactor.stop())
    reactor.run()

"""
初学者路径：
requests/urllib3 → Scrapy基础 → Scrapy进阶 → Scrapy-Redis分布式

进阶者路径：
已有爬虫经验 → Scrapy架构理解 → 性能优化 → 分布式部署

企业级路径：
需求分析 → 架构设计 → Scrapy开发 → Docker部署 → K8s运维
"""