#创建你的首个Scrapy工程 - 项目结构、配置与初始化完整指南

# myproject/__init__.py
"""
myproject - Scrapy项目包初始化

此文件为空或包含包级别的初始化代码
"""

# settings.py - 基础配置
BOT_NAME = 'myproject'
SPIDER_MODULES = ['myproject.spiders']
NEWSPIDER_MODULE = 'myproject.spiders'

# 遵守robots.txt规则
ROBOTSTXT_OBEY = True

# 设置User-Agent
USER_AGENT = 'myproject (+http://www.yourdomain.com)'

# 性能相关配置
CONCURRENT_REQUESTS = 16
CONCURRENT_REQUESTS_PER_DOMAIN = 8
DOWNLOAD_DELAY = 1
RANDOMIZE_DOWNLOAD_DELAY = 0.5

# 内存配置
CONCURRENT_ITEMS = 100
MEMUSAGE_LIMIT_MB = 2048
MEMUSAGE_NOTIFY_MAIL = ['admin@example.com']

# 下载器配置
DOWNLOAD_TIMEOUT = 180
DOWNLOAD_MAXSIZE = 1048576000  # 1GB
DOWNLOAD_WARNSIZE = 33554432   # 32MB

# 重试配置
RETRY_TIMES = 3
RETRY_HTTP_CODES = [500, 502, 503, 504, 408, 429]

# 自动限速配置
AUTOTHROTTLE_ENABLED = True
AUTOTHROTTLE_START_DELAY = 1
AUTOTHROTTLE_MAX_DELAY = 10
AUTOTHROTTLE_TARGET_CONCURRENCY = 2.0
AUTOTHROTTLE_DEBUG = False

# 中间件配置
DOWNLOADER_MIDDLEWARES = {
    'myproject.middlewares.RandomUserAgentMiddleware': 400,
    'myproject.middlewares.ProxyMiddleware': 410,
    'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None,
}

SPIDER_MIDDLEWARES = {
    'myproject.middlewares.CustomSpiderMiddleware': 543,
}

# 管道配置
ITEM_PIPELINES = {
    'myproject.pipelines.ValidationPipeline': 300,
    'myproject.pipelines.DuplicatesPipeline': 350,
    'myproject.pipelines.DatabasePipeline': 400,
}

# 数据导出配置
FEED_EXPORT_ENCODING = 'utf-8'
FEED_FORMAT = 'jsonlines'  # json, csv, xml, pickle, marshal
FEED_URI = 'data/%(name)s_%(time)s.json'

# 日志配置
LOG_LEVEL = 'INFO'
LOG_FILE = 'logs/scrapy.log'
LOG_FORMAT = '%(asctime)s [%(name)s] %(levelname)s: %(message)s'
LOG_DATEFORMAT = '%Y-%m-%d %H:%M:%S'

# settings.py - 生产环境优化配置
import os

# 基础配置
BOT_NAME = 'myproject'
SPIDER_MODULES = ['myproject.spiders']
NEWSPIDER_MODULE = 'myproject.spiders'

# 性能优化
CONCURRENT_REQUESTS = 32
CONCURRENT_REQUESTS_PER_DOMAIN = 8
CONCURRENT_ITEMS = 200

# 下载延迟
DOWNLOAD_DELAY = 1
RANDOMIZE_DOWNLOAD_DELAY = 0.5

# 自动限速
AUTOTHROTTLE_ENABLED = True
AUTOTHROTTLE_START_DELAY = 1
AUTOTHROTTLE_MAX_DELAY = 10
AUTOTHROTTLE_TARGET_CONCURRENCY = 2.0

# 超时设置
DOWNLOAD_TIMEOUT = 180
DOWNLOAD_MAXSIZE = 1048576000

# 重试设置
RETRY_TIMES = 3
RETRY_HTTP_CODES = [500, 502, 503, 504, 408, 429, 421, 423, 429]

# 连接池设置
REACTOR_THREADPOOL_MAXSIZE = 20
DNSCACHE_ENABLED = True
DNSCACHE_SIZE = 20000
DNS_TIMEOUT = 30

# 内存管理
MEMUSAGE_ENABLED = True
MEMUSAGE_LIMIT_MB = 2048
MEMUSAGE_NOTIFY_MAIL = ['admin@example.com']

# 日志设置
LOG_LEVEL = 'INFO'
LOG_FORMAT = '%(asctime)s [%(name)s] %(levelname)s: %(message)s'
LOG_DATEFORMAT = '%Y-%m-%d %H:%M:%S'

# 中间件设置
DOWNLOADER_MIDDLEWARES = {
    'myproject.middlewares.RandomUserAgentMiddleware': 400,
    'myproject.middlewares.ProxyMiddleware': 410,
    'myproject.middlewares.RetryMiddleware': 500,
    'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None,
}

# 管道设置
ITEM_PIPELINES = {
    'myproject.pipelines.ValidationPipeline': 300,
    'myproject.pipelines.CleaningPipeline': 350,
    'myproject.pipelines.StoragePipeline': 400,
}

# 环境变量配置
ENVIRONMENT = os.getenv('SCRAPE_ENV', 'development')
if ENVIRONMENT == 'production':
    LOG_LEVEL = 'WARNING'
    CONCURRENT_REQUESTS = 64
    CONCURRENT_REQUESTS_PER_DOMAIN = 16

# settings_dev.py - 开发环境配置
from .settings import *

CONCURRENT_REQUESTS = 8
CONCURRENT_REQUESTS_PER_DOMAIN = 4
DOWNLOAD_DELAY = 0.5
LOG_LEVEL = 'DEBUG'

# settings_prod.py - 生产环境配置
from .settings import *

CONCURRENT_REQUESTS = 32
CONCURRENT_REQUESTS_PER_DOMAIN = 8
DOWNLOAD_DELAY = 1
AUTOTHROTTLE_ENABLED = True
LOG_LEVEL = 'INFO'

# spiders/example.py - 基础爬虫模板
import scrapy

class ExampleSpider(scrapy.Spider):
    """
    示例爬虫 - 演示基本爬虫结构
    """
    name = 'example'
    allowed_domains = ['example.com']
    start_urls = ['http://example.com/']

    # 自定义设置（可选）
    custom_settings = {
        'DOWNLOAD_DELAY': 1,
        'RANDOMIZE_DOWNLOAD_DELAY': 0.5
    }

    def start_requests(self):
        """
        自定义起始请求方法
        """
        headers = {
            'User-Agent': 'Mozilla/5.0 (compatible; MyBot/1.0)'
        }
        for url in self.start_urls:
            yield scrapy.Request(
                url=url,
                headers=headers,
                callback=self.parse,
                meta={'page': 1}
            )

    def parse(self, response):
        """
        解析响应的方法
        """
        # 提取数据
        for item in response.css('div.item'):
            yield {
                'title': item.css('h2::text').get(),
                'price': item.css('span.price::text').get(),
                'url': response.url,
                'timestamp': self.crawler.stats.start_time.isoformat()
            }

        # 提取下一页链接
        next_page = response.css('a.next-page::attr(href)').get()
        if next_page:
            yield response.follow(
                next_page,
                callback=self.parse,
                meta={'page': response.meta.get('page', 1) + 1}
            )

# spiders/crawl_example.py - 爬行爬虫示例
import scrapy
from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor

class CrawlExampleSpider(CrawlSpider):
    """
    爬行爬虫示例 - 演示规则匹配和链接提取
    """
    name = 'crawl_example'
    allowed_domains = ['example.com']
    start_urls = ['http://example.com/']

    rules = (
        # 规则1: 提取分类页面链接
        Rule(
            LinkExtractor(
                allow=r'/category/',
                restrict_css='.category-nav'
            ),
            callback='parse_category',
            follow=True
        ),
        # 规则2: 提取产品页面链接
        Rule(
            LinkExtractor(
                allow=r'/product/\d+/',
                deny=r'/admin/'
            ),
            callback='parse_product',
            follow=False
        ),
    )

    def parse_category(self, response):
        """解析分类页面"""
        for product_link in response.css('.product-item a::attr(href)'):
            yield response.follow(product_link, callback=self.parse_product)

    def parse_product(self, response):
        """解析产品页面"""
        yield {
            'name': response.css('h1.product-name::text').get(),
            'price': response.css('.product-price::text').get(),
            'description': response.css('.product-description::text').get(),
            'url': response.url,
            'category': response.css('.breadcrumb a::text').getall()[-1]
        }

# test_performance.py - 性能测试脚本
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings
import time

def run_performance_test():
    """
    性能测试函数
    """
    settings = get_project_settings()
    settings.set('CONCURRENT_REQUESTS', 16)
    settings.set('DOWNLOAD_DELAY', 0.5)
    
    process = CrawlerProcess(settings)
    
    start_time = time.time()
    process.crawl('example')
    process.start()
    
    end_time = time.time()
    print(f"爬虫执行时间: {end_time - start_time:.2f}秒")

if __name__ == '__main__':
    run_performance_test()