#Scrapy自动限速AutoThrottle完全指南 - 智能频率控制与反爬规避技术详解

"""
AutoThrottle的核心优势：
1. 智能调节：根据网站响应自动调整请求频率
2. 并发控制：动态调节并发请求数量
3. 延迟适应：自动适应网站的响应时间
4. 反爬规避：模拟人类访问行为，降低被封概率
5. 性能优化：在不触发反爬的前提下最大化爬取效率
"""

"""
为什么需要限速：
1. 反爬虫机制：防止被目标网站识别为爬虫
2. 服务器压力：避免对目标服务器造成过大负担
3. IP封禁：降低IP被封的风险
4. 法律合规：遵守网站的爬虫政策和法律法规
5. 数据质量：确保请求响应的稳定性
"""

"""
AutoThrottle工作流程：

1. 初始阶段：使用起始延迟（AUTOTHROTTLE_START_DELAY）
2. 监测阶段：计算平均响应时间
3. 调整阶段：根据响应时间调整延迟
4. 并发控制：限制并发请求数量
5. 优化阶段：持续优化请求频率
"""

class AutoThrottleMechanism:
    """
    AutoThrottle机制演示
    """
    
    def calculate_target_delay(self, avg_response_time, target_concurrency=1.0):
        """
        计算目标延迟
        """
        # 目标延迟 = 平均响应时间 / 目标并发数
        target_delay = avg_response_time / target_concurrency
        return target_delay
    
    def adjust_concurrency(self, response_time, current_concurrency):
        """
        调整并发数
        """
        # 如果响应时间过长，降低并发数
        if response_time > 5.0:  # 响应时间超过5秒
            return max(1, current_concurrency * 0.8)  # 降低20%
        # 如果响应时间较短，可以适当提高并发数
        elif response_time < 1.0:  # 响应时间小于1秒
            return min(8, current_concurrency * 1.2)  # 提高20%，但不超过8
        else:
            return current_concurrency

# settings.py - AutoThrottle基础配置
AUTOTHROTTLE_ENABLED = True  # 启用AutoThrottle
AUTOTHROTTLE_START_DELAY = 5  # 起始延迟（秒）
AUTOTHROTTLE_MAX_DELAY = 60  # 最大延迟（秒）
AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0  # 目标并发数
AUTOTHROTTLE_DEBUG = True  # 调试模式，输出详细信息

"""
配置参数详细说明：

AUTOTHROTTLE_ENABLED:
- 功能：启用或禁用AutoThrottle
- 默认值：False
- 建议：生产环境设置为True

AUTOTHROTTLE_START_DELAY:
- 功能：初始请求延迟
- 默认值：5.0
- 建议：根据目标网站响应时间调整

AUTOTHROTTLE_MAX_DELAY:
- 功能：最大请求延迟
- 默认值：60.0
- 建议：设置合理的上限避免过长等待

AUTOTHROTTLE_TARGET_CONCURRENCY:
- 功能：目标并发请求数
- 默认值：1.0
- 建议：根据目标网站承受能力调整

AUTOTHROTTLE_DEBUG:
- 功能：调试信息输出
- 默认值：False
- 建议：开发调试时启用
"""

# settings.py - 高级配置策略
CUSTOM_SETTINGS = {
    # 高防护网站配置
    'HIGH_SECURITY_SITES': {
        'AUTOTHROTTLE_START_DELAY': 10,
        'AUTOTHROTTLE_MAX_DELAY': 120,
        'AUTOTHROTTLE_TARGET_CONCURRENCY': 0.5,
        'CONCURRENT_REQUESTS': 1,
        'DOWNLOAD_DELAY': 5,
    },
    
    # 普通网站配置
    'NORMAL_SITES': {
        'AUTOTHROTTLE_START_DELAY': 3,
        'AUTOTHROTTLE_MAX_DELAY': 60,
        'AUTOTHROTTLE_TARGET_CONCURRENCY': 2.0,
        'CONCURRENT_REQUESTS': 8,
    },
    
    # API接口配置
    'API_ENDPOINTS': {
        'AUTOTHROTTLE_START_DELAY': 1,
        'AUTOTHROTTLE_MAX_DELAY': 10,
        'AUTOTHROTTLE_TARGET_CONCURRENCY': 4.0,
        'CONCURRENT_REQUESTS': 16,
    }
}

class DynamicThrottleSettings:
    """
    动态限速配置
    """
    
    def __init__(self):
        self.site_profiles = {
            'ecommerce': {
                'start_delay': 2,
                'max_delay': 30,
                'target_concurrency': 1.5,
                'response_threshold': 2.0
            },
            'news': {
                'start_delay': 1,
                'max_delay': 15,
                'target_concurrency': 3.0,
                'response_threshold': 1.0
            },
            'social_media': {
                'start_delay': 5,
                'max_delay': 60,
                'target_concurrency': 0.8,
                'response_threshold': 3.0
            }
        }
    
    def get_settings_for_site(self, site_type):
        """
        根据网站类型获取配置
        """
        profile = self.site_profiles.get(site_type, self.site_profiles['news'])
        return {
            'AUTOTHROTTLE_START_DELAY': profile['start_delay'],
            'AUTOTHROTTLE_MAX_DELAY': profile['max_delay'],
            'AUTOTHROTTLE_TARGET_CONCURRENCY': profile['target_concurrency'],
            'RESPONSE_THRESHOLD': profile['response_threshold']
        }

import time
import random
import logging
from collections import defaultdict, deque
from datetime import datetime, timedelta

class CustomThrottleMiddleware:
    """
    自定义限速中间件
    """
    
    def __init__(self):
        self.request_times = defaultdict(list)  # 记录请求时间
        self.response_times = defaultdict(deque)  # 响应时间滑动窗口
        self.concurrency_counts = defaultdict(int)  # 并发计数
        self.site_configs = {}  # 网站配置
        self.logger = logging.getLogger(__name__)
    
    def process_request(self, request, spider):
        """
        处理请求，实施限速
        """
        domain = self._extract_domain(request.url)
        
        # 获取网站配置
        config = self._get_site_config(domain)
        
        # 检查并发限制
        if self._check_concurrency_limit(domain, config):
            delay = self._calculate_dynamic_delay(domain, config)
            if delay > 0:
                time.sleep(delay)
        
        # 记录请求时间
        self.request_times[domain].append(time.time())
        
        return None
    
    def process_response(self, request, response, spider):
        """
        处理响应，更新限速参数
        """
        domain = self._extract_domain(request.url)
        
        # 计算响应时间
        if hasattr(request, '_request_time'):
            response_time = time.time() - request._request_time
            self.response_times[domain].append(response_time)
            
            # 保持滑动窗口大小
            if len(self.response_times[domain]) > 10:
                self.response_times[domain].popleft()
        
        return response
    
    def _extract_domain(self, url):
        """
        提取域名
        """
        from urllib.parse import urlparse
        return urlparse(url).netloc
    
    def _get_site_config(self, domain):
        """
        获取网站配置
        """
        # 默认配置
        default_config = {
            'min_delay': 1,
            'max_delay': 30,
            'max_concurrency': 2,
            'response_window_size': 10
        }
        
        # 特殊网站配置
        special_configs = {
            'weibo.com': {'min_delay': 3, 'max_delay': 60, 'max_concurrency': 1},
            'zhihu.com': {'min_delay': 2, 'max_delay': 30, 'max_concurrency': 1.5},
            'taobao.com': {'min_delay': 5, 'max_delay': 120, 'max_concurrency': 0.8}
        }
        
        return special_configs.get(domain, default_config)
    
    def _check_concurrency_limit(self, domain, config):
        """
        检查并发限制
        """
        current_concurrency = self.concurrency_counts[domain]
        max_concurrency = config['max_concurrency']
        
        if current_concurrency >= max_concurrency:
            # 并发数已达到上限，需要等待
            return True
        
        # 增加并发计数
        self.concurrency_counts[domain] += 1
        return False
    
    def _calculate_dynamic_delay(self, domain, config):
        """
        计算动态延迟
        """
        # 获取最近的响应时间
        recent_responses = list(self.response_times[domain])
        
        if not recent_responses:
            # 如果没有历史数据，使用最小延迟
            return config['min_delay']
        
        # 计算平均响应时间
        avg_response_time = sum(recent_responses) / len(recent_responses)
        
        # 基于响应时间计算延迟
        calculated_delay = avg_response_time * 0.5  # 响应时间的一半作为延迟
        
        # 应用配置限制
        delay = max(config['min_delay'], min(calculated_delay, config['max_delay']))
        
        return delay

import time
import random
from collections import defaultdict
from datetime import datetime, timedelta
import statistics

class AdvancedThrottleMiddleware:
    """
    高级限速中间件
    """
    
    def __init__(self):
        self.domain_stats = defaultdict(dict)
        self.rate_limiter = {}
        self.burst_control = defaultdict(int)
        self.adaptive_params = defaultdict(dict)
    
    def process_request(self, request, spider):
        """
        高级限速处理
        """
        domain = self._extract_domain(request.url)
        
        # 更新统计信息
        self._update_stats(domain)
        
        # 计算延迟时间
        delay = self._calculate_advanced_delay(domain, request)
        
        if delay > 0:
            # 实施延迟
            time.sleep(delay)
        
        # 更新请求时间戳
        self.domain_stats[domain]['last_request'] = time.time()
        
        return None
    
    def _calculate_advanced_delay(self, domain, request):
        """
        计算高级延迟算法
        """
        stats = self.domain_stats[domain]
        
        # 基础延迟计算
        base_delay = self._calculate_base_delay(domain)
        
        # 峰值控制延迟
        peak_delay = self._calculate_peak_delay(domain)
        
        # 智能调整延迟
        smart_delay = self._calculate_smart_delay(domain)
        
        # 随机化延迟（避免规律性）
        random_factor = random.uniform(0.8, 1.2)
        
        # 最终延迟 = 基础延迟 + 峰值延迟 + 智能延迟
        final_delay = (base_delay + peak_delay + smart_delay) * random_factor
        
        # 应用全局限制
        final_delay = min(final_delay, 60)  # 最大延迟不超过60秒
        
        return max(0, final_delay)
    
    def _calculate_base_delay(self, domain):
        """
        计算基础延迟
        """
        stats = self.domain_stats[domain]
        
        if 'response_times' in stats and stats['response_times']:
            avg_response = statistics.mean(stats['response_times'])
            # 基础延迟 = 平均响应时间的一定比例
            return avg_response * 0.3
        
        return 1.0  # 默认延迟
    
    def _calculate_peak_delay(self, domain):
        """
        计算峰值控制延迟
        """
        stats = self.domain_stats[domain]
        
        # 检查是否在高峰时段（简单实现）
        current_hour = datetime.now().hour
        if 9 <= current_hour <= 17:  # 工作时间
            return 0.5  # 增加延迟
        else:
            return 0.0  # 正常延迟
    
    def _calculate_smart_delay(self, domain):
        """
        计算智能延迟
        """
        stats = self.domain_stats[domain]
        
        # 如果有错误历史，增加延迟
        if stats.get('error_count', 0) > 0:
            error_ratio = stats['error_count'] / stats.get('total_requests', 1)
            return error_ratio * 5  # 错误率 * 5秒
        
        return 0.0
    
    def _update_stats(self, domain):
        """
        更新统计信息
        """
        if domain not in self.domain_stats:
            self.domain_stats[domain] = {
                'total_requests': 0,
                'error_count': 0,
                'response_times': [],
                'request_times': [],
                'last_request': time.time()
            }
        
        # 更新请求计数
        self.domain_stats[domain]['total_requests'] += 1
        
        # 限制统计窗口大小
        if len(self.domain_stats[domain]['response_times']) > 50:
            self.domain_stats[domain]['response_times'] = \
                self.domain_stats[domain]['response_times'][-50:]
        
        if len(self.domain_stats[domain]['request_times']) > 50:
            self.domain_stats[domain]['request_times'] = \
                self.domain_stats[domain]['request_times'][-50:]

    def _extract_domain(self, url):
        """
        提取域名
        """
        from urllib.parse import urlparse
        return urlparse(url).netloc

import numpy as np
from sklearn.linear_model import LinearRegression
import statistics

class MLBasedThrottle:
    """
    基于机器学习的智能限速
    """
    
    def __init__(self):
        self.model = LinearRegression()
        self.training_data = []
        self.is_trained = False
    
    def collect_training_data(self, response_time, concurrency, success_rate, delay):
        """
        收集训练数据
        """
        features = [response_time, concurrency, success_rate]
        self.training_data.append((features, delay))
        
        # 当数据足够时训练模型
        if len(self.training_data) >= 20:
            self._train_model()
    
    def predict_optimal_delay(self, response_time, concurrency, success_rate):
        """
        预测最优延迟
        """
        if not self.is_trained:
            # 如果模型未训练，使用经验公式
            return self._fallback_prediction(response_time, concurrency, success_rate)
        
        features = np.array([[response_time, concurrency, success_rate]])
        predicted_delay = self.model.predict(features)[0]
        
        # 限制预测值范围
        return max(0.5, min(predicted_delay, 60.0))
    
    def _train_model(self):
        """
        训练模型
        """
        X = np.array([data[0] for data in self.training_data])
        y = np.array([data[1] for data in self.training_data])
        
        self.model.fit(X, y)
        self.is_trained = True
        
        # 清空训练数据，保留最新数据
        self.training_data = self.training_data[-10:]
    
    def _fallback_prediction(self, response_time, concurrency, success_rate):
        """
        备用预测算法
        """
        # 基于经验的预测
        base_delay = response_time * 0.5
        concurrency_factor = concurrency * 0.2
        success_factor = (1 - success_rate) * 10 if success_rate < 0.8 else 0
        
        return base_delay + concurrency_factor + success_factor

import time
from collections import deque
import statistics

class AdaptiveThrottleAlgorithm:
    """
    自适应限速算法
    """
    
    def __init__(self, window_size=20):
        self.window_size = window_size
        self.response_times = deque(maxlen=window_size)
        self.success_rates = deque(maxlen=window_size)
        self.delays = deque(maxlen=window_size)
        self.error_counts = 0
        self.total_requests = 0
    
    def update_metrics(self, response_time, success, delay_used):
        """
        更新指标
        """
        self.response_times.append(response_time)
        self.success_rates.append(1 if success else 0)
        self.delays.append(delay_used)
        
        self.total_requests += 1
        if not success:
            self.error_counts += 1
    
    def calculate_next_delay(self):
        """
        计算下次延迟
        """
        if len(self.response_times) < 5:
            return 1.0  # 初始延迟
        
        # 计算各项指标
        avg_response_time = statistics.mean(self.response_times)
        current_success_rate = statistics.mean(self.success_rates)
        avg_delay = statistics.mean(self.delays) if self.delays else 1.0
        
        # 基于成功率调整
        success_adjustment = 1.0
        if current_success_rate < 0.7:
            success_adjustment = 2.0  # 成功率低，增加延迟
        elif current_success_rate > 0.95:
            success_adjustment = 0.8  # 成功率高，减少延迟
        
        # 基于响应时间调整
        response_adjustment = min(avg_response_time / 2.0, 5.0)
        
        # 计算最终延迟
        base_delay = avg_response_time * 0.5
        next_delay = base_delay * success_adjustment + response_adjustment
        
        # 应用边界限制
        next_delay = max(0.5, min(next_delay, 30.0))
        
        return next_delay
    
    def get_performance_metrics(self):
        """
        获取性能指标
        """
        if self.total_requests == 0:
            return {}
        
        return {
            'success_rate': (self.total_requests - self.error_counts) / self.total_requests,
            'avg_response_time': statistics.mean(self.response_times) if self.response_times else 0,
            'avg_delay': statistics.mean(self.delays) if self.delays else 0,
            'error_rate': self.error_counts / self.total_requests,
            'requests_per_minute': self.total_requests / (time.time() / 60)
        }

import threading
import time
from collections import defaultdict

class DynamicConcurrencyController:
    """
    动态并发控制器
    """
    
    def __init__(self, max_concurrent=8, min_concurrent=1):
        self.max_concurrent = max_concurrent
        self.min_concurrent = min_concurrent
        self.current_concurrent = min_concurrent
        self.active_requests = defaultdict(int)
        self.response_times = defaultdict(list)
        self.lock = threading.Lock()
        self.adjustment_history = []
    
    def acquire_slot(self, domain):
        """
        获取请求槽位
        """
        with self.lock:
            if self.active_requests[domain] < self.current_concurrent:
                self.active_requests[domain] += 1
                return True
            else:
                return False
    
    def release_slot(self, domain):
        """
        释放请求槽位
        """
        with self.lock:
            if self.active_requests[domain] > 0:
                self.active_requests[domain] -= 1
    
    def adjust_concurrency(self, domain, response_time, success=True):
        """
        调整并发数
        """
        with self.lock:
            # 记录响应时间
            self.response_times[domain].append(response_time)
            if len(self.response_times[domain]) > 10:
                self.response_times[domain] = self.response_times[domain][-10:]
            
            # 计算平均响应时间
            avg_response = sum(self.response_times[domain]) / len(self.response_times[domain])
            
            # 根据响应时间和成功率调整并发数
            if avg_response > 5.0:  # 响应时间过长
                self.current_concurrent = max(
                    self.min_concurrent,
                    self.current_concurrent * 0.8
                )
            elif avg_response < 1.0 and success:  # 响应时间短且成功
                self.current_concurrent = min(
                    self.max_concurrent,
                    self.current_concurrent * 1.1
                )
            
            # 记录调整历史
            adjustment = {
                'timestamp': time.time(),
                'domain': domain,
                'response_time': response_time,
                'new_concurrency': self.current_concurrent,
                'success': success
            }
            self.adjustment_history.append(adjustment)
            
            # 保持历史记录大小
            if len(self.adjustment_history) > 100:
                self.adjustment_history = self.adjustment_history[-100:]
    
    def get_current_concurrency(self):
        """
        获取当前并发数
        """
        with self.lock:
            return self.current_concurrent

class ConcurrencyControlMiddleware:
    """
    并发控制中间件
    """
    
    def __init__(self):
        self.controller = DynamicConcurrencyController()
        self.pending_requests = {}
    
    def process_request(self, request, spider):
        """
        处理请求并发控制
        """
        domain = self._extract_domain(request.url)
        
        # 尝试获取槽位
        if not self.controller.acquire_slot(domain):
            # 槽位不足，延迟请求或排队
            request.meta['queued_time'] = time.time()
            # 这里可以实现请求排队逻辑
            return None
        
        # 记录请求开始时间
        request.meta['request_start_time'] = time.time()
        
        return None
    
    def process_response(self, request, response, spider):
        """
        处理响应，并发控制
        """
        domain = self._extract_domain(request.url)
        
        # 计算响应时间
        if 'request_start_time' in request.meta:
            response_time = time.time() - request.meta['request_start_time']
            
            # 调整并发控制
            success = response.status == 200
            self.controller.adjust_concurrency(domain, response_time, success)
        
        # 释放槽位
        self.controller.release_slot(domain)
        
        return response
    
    def process_exception(self, request, exception, spider):
        """
        处理异常，并发控制
        """
        domain = self._extract_domain(request.url)
        
        # 计算虚拟响应时间
        if 'request_start_time' in request.meta:
            response_time = time.time() - request.meta['request_start_time']
            
            # 以失败处理调整并发
            self.controller.adjust_concurrency(domain, response_time, success=False)
        
        # 释放槽位
        self.controller.release_slot(domain)
    
    def _extract_domain(self, url):
        """
        提取域名
        """
        from urllib.parse import urlparse
        return urlparse(url).netloc

import time
from collections import defaultdict, deque
import statistics

class ResponseTimeMonitor:
    """
    响应时间监测器
    """
    
    def __init__(self, window_size=50):
        self.window_size = window_size
        self.domains = defaultdict(lambda: {
            'response_times': deque(maxlen=window_size),
            'timestamps': deque(maxlen=window_size),
            'success_counts': deque(maxlen=window_size),
            'error_counts': deque(maxlen=window_size)
        })
        self.global_stats = {
            'total_requests': 0,
            'total_success': 0,
            'total_errors': 0,
            'start_time': time.time()
        }
    
    def record_response(self, domain, response_time, success=True):
        """
        记录响应时间
        """
        domain_data = self.domains[domain]
        
        domain_data['response_times'].append(response_time)
        domain_data['timestamps'].append(time.time())
        
        if success:
            domain_data['success_counts'].append(1)
            domain_data['error_counts'].append(0)
            self.global_stats['total_success'] += 1
        else:
            domain_data['success_counts'].append(0)
            domain_data['error_counts'].append(1)
            self.global_stats['total_errors'] += 1
        
        self.global_stats['total_requests'] += 1
    
    def get_domain_stats(self, domain):
        """
        获取域名统计信息
        """
        if domain not in self.domains:
            return None
        
        domain_data = self.domains[domain]
        
        if not domain_data['response_times']:
            return None
        
        response_times = list(domain_data['response_times'])
        success_counts = list(domain_data['success_counts'])
        error_counts = list(domain_data['error_counts'])
        
        return {
            'avg_response_time': statistics.mean(response_times),
            'median_response_time': statistics.median(response_times),
            'min_response_time': min(response_times),
            'max_response_time': max(response_times),
            'std_deviation': statistics.stdev(response_times) if len(response_times) > 1 else 0,
            'success_rate': sum(success_counts) / len(success_counts),
            'error_rate': sum(error_counts) / len(error_counts),
            'sample_size': len(response_times)
        }
    
    def get_trend_analysis(self, domain, lookback_minutes=10):
        """
        获取趋势分析
        """
        if domain not in self.domains:
            return None
        
        domain_data = self.domains[domain]
        timestamps = list(domain_data['timestamps'])
        response_times = list(domain_data['response_times'])
        
        if not timestamps:
            return None
        
        # 计算最近一段时间的趋势
        cutoff_time = time.time() - (lookback_minutes * 60)
        recent_indices = [i for i, t in enumerate(timestamps) if t >= cutoff_time]
        
        if not recent_indices:
            return None
        
        recent_response_times = [response_times[i] for i in recent_indices]
        
        if len(recent_response_times) < 2:
            return {'trend': 'insufficient_data'}
        
        # 简单线性趋势分析
        n = len(recent_response_times)
        x = list(range(n))
        slope = (n * sum(x[i] * recent_response_times[i] for i in range(n)) - 
                sum(x) * sum(recent_response_times)) / (n * sum(i*i for i in x) - sum(x)**2)
        
        trend_direction = 'increasing' if slope > 0.1 else 'decreasing' if slope < -0.1 else 'stable'
        
        return {
            'trend_direction': trend_direction,
            'slope': slope,
            'recent_avg': statistics.mean(recent_response_times),
            'data_points': len(recent_response_times)
        }

class MonitoringMiddleware:
    """
    监测中间件
    """
    
    def __init__(self):
        self.monitor = ResponseTimeMonitor()
        self.anomaly_detector = ResponseAnomalyDetector()
    
    def process_request(self, request, spider):
        """
        记录请求开始时间
        """
        request.meta['monitor_start_time'] = time.time()
        return None
    
    def process_response(self, request, response, spider):
        """
        处理响应，更新监测数据
        """
        if 'monitor_start_time' in request.meta:
            response_time = time.time() - request.meta['monitor_start_time']
            domain = self._extract_domain(request.url)
            
            success = response.status == 200
            self.monitor.record_response(domain, response_time, success)
            
            # 检测异常
            anomaly = self.anomaly_detector.detect_anomaly(
                domain, response_time, success
            )
            
            if anomaly:
                spider.logger.warning(f"Anomaly detected for {domain}: {anomaly}")
        
        return response
    
    def get_performance_report(self):
        """
        获取性能报告
        """
        return {
            'global_stats': self.monitor.global_stats,
            'domain_reports': {
                domain: self.monitor.get_domain_stats(domain)
                for domain in self.monitor.domains.keys()
            }
        }
    
    def _extract_domain(self, url):
        """
        提取域名
        """
        from urllib.parse import urlparse
        return urlparse(url).netloc

class ResponseAnomalyDetector:
    """
    响应异常检测器
    """
    
    def __init__(self):
        self.baseline_data = defaultdict(list)
        self.threshold_multiplier = 3.0  # 异常阈值倍数
    
    def detect_anomaly(self, domain, response_time, success):
        """
        检测响应异常
        """
        # 添加当前数据到基线
        if success:
            self.baseline_data[domain].append(response_time)
            if len(self.baseline_data[domain]) > 50:
                self.baseline_data[domain] = self.baseline_data[domain][-50:]
        
        if len(self.baseline_data[domain]) < 10:
            return None  # 数据不足，无法判断
        
        # 计算基线统计
        baseline_times = self.baseline_data[domain]
        avg_time = statistics.mean(baseline_times)
        std_dev = statistics.stdev(baseline_times)
        
        # 检测异常条件
        anomalies = []
        
        if not success:
            anomalies.append("Request failed")
        
        if response_time > avg_time + (self.threshold_multiplier * std_dev):
            anomalies.append(f"Response time too slow: {response_time:.2f}s (baseline: {avg_time:.2f}s)")
        
        if response_time < avg_time - (self.threshold_multiplier * std_dev) and response_time < 0.1:
            anomalies.append(f"Response time too fast: {response_time:.2f}s (possible bot detection)")
        
        return ", ".join(anomalies) if anomalies else None

import time
import random
from enum import Enum
from collections import defaultdict

class RateLimitStrategy(Enum):
    CONSERVATIVE = "conservative"      # 保守策略
    BALANCED = "balanced"             # 平衡策略
    AGGRESSIVE = "aggressive"         # 激进策略
    ADAPTIVE = "adaptive"             # 自适应策略

class MultiDimensionalThrottle:
    """
    多维度限速策略
    """
    
    def __init__(self, strategy=RateLimitStrategy.BALANCED):
        self.strategy = strategy
        self.domain_configs = defaultdict(dict)
        self.performance_history = defaultdict(list)
        self.current_delays = defaultdict(float)
    
    def get_throttle_config(self, domain):
        """
        根据策略获取限速配置
        """
        base_configs = {
            RateLimitStrategy.CONSERVATIVE: {
                'min_delay': 3.0,
                'max_delay': 60.0,
                'initial_concurrency': 1,
                'aggression_factor': 0.5
            },
            RateLimitStrategy.BALANCED: {
                'min_delay': 1.0,
                'max_delay': 30.0,
                'initial_concurrency': 2,
                'aggression_factor': 1.0
            },
            RateLimitStrategy.AGGRESSIVE: {
                'min_delay': 0.5,
                'max_delay': 10.0,
                'initial_concurrency': 4,
                'aggression_factor': 2.0
            },
            RateLimitStrategy.ADAPTIVE: {
                'min_delay': 1.0,
                'max_delay': 60.0,
                'initial_concurrency': 2,
                'aggression_factor': 1.0
            }
        }
        
        config = base_configs[self.strategy].copy()
        
        # 根据域名特征调整配置
        if 'api' in domain or 'json' in domain:
            # API接口可以更激进
            config['min_delay'] *= 0.5
            config['max_delay'] *= 0.8
            config['aggression_factor'] *= 1.5
        elif any(blocked in domain for blocked in ['weibo', 'zhihu', 'doubao']):
            # 高防护网站需要更保守
            config['min_delay'] *= 2.0
            config['max_delay'] *= 2.0
            config['aggression_factor'] *= 0.5
        
        return config
    
    def calculate_delay(self, domain, response_time, success_rate, error_count):
        """
        计算延迟时间
        """
        config = self.get_throttle_config(domain)
        
        # 基础延迟
        base_delay = config['min_delay']
        
        # 根据响应时间调整
        if response_time > 5.0:
            base_delay += (response_time - 5.0) * 0.5
        
        # 根据成功率调整
        if success_rate < 0.8:
            base_delay *= (1.0 - success_rate) * 3.0
        
        # 根据错误数量调整
        if error_count > 0:
            base_delay *= min(1.0 + error_count * 0.2, 5.0)
        
        # 应用策略因子
        base_delay *= config['aggression_factor']
        
        # 应用随机化（避免规律性）
        random_factor = random.uniform(0.8, 1.2)
        final_delay = base_delay * random_factor
        
        # 应用边界限制
        final_delay = max(config['min_delay'], min(final_delay, config['max_delay']))
        
        # 更新当前延迟
        self.current_delays[domain] = final_delay
        
        return final_delay
    
    def update_performance(self, domain, response_time, success):
        """
        更新性能历史
        """
        self.performance_history[domain].append({
            'timestamp': time.time(),
            'response_time': response_time,
            'success': success
        })
        
        # 保持历史记录大小
        if len(self.performance_history[domain]) > 100:
            self.performance_history[domain] = self.performance_history[domain][-100:]
    
    def get_performance_metrics(self, domain):
        """
        获取性能指标
        """
        if domain not in self.performance_history:
            return None
        
        history = self.performance_history[domain]
        if not history:
            return None
        
        recent = history[-20:]  # 最近20次请求
        
        response_times = [h['response_time'] for h in recent]
        successes = [h['success'] for h in recent]
        
        return {
            'avg_response_time': sum(response_times) / len(response_times),
            'success_rate': sum(successes) / len(successes),
            'error_rate': 1 - (sum(successes) / len(successes)),
            'current_delay': self.current_delays.get(domain, 1.0)
        }

class StrategySwitchingMiddleware:
    """
    策略切换中间件
    """
    
    def __init__(self):
        self.throttler = MultiDimensionalThrottle()
        self.strategy_states = defaultdict(dict)
        self.performance_monitors = {}
    
    def process_request(self, request, spider):
        """
        处理请求，应用限速策略
        """
        domain = self._extract_domain(request.url)
        
        # 获取当前性能指标
        metrics = self.throttler.get_performance_metrics(domain)
        
        if metrics:
            # 根据性能动态调整策略
            self._adjust_strategy_if_needed(domain, metrics)
        
        # 记录请求时间
        request.meta['request_start_time'] = time.time()
        
        return None
    
    def process_response(self, request, response, spider):
        """
        处理响应，更新限速参数
        """
        domain = self._extract_domain(request.url)
        
        # 计算响应时间
        if 'request_start_time' in request.meta:
            response_time = time.time() - request.meta['request_start_time']
            success = response.status == 200
            
            # 更新性能历史
            self.throttler.update_performance(domain, response_time, success)
            
            # 计算新的延迟
            metrics = self.throttler.get_performance_metrics(domain)
            if metrics:
                delay = self.throttler.calculate_delay(
                    domain,
                    metrics['avg_response_time'],
                    metrics['success_rate'],
                    1 - metrics['success_rate']
                )
                
                # 如果延迟较大，记录警告
                if delay > 10:
                    spider.logger.warning(f"High delay applied to {domain}: {delay:.2f}s")
        
        return response
    
    def _adjust_strategy_if_needed(self, domain, metrics):
        """
        根据性能指标调整策略
        """
        current_strategy = self.throttler.strategy
        current_metrics = self.throttler.get_performance_metrics(domain)
        
        if not current_metrics:
            return
        
        # 如果成功率持续很低，切换到保守策略
        if current_metrics['success_rate'] < 0.5:
            if current_strategy != RateLimitStrategy.CONSERVATIVE:
                self.throttler.strategy = RateLimitStrategy.CONSERVATIVE
                self.strategy_states[domain]['changed_at'] = time.time()
                self.strategy_states[domain]['reason'] = 'Low success rate'
        
        # 如果性能良好，可以考虑更激进的策略
        elif (current_metrics['success_rate'] > 0.95 and 
              current_metrics['avg_response_time'] < 2.0):
            if current_strategy == RateLimitStrategy.CONSERVATIVE:
                self.throttler.strategy = RateLimitStrategy.BALANCED
                self.strategy_states[domain]['changed_at'] = time.time()
                self.strategy_states[domain]['reason'] = 'Good performance'
    
    def _extract_domain(self, url):
        """
        提取域名
        """
        from urllib.parse import urlparse
        return urlparse(url).netloc

import time
import threading
from datetime import datetime, timedelta
from collections import defaultdict, deque

class PerformanceDashboard:
    """
    性能监控仪表板
    """
    
    def __init__(self):
        self.metrics = defaultdict(lambda: defaultdict(deque))
        self.alerts = deque(maxlen=100)
        self.running = False
        self.monitor_thread = None
        
        # 监控指标
        self.metric_keys = [
            'response_time', 'success_rate', 'requests_per_minute',
            'concurrency', 'delay_applied', 'error_rate'
        ]
    
    def start_monitoring(self):
        """
        开始监控
        """
        if not self.running:
            self.running = True
            self.monitor_thread = threading.Thread(target=self._monitor_loop, daemon=True)
            self.monitor_thread.start()
    
    def stop_monitoring(self):
        """
        停止监控
        """
        self.running = False
        if self.monitor_thread:
            self.monitor_thread.join()
    
    def _monitor_loop(self):
        """
        监控循环
        """
        while self.running:
            try:
                self._collect_system_metrics()
                time.sleep(5)  # 每5秒收集一次
            except Exception as e:
                print(f"Monitoring error: {e}")
    
    def _collect_system_metrics(self):
        """
        收集系统指标
        """
        timestamp = time.time()
        
        # 这里可以集成系统资源监控
        # CPU使用率、内存使用率等
        pass
    
    def record_metric(self, domain, metric_type, value):
        """
        记录指标
        """
        # 保持最近1000个数据点
        if len(self.metrics[domain][metric_type]) >= 1000:
            self.metrics[domain][metric_type].popleft()
        
        self.metrics[domain][metric_type].append({
            'timestamp': time.time(),
            'value': value
        })
    
    def get_current_status(self):
        """
        获取当前状态
        """
        status = {}
        
        for domain in self.metrics.keys():
            domain_status = {}
            
            for metric_type in self.metric_keys:
                if self.metrics[domain][metric_type]:
                    recent_values = [
                        m['value'] for m in list(self.metrics[domain][metric_type])[-10:]
                    ]
                    
                    if recent_values:
                        domain_status[metric_type] = {
                            'current': recent_values[-1],
                            'average': sum(recent_values) / len(recent_values),
                            'trend': self._calculate_trend(recent_values)
                        }
            
            if domain_status:
                status[domain] = domain_status
        
        return status
    
    def _calculate_trend(self, values):
        """
        计算趋势
        """
        if len(values) < 2:
            return 'insufficient_data'
        
        recent = values[-5:] if len(values) >= 5 else values
        earlier = values[-10:-5] if len(values) >= 10 else values[:len(values)//2]
        
        recent_avg = sum(recent) / len(recent)
        earlier_avg = sum(earlier) / len(earlier) if earlier else recent_avg
        
        if abs(recent_avg - earlier_avg) < 0.01:  # 基本无变化
            return 'stable'
        elif recent_avg > earlier_avg:
            return 'increasing'
        else:
            return 'decreasing'
    
    def get_alerts(self, minutes=10):
        """
        获取近期警报
        """
        cutoff_time = time.time() - (minutes * 60)
        recent_alerts = [
            alert for alert in list(self.alerts)
            if alert['timestamp'] > cutoff_time
        ]
        return recent_alerts
    
    def add_alert(self, domain, message, severity='warning'):
        """
        添加警报
        """
        alert = {
            'timestamp': time.time(),
            'domain': domain,
            'message': message,
            'severity': severity
        }
        self.alerts.append(alert)
    
    def get_recommendations(self):
        """
        获取优化建议
        """
        recommendations = []
        status = self.get_current_status()
        
        for domain, domain_status in status.items():
            if 'error_rate' in domain_status:
                error_rate = domain_status['error_rate']['current']
                if error_rate > 0.3:
                    recommendations.append({
                        'domain': domain,
                        'issue': 'High error rate detected',
                        'recommendation': 'Consider switching to conservative throttling strategy',
                        'severity': 'high'
                    })
            
            if 'response_time' in domain_status:
                response_time = domain_status['response_time']['current']
                if response_time > 10:
                    recommendations.append({
                        'domain': domain,
                        'issue': 'Slow response time',
                        'recommendation': 'Increase delays or reduce concurrency',
                        'severity': 'medium'
                    })
        
        return recommendations

# 配置调整
AUTOTHROTTLE_TARGET_CONCURRENCY = 2.0  # 提高目标并发数
AUTOTHROTTLE_MAX_DELAY = 30  # 限制最大延迟

# 或者使用自定义中间件进行更精细的控制
class EfficiencyFocusedMiddleware:
    def process_request(self, request, spider):
        # 对于API接口，使用较低的延迟
        if 'api' in request.url:
            time.sleep(0.5)
        return None

# 严格配置
AUTOTHROTTLE_START_DELAY = 5
AUTOTHROTTLE_MAX_DELAY = 60
AUTOTHROTTLE_TARGET_CONCURRENCY = 0.5
CONCURRENT_REQUESTS = 1

# 结合代理使用
DOWNLOADER_MIDDLEWARES = {
    'myproject.middlewares.ProxyMiddleware': 350,
    'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 400,
}

class DynamicSiteThrottle:
    def __init__(self):
        self.ajax_delays = {}
    
    def process_request(self, request, spider):
        if self._is_ajax_request(request):
            # AJAX请求使用不同的限速策略
            domain = self._extract_domain(request.url)
            delay = self.ajax_delays.get(domain, 2.0)
            time.sleep(delay)
        return None
    
    def _is_ajax_request(self, request):
        # 检测是否为AJAX请求
        return any(keyword in request.url for keyword in ['ajax', 'api', 'json'])

class DomainSpecificThrottle:
    def __init__(self):
        self.domain_configs = {
            'ecommerce_site.com': {
                'delay_range': (2, 5),
                'concurrency': 1
            },
            'news_site.com': {
                'delay_range': (1, 3),
                'concurrency': 2
            }
        }
    
    def get_domain_config(self, domain):
        return self.domain_configs.get(domain, {
            'delay_range': (1, 3),
            'concurrency': 1
        })