#Scrapy代理IP池集成完全指南 - 动态代理切换与IP封禁规避技术详解

"""
代理IP的工作原理：

客户端 -> 代理服务器 -> 目标服务器
  ↓           ↓           ↓
发送请求 -> 转发请求 -> 返回响应
"""

"""
按协议类型分类：
1. HTTP代理：支持HTTP协议的代理
2. HTTPS代理：支持HTTPS协议的代理
3. SOCKS4代理：SOCKS协议第4版
4. SOCKS5代理：SOCKS协议第5版，支持UDP和认证

按匿名程度分类：
1. 透明代理：目标服务器知道你使用了代理
2. 匿名代理：目标服务器不知道你的真实IP
3. 高匿代理：目标服务器不知道你使用了代理也不知道你的真实IP
"""

"""
代理IP的主要应用场景：
1. 规避IP封禁：绕过目标网站的IP限制
2. 地理位置伪装：获取特定地区的数据
3. 并发请求：使用多个IP同时请求
4. 反爬虫对抗：突破反爬虫限制
5. 数据采集：大规模数据抓取
"""

"""
免费代理特点：
- 优点：成本低，易于获取
- 缺点：稳定性差，存活率低，速度慢，安全性未知
- 适用场景：小规模测试，临时使用

获取渠道：
- 免费代理网站
- 代理API接口
- 代理池服务
"""

"""
付费代理特点：
- 优点：稳定性好，速度快，服务质量高
- 缺点：成本较高，需要持续付费
- 适用场景：商业项目，大规模爬虫

付费代理提供商：
- 芝麻代理
- 快代理
- 代理云
- 各大代理服务商
"""

"""
自建代理特点：
- 优点：完全可控，安全性高，成本可预测
- 缺点：需要技术维护，初期投入大
- 适用场景：长期项目，对安全性要求高的场景

自建代理方案：
- 购买海外VPS
- 部署代理服务
- 维护代理池
"""

import random
import logging

class SimpleProxyMiddleware:
    """
    简单代理中间件
    """
    
    def __init__(self):
        self.proxies = [
            'http://proxy1.com:8080',
            'http://proxy2.com:8080',
            'http://proxy3.com:8080',
        ]
        self.logger = logging.getLogger(__name__)
    
    def process_request(self, request, spider):
        """
        为请求分配代理
        """
        if 'proxy' not in request.meta:
            proxy = random.choice(self.proxies)
            request.meta['proxy'] = proxy
            self.logger.info(f"Assigned proxy {proxy} to {request.url}")
        
        return None

import random
import logging

class ConfigurableProxyMiddleware:
    """
    配置化代理中间件
    """
    
    def __init__(self, proxy_list, retry_times=3):
        self.proxy_list = proxy_list
        self.retry_times = retry_times
        self.logger = logging.getLogger(__name__)
    
    @classmethod
    def from_crawler(cls, crawler):
        """
        从爬虫配置创建中间件
        """
        proxy_list = crawler.settings.getlist('PROXY_LIST', [])
        retry_times = crawler.settings.getint('PROXY_RETRY_TIMES', 3)
        return cls(proxy_list, retry_times)
    
    def process_request(self, request, spider):
        """
        处理请求，分配代理
        """
        # 如果请求已经指定了代理，直接使用
        if request.meta.get('proxy'):
            return None
        
        # 从配置中随机选择代理
        if self.proxy_list:
            proxy = random.choice(self.proxy_list)
            request.meta['proxy'] = proxy
            request.meta['download_timeout'] = 30  # 设置超时时间
            self.logger.info(f"Assigned proxy {proxy} to {request.url}")
        
        return None
    
    def process_response(self, request, response, spider):
        """
        处理响应，检测代理是否有效
        """
        # 如果响应状态异常，可能代理失效
        if response.status in [403, 404, 500]:
            proxy = request.meta.get('proxy')
            if proxy:
                self.logger.warning(f"Proxy {proxy} returned status {response.status}")
        
        return response
    
    def process_exception(self, request, exception, spider):
        """
        处理异常，可能代理失效
        """
        proxy = request.meta.get('proxy')
        if proxy:
            self.logger.error(f"Proxy {proxy} failed with exception: {exception}")
        
        # 如果达到重试次数，尝试更换代理
        retry_times = request.meta.get('proxy_retry_times', 0)
        if retry_times < self.retry_times:
            new_request = request.copy()
            new_request.meta['proxy_retry_times'] = retry_times + 1
            new_request.dont_filter = True
            return new_request
        
        return None

import base64
import random

class AuthenticatedProxyMiddleware:
    """
    支持认证的代理中间件
    """
    
    def __init__(self, proxy_auth_list):
        """
        初始化认证代理列表
        格式: ['username:password@proxy_host:port', ...]
        """
        self.proxy_auth_list = proxy_auth_list
    
    @classmethod
    def from_crawler(cls, crawler):
        proxy_auth_list = crawler.settings.getlist('PROXY_AUTH_LIST', [])
        return cls(proxy_auth_list)
    
    def process_request(self, request, spider):
        """
        处理带认证的代理请求
        """
        if not self.proxy_auth_list:
            return None
        
        # 随机选择一个认证代理
        proxy_with_auth = random.choice(self.proxy_auth_list)
        
        # 解析用户名密码
        if '@' in proxy_with_auth:
            auth_and_proxy = proxy_with_auth.split('@')
            auth = auth_and_proxy[0]
            proxy = auth_and_proxy[1]
            
            # 创建认证头部
            credentials = base64.b64encode(auth.encode()).decode()
            request.headers['Proxy-Authorization'] = f'Basic {credentials}'
            
            # 设置代理
            request.meta['proxy'] = f'http://{proxy}'
        else:
            # 如果没有认证信息，直接使用代理
            request.meta['proxy'] = f'http://{proxy_with_auth}'
        
        return None

import redis
import json
import time
from collections import defaultdict

class RedisProxyPoolManager:
    """
    基于Redis的代理池管理器
    """
    
    def __init__(self, redis_host='localhost', redis_port=6379, redis_db=0):
        self.redis_client = redis.Redis(
            host=redis_host,
            port=redis_port,
            db=redis_db,
            decode_responses=True
        )
        
        # Redis键名定义
        self.pool_key = 'proxy_pool:available'
        self.bad_key = 'proxy_pool:bad'
        self.stats_key = 'proxy_pool:stats'
        self.test_key = 'proxy_pool:test_results'
    
    def add_proxy(self, proxy, proxy_type='http', anonymous_level='high'):
        """
        添加代理到池中
        """
        proxy_info = {
            'proxy': proxy,
            'type': proxy_type,
            'anonymous_level': anonymous_level,
            'added_time': time.time(),
            'last_tested': 0,
            'success_count': 0,
            'failure_count': 0,
            'score': 100  # 初始分数
        }
        
        # 添加到可用代理池
        self.redis_client.zadd(self.pool_key, {json.dumps(proxy_info): proxy_info['score']})
    
    def get_proxy(self):
        """
        获取一个可用代理
        """
        # 获取分数最高的代理
        proxies = self.redis_client.zrevrange(self.pool_key, 0, 0, withscores=True)
        
        if proxies:
            proxy_info = json.loads(proxies[0][0])
            score = int(proxies[0][1])
            
            # 更新使用次数
            proxy_info['last_used'] = time.time()
            self.redis_client.zrem(self.pool_key, json.dumps(proxy_info))
            self.redis_client.zadd(self.pool_key, {json.dumps(proxy_info): score})
            
            return proxy_info['proxy']
        
        return None
    
    def mark_proxy_good(self, proxy):
        """
        标记代理为好用
        """
        self._update_proxy_score(proxy, 5)  # 增加分数
    
    def mark_proxy_bad(self, proxy, reason='unknown'):
        """
        标记代理为不可用
        """
        # 从可用池移到坏代理池
        self._remove_proxy_from_pool(proxy, self.pool_key)
        bad_proxy_info = {
            'proxy': proxy,
            'reason': reason,
            'marked_time': time.time()
        }
        self.redis_client.lpush(self.bad_key, json.dumps(bad_proxy_info))
        
        self._update_proxy_score(proxy, -20)  # 降低分数
    
    def _update_proxy_score(self, proxy, delta_score):
        """
        更新代理分数
        """
        # 查找代理信息
        all_proxies = self.redis_client.zrange(self.pool_key, 0, -1, withscores=True)
        for proxy_str, score in all_proxies:
            proxy_info = json.loads(proxy_str)
            if proxy_info['proxy'] == proxy:
                new_score = max(0, min(100, score + delta_score))
                self.redis_client.zrem(self.pool_key, proxy_str)
                proxy_info['score'] = new_score
                self.redis_client.zadd(self.pool_key, {json.dumps(proxy_info): new_score})
                break
    
    def _remove_proxy_from_pool(self, proxy, pool_key):
        """
        从代理池中移除指定代理
        """
        all_proxies = self.redis_client.zrange(pool_key, 0, -1)
        for proxy_str in all_proxies:
            proxy_info = json.loads(proxy_str)
            if proxy_info['proxy'] == proxy:
                self.redis_client.zrem(pool_key, proxy_str)
                break
    
    def get_pool_stats(self):
        """
        获取代理池统计信息
        """
        available_count = self.redis_client.zcard(self.pool_key)
        bad_count = self.redis_client.llen(self.bad_key)
        
        return {
            'available_count': available_count,
            'bad_count': bad_count,
            'total_count': available_count + bad_count
        }

import time
import random
from scrapy.exceptions import IgnoreRequest

class AdvancedProxyPoolMiddleware:
    """
    高级代理池中间件
    """
    
    def __init__(self, proxy_manager):
        self.proxy_manager = proxy_manager
        self.failed_requests = {}  # 记录失败请求
        self.request_attempts = {}  # 记录请求尝试次数
    
    @classmethod
    def from_crawler(cls, crawler):
        # 从配置创建代理管理器
        redis_host = crawler.settings.get('REDIS_HOST', 'localhost')
        redis_port = crawler.settings.getint('REDIS_PORT', 6379)
        redis_db = crawler.settings.getint('REDIS_DB', 0)
        
        proxy_manager = RedisProxyPoolManager(redis_host, redis_port, redis_db)
        return cls(proxy_manager)
    
    def process_request(self, request, spider):
        """
        为请求分配代理
        """
        # 如果请求已经指定了代理，直接使用
        if request.meta.get('proxy'):
            return None
        
        # 获取代理
        proxy = self.proxy_manager.get_proxy()
        if proxy:
            request.meta['proxy'] = proxy
            request.meta['proxy_assigned_time'] = time.time()
            
            # 记录请求尝试次数
            request_fingerprint = self._get_request_fingerprint(request)
            self.request_attempts[request_fingerprint] = self.request_attempts.get(request_fingerprint, 0) + 1
            
            spider.logger.info(f"Assigned proxy {proxy} to {request.url}")
        else:
            spider.logger.warning(f"No available proxy for {request.url}")
            # 没有可用代理时的处理策略
            max_attempts = spider.crawler.settings.getint('MAX_PROXY_ATTEMPTS', 3)
            current_attempts = self.request_attempts.get(request_fingerprint, 1)
            
            if current_attempts <= max_attempts:
                # 短暂延迟后重试
                import time
                time.sleep(random.uniform(1, 3))
                new_request = request.copy()
                new_request.dont_filter = True
                return new_request
            else:
                # 达到最大尝试次数，忽略请求
                raise IgnoreRequest("No available proxy after maximum attempts")
        
        return None
    
    def process_response(self, request, response, spider):
        """
        处理响应，更新代理状态
        """
        proxy = request.meta.get('proxy')
        if proxy:
            if response.status == 200:
                # 请求成功，标记代理为好用
                self.proxy_manager.mark_proxy_good(proxy)
                spider.logger.debug(f"Proxy {proxy} marked as good for {request.url}")
            elif response.status in [403, 404, 429, 503]:
                # 请求失败，标记代理为不可用
                reason = f"Status {response.status}"
                self.proxy_manager.mark_proxy_bad(proxy, reason)
                spider.logger.warning(f"Proxy {proxy} marked as bad: {reason}")
        
        return response
    
    def process_exception(self, request, exception, spider):
        """
        处理异常，更新代理状态
        """
        proxy = request.meta.get('proxy')
        if proxy:
            reason = str(exception)
            self.proxy_manager.mark_proxy_bad(proxy, reason)
            spider.logger.error(f"Proxy {proxy} failed with exception: {reason}")
        
        # 检查是否需要重试
        request_fingerprint = self._get_request_fingerprint(request)
        current_attempts = self.request_attempts.get(request_fingerprint, 1)
        max_attempts = spider.crawler.settings.getint('MAX_REQUEST_ATTEMPTS', 3)
        
        if current_attempts < max_attempts:
            # 尝试使用新代理重试
            new_request = request.copy()
            new_request.meta.pop('proxy', None)  # 移除旧代理
            new_request.dont_filter = True
            return new_request
        
        return None
    
    def _get_request_fingerprint(self, request):
        """
        获取请求指纹，用于跟踪请求尝试次数
        """
        return request.url + str(hash(request.body) if request.body else 0)

import requests
import time
import socket
from urllib.parse import urlparse

class ProxyQualityChecker:
    """
    代理质量检测器
    """
    
    def __init__(self, test_url='http://httpbin.org/ip', timeout=10):
        self.test_url = test_url
        self.timeout = timeout
        self.test_headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
        }
    
    def check_proxy_quality(self, proxy):
        """
        检测代理质量
        """
        proxy_dict = {
            'http': proxy,
            'https': proxy
        }
        
        start_time = time.time()
        result = {
            'proxy': proxy,
            'success': False,
            'response_time': float('inf'),
            'anonymity': 'transparent',  # transparent, anonymous, elite
            'country': None,
            'isp': None,
            'error': None
        }
        
        try:
            # 发送测试请求
            response = requests.get(
                self.test_url,
                proxies=proxy_dict,
                headers=self.test_headers,
                timeout=self.timeout
            )
            
            response_time = time.time() - start_time
            
            if response.status_code == 200:
                result['success'] = True
                result['response_time'] = response_time
                
                # 检测匿名程度
                result['anonymity'] = self._detect_anonymity(proxy, response)
                
                # 解析响应内容获取更多信息
                try:
                    import json
                    ip_info = response.json()
                    if 'origin' in ip_info:
                        result['ip_address'] = ip_info['origin']
                except:
                    pass
            else:
                result['error'] = f"HTTP {response.status_code}"
                
        except requests.exceptions.Timeout:
            result['error'] = "Timeout"
        except requests.exceptions.ConnectionError:
            result['error'] = "Connection Error"
        except Exception as e:
            result['error'] = str(e)
        
        return result
    
    def _detect_anonymity(self, proxy, response):
        """
        检测代理匿名程度
        """
        try:
            # 这里可以实现更复杂的匿名度检测逻辑
            # 通过比较请求头等信息来判断匿名程度
            return 'elite'  # 假设都是高匿代理
        except:
            return 'transparent'
    
    def batch_check_proxies(self, proxies, max_workers=10):
        """
        批量检测代理质量
        """
        from concurrent.futures import ThreadPoolExecutor, as_completed
        
        results = []
        
        with ThreadPoolExecutor(max_workers=max_workers) as executor:
            future_to_proxy = {
                executor.submit(self.check_proxy_quality, proxy): proxy 
                for proxy in proxies
            }
            
            for future in as_completed(future_to_proxy):
                result = future.result()
                results.append(result)
        
        return results

import time
from datetime import datetime, timedelta

class ProxyHealthCheckMiddleware:
    """
    代理健康检查中间件
    """
    
    def __init__(self):
        self.proxy_health = {}  # 代理健康状态
        self.last_check_time = {}  # 最后检查时间
        self.health_check_interval = 300  # 5分钟检查一次
        self.min_success_rate = 0.7  # 最小成功率
        self.min_response_time = 5.0  # 最大响应时间(秒)
    
    def process_request(self, request, spider):
        """
        检查代理健康状态
        """
        proxy = request.meta.get('proxy')
        if proxy:
            # 检查是否需要重新检测代理健康状态
            current_time = time.time()
            last_check = self.last_check_time.get(proxy, 0)
            
            if current_time - last_check > self.health_check_interval:
                self._check_proxy_health(proxy, spider)
                self.last_check_time[proxy] = current_time
        
        return None
    
    def process_response(self, request, response, spider):
        """
        更新代理健康状态
        """
        proxy = request.meta.get('proxy')
        if proxy:
            if proxy not in self.proxy_health:
                self.proxy_health[proxy] = {
                    'success_count': 0,
                    'failure_count': 0,
                    'total_requests': 0,
                    'avg_response_time': 0,
                    'last_response_time': 0,
                    'health_score': 100
                }
            
            health = self.proxy_health[proxy]
            health['total_requests'] += 1
            
            if response.status == 200:
                health['success_count'] += 1
                # 计算响应时间
                request_time = time.time() - request.meta.get('request_start_time', time.time())
                health['last_response_time'] = request_time
                
                # 更新平均响应时间
                old_avg = health['avg_response_time']
                total_req = health['total_requests']
                health['avg_response_time'] = ((old_avg * (total_req - 1) + request_time) / total_req)
            else:
                health['failure_count'] += 1
        
        return response
    
    def _check_proxy_health(self, proxy, spider):
        """
        检查代理健康状态
        """
        if proxy in self.proxy_health:
            health = self.proxy_health[proxy]
            success_rate = health['success_count'] / max(1, health['total_requests'])
            
            # 计算健康分数
            score = 100
            score -= (1 - success_rate) * 50  # 成功率影响50分
            score -= min(health['avg_response_time'] / self.min_response_time * 30, 30)  # 响应时间影响30分
            score -= health['failure_count'] * 2  # 失败次数影响2分/次
            
            health['health_score'] = max(0, min(100, score))
            
            spider.logger.debug(f"Proxy {proxy} health score: {health['health_score']}")
            
            # 如果健康分数过低，可以考虑暂时停用代理
            if health['health_score'] < 30:
                spider.logger.warning(f"Proxy {proxy} health score too low: {health['health_score']}")
    
    def get_healthy_proxies(self):
        """
        获取健康代理列表
        """
        healthy_proxies = []
        for proxy, health in self.proxy_health.items():
            if health['health_score'] >= 50:  # 健康分数大于50的代理
                healthy_proxies.append(proxy)
        return healthy_proxies

import random
import time
from collections import defaultdict, deque

class SmartProxySwitchMiddleware:
    """
    智能代理切换中间件
    """
    
    def __init__(self):
        self.proxy_stats = defaultdict(lambda: {
            'success_count': 0,
            'failure_count': 0,
            'total_requests': 0,
            'consecutive_failures': 0,
            'last_used': 0,
            'score': 100,
            'response_times': deque(maxlen=10)  # 最近10次响应时间
        })
        
        self.switch_threshold = 3  # 连续失败次数阈值
        self.min_proxy_score = 30  # 最低代理分数
        self.proxy_selection_strategy = 'weighted_random'  # 选择策略
    
    def process_request(self, request, spider):
        """
        智能选择代理
        """
        # 获取可用代理
        available_proxies = self._get_available_proxies()
        
        if not available_proxies:
            spider.logger.warning("No available proxies")
            return None
        
        # 根据策略选择代理
        selected_proxy = self._select_proxy(available_proxies)
        
        if selected_proxy:
            request.meta['proxy'] = selected_proxy
            self.proxy_stats[selected_proxy]['last_used'] = time.time()
            self.proxy_stats[selected_proxy]['total_requests'] += 1
            
            # 记录请求开始时间，用于计算响应时间
            request.meta['request_start_time'] = time.time()
        
        return None
    
    def process_response(self, request, response, spider):
        """
        处理成功响应，更新代理状态
        """
        proxy = request.meta.get('proxy')
        if proxy:
            stats = self.proxy_stats[proxy]
            stats['success_count'] += 1
            stats['consecutive_failures'] = 0  # 重置连续失败次数
            
            # 记录响应时间
            if 'request_start_time' in request.meta:
                response_time = time.time() - request.meta['request_start_time']
                stats['response_times'].append(response_time)
            
            # 更新代理分数
            self._update_proxy_score(proxy)
        
        return response
    
    def process_exception(self, request, exception, spider):
        """
        处理异常，更新代理状态
        """
        proxy = request.meta.get('proxy')
        if proxy:
            stats = self.proxy_stats[proxy]
            stats['failure_count'] += 1
            stats['consecutive_failures'] += 1
            
            # 如果连续失败次数超过阈值，降低代理分数
            if stats['consecutive_failures'] >= self.switch_threshold:
                self._penalize_proxy(proxy, spider)
        
        return None
    
    def _get_available_proxies(self):
        """
        获取可用代理列表
        """
        available = []
        current_time = time.time()
        
        for proxy, stats in self.proxy_stats.items():
            # 检查代理分数是否足够高
            if stats['score'] >= self.min_proxy_score:
                available.append(proxy)
        
        return available
    
    def _select_proxy(self, available_proxies):
        """
        根据策略选择代理
        """
        if not available_proxies:
            return None
        
        if self.proxy_selection_strategy == 'weighted_random':
            return self._weighted_random_selection(available_proxies)
        elif self.proxy_selection_strategy == 'round_robin':
            return self._round_robin_selection(available_proxies)
        elif self.proxy_selection_strategy == 'best_score':
            return self._best_score_selection(available_proxies)
        else:
            return random.choice(available_proxies)
    
    def _weighted_random_selection(self, available_proxies):
        """
        加权随机选择
        """
        scores = [self.proxy_stats[proxy]['score'] for proxy in available_proxies]
        total_score = sum(scores)
        
        if total_score <= 0:
            return random.choice(available_proxies)
        
        # 归一化权重
        weights = [score / total_score for score in scores]
        
        # 加权随机选择
        import random
        return random.choices(available_proxies, weights=weights)[0]
    
    def _round_robin_selection(self, available_proxies):
        """
        轮询选择
        """
        # 简单实现：选择最少使用的代理
        min_last_used = min(
            self.proxy_stats[proxy]['last_used'] for proxy in available_proxies
        )
        
        candidates = [
            proxy for proxy in available_proxies
            if self.proxy_stats[proxy]['last_used'] == min_last_used
        ]
        
        return random.choice(candidates)
    
    def _best_score_selection(self, available_proxies):
        """
        选择分数最高的代理
        """
        best_proxy = max(available_proxies, key=lambda p: self.proxy_stats[p]['score'])
        return best_proxy
    
    def _update_proxy_score(self, proxy):
        """
        更新代理分数
        """
        stats = self.proxy_stats[proxy]
        
        # 基于成功率计算分数
        success_rate = stats['success_count'] / max(1, stats['total_requests'])
        success_score = success_rate * 60  # 成功率占60分
        
        # 基于响应时间计算分数（越快越好）
        avg_response_time = sum(stats['response_times']) / max(1, len(stats['response_times']))
        time_score = max(0, 40 - (avg_response_time * 10))  # 响应时间占40分
        
        # 基于连续失败惩罚
        failure_penalty = min(stats['consecutive_failures'] * 10, 50)
        
        new_score = max(0, success_score + time_score - failure_penalty)
        stats['score'] = new_score
    
    def _penalize_proxy(self, proxy, spider):
        """
        惩罚表现差的代理
        """
        self.proxy_stats[proxy]['score'] = max(0, self.proxy_stats[proxy]['score'] - 30)
        spider.logger.warning(f"Penalized proxy {proxy}, new score: {self.proxy_stats[proxy]['score']}")

import base64
import hmac
import hashlib
import time

class AuthenticatedProxyManager:
    """
    认证代理管理器
    """
    
    def __init__(self, username, password, api_key=None):
        self.username = username
        self.password = password
        self.api_key = api_key
    
    def create_authenticated_proxy_url(self, host, port, method='basic'):
        """
        创建认证代理URL
        """
        if method == 'basic':
            # 基本认证
            credentials = f"{self.username}:{self.password}"
            encoded_credentials = base64.b64encode(credentials.encode()).decode()
            return f"http://{host}:{port}", {'Proxy-Authorization': f'Basic {encoded_credentials}'}
        elif method == 'digest':
            # 摘要认证（简化版）
            return self._create_digest_auth_proxy(host, port)
        elif method == 'apikey':
            # API密钥认证
            headers = {'Proxy-Key': self.api_key}
            return f"http://{host}:{port}", headers
        else:
            return f"http://{host}:{port}", {}
    
    def _create_digest_auth_proxy(self, host, port):
        """
        创建摘要认证代理（简化实现）
        """
        # 实际的摘要认证需要更复杂的实现
        # 这里提供基本框架
        headers = {
            'Proxy-Authorization': f'Digest username="{self.username}"'
        }
        return f"http://{host}:{port}", headers

class AdvancedAuthProxyMiddleware:
    """
    高级认证代理中间件
    """
    
    def __init__(self):
        self.auth_manager = None
        self.proxy_configs = {}
    
    @classmethod
    def from_crawler(cls, crawler):
        # 从配置中读取认证信息
        username = crawler.settings.get('PROXY_USERNAME')
        password = crawler.settings.get('PROXY_PASSWORD')
        api_key = crawler.settings.get('PROXY_API_KEY')
        
        instance = cls()
        if username and password:
            instance.auth_manager = AuthenticatedProxyManager(username, password, api_key)
        
        # 读取代理配置
        instance.proxy_configs = crawler.settings.get('PROXY_CONFIGS', {})
        
        return instance
    
    def process_request(self, request, spider):
        """
        处理认证代理请求
        """
        if not self.auth_manager:
            return None
        
        # 获取需要认证的代理
        proxy = request.meta.get('proxy')
        if not proxy:
            # 如果没有指定代理，从配置中选择一个
            proxy_config = self._select_proxy_config()
            if proxy_config:
                host = proxy_config['host']
                port = proxy_config['port']
                auth_method = proxy_config.get('auth_method', 'basic')
                
                proxy_url, headers = self.auth_manager.create_authenticated_proxy_url(
                    host, port, auth_method
                )
                
                request.meta['proxy'] = proxy_url
                # 添加认证头部
                for header, value in headers.items():
                    request.headers[header] = value
        
        return None
    
    def _select_proxy_config(self):
        """
        选择代理配置
        """
        if not self.proxy_configs:
            return None
        
        # 简单的轮询选择
        import random
        configs = list(self.proxy_configs.values())
        return random.choice(configs) if configs else None

import random
import time
from enum import Enum
from collections import OrderedDict

class RotationStrategy(Enum):
    RANDOM = "random"
    ROUND_ROBIN = "round_robin"
    WEIGHTED = "weighted"
    LEAST_USED = "least_used"
    FASTEST = "fastest"

class ProxyRotationManager:
    """
    代理轮换管理器
    """
    
    def __init__(self, proxies, strategy=RotationStrategy.RANDOM):
        self.proxies = proxies
        self.strategy = strategy
        self.proxy_usage_count = {proxy: 0 for proxy in proxies}
        self.proxy_response_times = {proxy: [] for proxy in proxies}
        self.current_index = 0
        self.last_used_time = {proxy: 0 for proxy in proxies}
    
    def select_proxy(self):
        """
        根据策略选择代理
        """
        if not self.proxies:
            return None
        
        if self.strategy == RotationStrategy.RANDOM:
            return self._random_selection()
        elif self.strategy == RotationStrategy.ROUND_ROBIN:
            return self._round_robin_selection()
        elif self.strategy == RotationStrategy.WEIGHTED:
            return self._weighted_selection()
        elif self.strategy == RotationStrategy.LEAST_USED:
            return self._least_used_selection()
        elif self.strategy == RotationStrategy.FASTEST:
            return self._fastest_selection()
        else:
            return random.choice(self.proxies)
    
    def _random_selection(self):
        """
        随机选择
        """
        return random.choice(self.proxies)
    
    def _round_robin_selection(self):
        """
        轮询选择
        """
        proxy = self.proxies[self.current_index % len(self.proxies)]
        self.current_index += 1
        return proxy
    
    def _weighted_selection(self):
        """
        加权选择（基于成功率）
        """
        # 这里使用简化的权重计算，实际应用中可能需要更复杂的算法
        weights = []
        for proxy in self.proxies:
            # 基于响应时间的倒数作为权重（响应时间越短，权重越高）
            times = self.proxy_response_times[proxy]
            if times:
                avg_time = sum(times) / len(times)
                weight = 1.0 / max(avg_time, 0.1)  # 避免除零
            else:
                weight = 1.0
            weights.append(weight)
        
        total_weight = sum(weights)
        if total_weight <= 0:
            return random.choice(self.proxies)
        
        # 归一化权重
        normalized_weights = [w / total_weight for w in weights]
        return random.choices(self.proxies, weights=normalized_weights)[0]
    
    def _least_used_selection(self):
        """
        选择使用次数最少的代理
        """
        min_usage = min(self.proxy_usage_count.values())
        least_used_proxies = [
            proxy for proxy, count in self.proxy_usage_count.items()
            if count == min_usage
        ]
        return random.choice(least_used_proxies)
    
    def _fastest_selection(self):
        """
        选择平均响应时间最短的代理
        """
        fastest_time = float('inf')
        fastest_proxy = None
        
        for proxy, times in self.proxy_response_times.items():
            if times:
                avg_time = sum(times) / len(times)
                if avg_time < fastest_time:
                    fastest_time = avg_time
                    fastest_proxy = proxy
        
        return fastest_proxy or random.choice(self.proxies)
    
    def record_usage(self, proxy, response_time=None):
        """
        记录代理使用情况
        """
        if proxy in self.proxy_usage_count:
            self.proxy_usage_count[proxy] += 1
            self.last_used_time[proxy] = time.time()
            
            if response_time is not None and proxy in self.proxy_response_times:
                # 保持最近10次响应时间记录
                times = self.proxy_response_times[proxy]
                times.append(response_time)
                if len(times) > 10:
                    times.pop(0)  # 移除最旧的记录

class StrategyBasedProxyMiddleware:
    """
    基于策略的代理中间件
    """
    
    def __init__(self, rotation_manager):
        self.rotation_manager = rotation_manager
    
    @classmethod
    def from_crawler(cls, crawler):
        # 从配置读取代理列表和策略
        proxies = crawler.settings.getlist('ROTATION_PROXIES', [])
        strategy_name = crawler.settings.get('ROTATION_STRATEGY', 'random')
        
        try:
            strategy = RotationStrategy(strategy_name.lower())
        except ValueError:
            strategy = RotationStrategy.RANDOM
        
        rotation_manager = ProxyRotationManager(proxies, strategy)
        return cls(rotation_manager)
    
    def process_request(self, request, spider):
        """
        使用轮换策略选择代理
        """
        if 'proxy' not in request.meta and self.rotation_manager.proxies:
            proxy = self.rotation_manager.select_proxy()
            if proxy:
                request.meta['proxy'] = proxy
                request.meta['selection_time'] = time.time()
        
        return None
    
    def process_response(self, request, response, spider):
        """
        记录代理使用情况和响应时间
        """
        proxy = request.meta.get('proxy')
        if proxy and 'selection_time' in request.meta:
            response_time = time.time() - request.meta['selection_time']
            self.rotation_manager.record_usage(proxy, response_time)
        
        return response

import time
import threading
from datetime import datetime, timedelta
from collections import defaultdict
import logging

class ProxyPoolMonitor:
    """
    代理池监控系统
    """
    
    def __init__(self, proxy_manager, check_interval=300):  # 5分钟检查一次
        self.proxy_manager = proxy_manager
        self.check_interval = check_interval
        self.monitoring = False
        self.monitor_thread = None
        self.logger = logging.getLogger(__name__)
        
        # 监控统计数据
        self.stats = {
            'total_checked': 0,
            'good_proxies': 0,
            'bad_proxies': 0,
            'average_response_time': 0,
            'success_rate': 0
        }
    
    def start_monitoring(self):
        """
        开始监控
        """
        if not self.monitoring:
            self.monitoring = True
            self.monitor_thread = threading.Thread(target=self._monitor_loop, daemon=True)
            self.monitor_thread.start()
            self.logger.info("Proxy pool monitoring started")
    
    def stop_monitoring(self):
        """
        停止监控
        """
        self.monitoring = False
        if self.monitor_thread:
            self.monitor_thread.join()
        self.logger.info("Proxy pool monitoring stopped")
    
    def _monitor_loop(self):
        """
        监控循环
        """
        while self.monitoring:
            try:
                self._perform_health_check()
                time.sleep(self.check_interval)
            except Exception as e:
                self.logger.error(f"Error in monitoring loop: {e}")
                time.sleep(60)  # 出错后等待1分钟再继续
    
    def _perform_health_check(self):
        """
        执行健康检查
        """
        self.logger.info("Starting proxy pool health check...")
        
        # 这里可以集成前面的代理质量检测器
        # 暂时使用简单的检查逻辑
        pool_stats = self.proxy_manager.get_pool_stats()
        
        self.stats['total_checked'] += pool_stats['available_count']
        self.stats['good_proxies'] = pool_stats['available_count']
        self.stats['bad_proxies'] = pool_stats['bad_count']
        
        self.logger.info(f"Health check completed. Available: {pool_stats['available_count']}, Bad: {pool_stats['bad_count']}")
    
    def get_monitoring_report(self):
        """
        获取监控报告
        """
        return {
            'timestamp': datetime.now().isoformat(),
            'stats': self.stats.copy(),
            'pool_status': self.proxy_manager.get_pool_stats()
        }

class MonitoringProxyMiddleware:
    """
    监控代理中间件
    """
    
    def __init__(self):
        self.request_stats = defaultdict(int)
        self.response_stats = defaultdict(int)
        self.error_stats = defaultdict(int)
        self.start_time = time.time()
    
    def process_request(self, request, spider):
        """
        记录请求统计
        """
        self.request_stats['total'] += 1
        proxy = request.meta.get('proxy')
        if proxy:
            self.request_stats['with_proxy'] += 1
        else:
            self.request_stats['without_proxy'] += 1
        
        return None
    
    def process_response(self, request, response, spider):
        """
        记录响应统计
        """
        self.response_stats['total'] += 1
        self.response_stats[f"status_{response.status}"] += 1
        
        proxy = request.meta.get('proxy')
        if proxy and response.status == 200:
            self.response_stats['success_with_proxy'] += 1
        
        return response
    
    def process_exception(self, request, exception, spider):
        """
        记录异常统计
        """
        self.error_stats[type(exception).__name__] += 1
        proxy = request.meta.get('proxy')
        if proxy:
            self.error_stats['errors_with_proxy'] += 1
        else:
            self.error_stats['errors_without_proxy'] += 1
    
    def get_statistics(self):
        """
        获取统计信息
        """
        total_time = time.time() - self.start_time
        return {
            'uptime': total_time,
            'requests': dict(self.request_stats),
            'responses': dict(self.response_stats),
            'errors': dict(self.error_stats),
            'requests_per_second': self.request_stats['total'] / max(total_time, 1)
        }

import re
from urllib.parse import urlparse

class IntelligentProxyRouter:
    """
    智能代理路由器
    根据目标网站特征选择合适的代理
    """
    
    def __init__(self):
        self.proxy_rules = {
            'geolocation': {},  # 地理位置规则
            'antibot': {},      # 反爬规则
            'performance': {}   # 性能规则
        }
        
        self.domain_proxy_mapping = {}  # 域名到代理的映射
    
    def add_routing_rule(self, domain_pattern, proxy_list, rule_type='geolocation'):
        """
        添加路由规则
        """
        if rule_type not in self.proxy_rules:
            self.proxy_rules[rule_type] = {}
        
        self.proxy_rules[rule_type][domain_pattern] = proxy_list
    
    def route_proxy(self, url, available_proxies):
        """
        为URL路由合适的代理
        """
        parsed = urlparse(url)
        domain = parsed.netloc.lower()
        
        # 检查是否有针对此域名的特定规则
        for pattern, proxy_list in self.domain_proxy_mapping.items():
            if re.search(pattern, domain):
                # 返回符合规则的代理
                for proxy in proxy_list:
                    if proxy in available_proxies:
                        return proxy
        
        # 根据域名特征选择代理
        if self._is_antibot_heavy_site(domain):
            return self._select_high_anonymity_proxy(available_proxies)
        elif self._requires_geographic_access(domain):
            return self._select_location_specific_proxy(domain, available_proxies)
        else:
            return self._select_standard_proxy(available_proxies)
    
    def _is_antibot_heavy_site(self, domain):
        """
        检查是否为反爬严格的网站
        """
        antibot_indicators = [
            'captcha', 'verify', 'shield', 'protect', 'security'
        ]
        return any(indicator in domain for indicator in antibot_indicators)
    
    def _requires_geographic_access(self, domain):
        """
        检查是否需要地理位置访问
        """
        geo_indicators = [
            'gov', 'edu', 'local', 'region', 'country'
        ]
        return any(indicator in domain for indicator in geo_indicators)
    
    def _select_high_anonymity_proxy(self, available_proxies):
        """
        选择高匿名代理
        """
        # 这里需要代理质量检测的结果
        # 简化实现：返回第一个代理
        return available_proxies[0] if available_proxies else None
    
    def _select_location_specific_proxy(self, domain, available_proxies):
        """
        选择位置特定代理
        """
        return available_proxies[0] if available_proxies else None
    
    def _select_standard_proxy(self, available_proxies):
        """
        选择标准代理
        """
        return available_proxies[0] if available_proxies else None

class RoutingProxyMiddleware:
    """
    路由代理中间件
    """
    
    def __init__(self, router):
        self.router = router
    
    @classmethod
    def from_crawler(cls, crawler):
        router = IntelligentProxyRouter()
        
        # 从配置加载路由规则
        routing_rules = crawler.settings.get('PROXY_ROUTING_RULES', {})
        for domain_pattern, proxy_list in routing_rules.items():
            router.add_routing_rule(domain_pattern, proxy_list)
        
        return cls(router)
    
    def process_request(self, request, spider):
        """
        根据URL路由代理
        """
        if 'proxy' not in request.meta:
            # 获取可用代理（这里简化，实际需要从代理池获取）
            available_proxies = [  # 模拟可用代理列表
                'http://proxy1:8080',
                'http://proxy2:8080',
                'http://proxy3:8080'
            ]
            
            selected_proxy = self.router.route_proxy(request.url, available_proxies)
            if selected_proxy:
                request.meta['proxy'] = selected_proxy
        
        return None

import asyncio
import aiohttp
from concurrent.futures import ThreadPoolExecutor
import time
from functools import lru_cache

class OptimizedProxyManager:
    """
    性能优化的代理管理器
    """
    
    def __init__(self):
        self.proxy_cache = {}  # 代理缓存
        self.testing_semaphore = asyncio.Semaphore(10)  # 限制并发测试数
        self.executor = ThreadPoolExecutor(max_workers=20)
    
    @lru_cache(maxsize=1000)
    def get_cached_proxy(self, domain):
        """
        获取缓存的代理（使用LRU缓存）
        """
        # 这里可以实现基于域名的代理缓存逻辑
        return None
    
    async def batch_test_proxies_async(self, proxies, test_url='http://httpbin.org/ip'):
        """
        异步批量测试代理
        """
        async with aiohttp.ClientSession() as session:
            tasks = [self._test_single_proxy_async(session, proxy, test_url) for proxy in proxies]
            results = await asyncio.gather(*tasks, return_exceptions=True)
            return results
    
    async def _test_single_proxy_async(self, session, proxy, test_url):
        """
        异步测试单个代理
        """
        async with self.testing_semaphore:
            try:
                start_time = time.time()
                async with session.get(
                    test_url,
                    proxy=proxy,
                    timeout=aiohttp.ClientTimeout(total=10)
                ) as response:
                    response_time = time.time() - start_time
                    return {
                        'proxy': proxy,
                        'success': response.status == 200,
                        'response_time': response_time,
                        'status_code': response.status
                    }
            except Exception as e:
                return {
                    'proxy': proxy,
                    'success': False,
                    'error': str(e)
                }

class PerformanceProxyMiddleware:
    """
    性能优化代理中间件
    """
    
    def __init__(self):
        self.proxy_manager = OptimizedProxyManager()
        self.fast_proxy_cache = {}  # 快速代理缓存
        self.cache_ttl = 300  # 缓存5分钟
    
    def process_request(self, request, spider):
        """
        高性能代理分配
        """
        # 检查缓存
        domain = self._extract_domain(request.url)
        cached_proxy = self._get_cached_proxy(domain)
        
        if cached_proxy:
            request.meta['proxy'] = cached_proxy
            return None
        
        # 获取新代理（这里简化实现）
        proxy = self._get_optimal_proxy(request, spider)
        if proxy:
            request.meta['proxy'] = proxy
            self._cache_proxy(domain, proxy)
        
        return None
    
    def _extract_domain(self, url):
        """
        提取域名
        """
        from urllib.parse import urlparse
        return urlparse(url).netloc
    
    def _get_cached_proxy(self, domain):
        """
        获取缓存代理
        """
        if domain in self.fast_proxy_cache:
            proxy_info = self.fast_proxy_cache[domain]
            if time.time() - proxy_info['timestamp'] < self.cache_ttl:
                return proxy_info['proxy']
            else:
                # 缓存过期，移除
                del self.fast_proxy_cache[domain]
        
        return None
    
    def _cache_proxy(self, domain, proxy):
        """
        缓存代理
        """
        self.fast_proxy_cache[domain] = {
            'proxy': proxy,
            'timestamp': time.time()
        }
    
    def _get_optimal_proxy(self, request, spider):
        """
        获取最优代理（简化实现）
        """
        # 实际应用中会从代理池中选择最优代理
        return 'http://optimal-proxy:8080'

class TimeoutHandlingMiddleware:
    def process_request(self, request, spider):
        # 设置适当的超时时间
        request.meta.setdefault('download_timeout', 30)
        request.meta.setdefault('download_delay', 1)
        return None

class BanHandlingMiddleware:
    def process_response(self, request, response, spider):
        if response.status in [403, 429]:
            # 标记当前代理为不可用
            proxy = request.meta.get('proxy')
            if proxy:
                spider.crawler.engine.slot.scheduler.enqueue(request)  # 重新调度
                # 在代理池中标记此代理
        return response

class QualityStabilityMiddleware:
    def __init__(self):
        self.proxy_quality_history = {}
        self.stability_threshold = 0.8
    
    def process_response(self, request, response, spider):
        proxy = request.meta.get('proxy')
        if proxy:
            # 维护代理质量历史
            if proxy not in self.proxy_quality_history:
                self.proxy_quality_history[proxy] = []
            
            success = response.status == 200
            self.proxy_quality_history[proxy].append(success)
            
            # 保持最近10次记录
            if len(self.proxy_quality_history[proxy]) > 10:
                self.proxy_quality_history[proxy] = self.proxy_quality_history[proxy][-10:]
            
            # 计算稳定性
            if len(self.proxy_quality_history[proxy]) >= 5:
                stability = sum(self.proxy_quality_history[proxy]) / len(self.proxy_quality_history[proxy])
                if stability < self.stability_threshold:
                    # 代理不稳定，考虑更换
                    spider.logger.warning(f"Proxy {proxy} stability low: {stability}")
        
        return response