#Spider实战指南 - Request、Response、yield深度解析与爬虫逻辑实现

import scrapy

class ExampleSpider(scrapy.Spider):
    """
    基础Spider示例
    """
    # 爬虫名称（必须唯一）
    name = 'example'
    
    # 允许的域名（限制爬取范围）
    allowed_domains = ['example.com']
    
    # 起始URL列表
    start_urls = ['http://example.com']
    
    def parse(self, response):
        """
        默认回调函数，处理起始URL的响应
        """
        # 提取数据
        for item in response.css('div.item'):
            yield {
                'title': item.css('h2::text').get(),
                'price': item.css('span.price::text').get(),
                'url': response.url
            }
        
        # 提取下一页链接
        next_page = response.css('a.next::attr(href)').get()
        if next_page:
            yield response.follow(
                next_page,
                callback=self.parse
            )

class LifecycleSpider(scrapy.Spider):
    """
    演示Spider生命周期的示例
    """
    name = 'lifecycle'
    start_urls = ['http://example.com']
    
    def __init__(self, *args, **kwargs):
        """
        初始化方法
        """
        super().__init__(*args, **kwargs)
        self.logger.info("Spider initialized")
    
    @classmethod
    def from_crawler(cls, crawler, *args, **kwargs):
        """
        从crawler创建Spider实例
        """
        spider = super().from_crawler(crawler, *args, **kwargs)
        # 可以在这里注册信号处理器
        crawler.signals.connect(spider.spider_opened, signal=scrapy.signals.spider_opened)
        return spider
    
    def spider_opened(self, spider):
        """
        爬虫开启时的回调
        """
        self.logger.info(f"Spider {spider.name} opened")
    
    def start_requests(self):
        """
        生成起始请求
        """
        for url in self.start_urls:
            yield scrapy.Request(
                url=url,
                callback=self.parse,
                headers={'User-Agent': 'Custom Bot 1.0'}
            )
    
    def parse(self, response):
        """
        解析响应的主要方法
        """
        yield {'title': response.css('title::text').get()}
    
    def closed(self, reason):
        """
        爬虫关闭时的回调
        """
        self.logger.info(f"Spider closed, reason: {reason}")

"""
Request对象的完整构造参数：

url: 请求URL
callback: 响应处理回调函数
method: HTTP方法（GET/POST等）
headers: 请求头字典
body: 请求体
cookies: cookies字典
meta: 元数据字典，用于在请求间传递数据
dont_filter: 是否跳过去重过滤
errback: 错误处理回调函数
priority: 请求优先级
"""

import scrapy

class RequestSpider(scrapy.Spider):
    name = 'request_example'
    
    def start_requests(self):
        # GET请求示例
        yield scrapy.Request(
            url='http://example.com/api/data',
            callback=self.parse_get_data,
            method='GET',
            headers={
                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
                'Accept': 'application/json',
                'Authorization': 'Bearer token123'
            },
            meta={'request_type': 'api_call'}
        )
        
        # POST请求示例
        yield scrapy.Request(
            url='http://example.com/login',
            callback=self.parse_login,
            method='POST',
            headers={'Content-Type': 'application/json'},
            body='{"username": "user", "password": "pass"}',
            meta={'login_step': True}
        )
        
        # 带Cookies的请求
        yield scrapy.Request(
            url='http://example.com/protected',
            callback=self.parse_protected,
            cookies={'session_id': 'abc123'},
            meta={'requires_auth': True}
        )
    
    def parse_get_data(self, response):
        data = response.json() if response.headers.get('content-type', b'').lower().find(b'application/json') != -1 else response.text
        yield {'api_data': data}
    
    def parse_login(self, response):
        # 登录成功后，使用返回的cookies进行后续请求
        yield scrapy.Request(
            url='http://example.com/dashboard',
            callback=self.parse_dashboard,
            cookies=response.cookies
        )
    
    def parse_protected(self, response):
        yield {'protected_content': response.text[:200]}
    
    def parse_dashboard(self, response):
        yield {'dashboard_data': response.css('div.content::text').getall()}

class PrioritySpider(scrapy.Spider):
    name = 'priority_example'
    
    def start_requests(self):
        # 高优先级请求
        yield scrapy.Request(
            url='http://example.com/important',
            callback=self.parse,
            priority=100  # 高优先级
        )
        
        # 普通优先级请求
        yield scrapy.Request(
            url='http://example.com/normal',
            callback=self.parse,
            priority=0  # 默认优先级
        )
        
        # 低优先级请求
        yield scrapy.Request(
            url='http://example.com/low_priority',
            callback=self.parse,
            priority=-100  # 低优先级
        )
    
    def parse(self, response):
        yield {
            'url': response.url,
            'status': response.status,
            'priority_demo': True
        }

def parse_response_attributes(self, response):
    """
    Response对象属性详解
    """
    # 基本属性
    url = response.url              # 响应URL
    status = response.status        # HTTP状态码
    headers = response.headers      # 响应头
    body = response.body            # 响应体（bytes）
    text = response.text            # 响应体（字符串）
    request = response.request      # 对应的请求对象
    meta = response.meta            # 元数据
    
    # 编码相关
    encoding = response.encoding    # 响应编码
    selector = response.selector    # 选择器对象
    
    # 处理响应
    return {
        'url': url,
        'status': status,
        'encoding': encoding,
        'content_length': len(body),
        'content_type': headers.get('Content-Type', b'').decode()
    }

def parse_response_methods(self, response):
    """
    Response对象方法详解
    """
    # CSS选择器
    titles = response.css('h1::text').getall()
    first_title = response.css('h1::text').get()
    
    # XPath选择器
    links = response.xpath('//a/@href').getall()
    
    # 提取JSON数据
    if response.headers.get('content-type', b'').lower().find(b'application/json') != -1:
        json_data = response.json()
    
    # 跟随链接
    next_page = response.follow('next.html', callback=self.parse_next)
    
    # URL处理
    absolute_url = response.urljoin('/relative/path')
    
    # 提取文本内容
    clean_text = response.css('div.content ::text').getall()
    
    return {
        'titles': titles,
        'links': links,
        'absolute_url': absolute_url,
        'clean_text': ' '.join(clean_text).strip()
    }

class ResponseTypeSpider(scrapy.Spider):
    name = 'response_type'
    
    def parse(self, response):
        content_type = response.headers.get('content-type', b'').lower().decode()
        
        if 'application/json' in content_type:
            # JSON响应
            data = response.json()
            yield {'json_data': data}
        
        elif 'text/html' in content_type:
            # HTML响应
            title = response.css('title::text').get()
            links = response.css('a::attr(href)').getall()
            yield {
                'title': title,
                'links_count': len(links),
                'content_type': 'html'
            }
        
        elif 'application/xml' in content_type or 'text/xml' in content_type:
            # XML响应
            items = response.xpath('//item/title/text()').getall()
            yield {'xml_items': items, 'content_type': 'xml'}
        
        else:
            # 其他类型响应
            yield {
                'content_type': content_type,
                'content_length': len(response.body),
                'sample': response.text[:100] if response.text else ''
            }

def parse_with_yield_examples(self, response):
    """
    yield的不同返回类型示例
    """
    # 1. yield 字典 - 返回结构化数据
    yield {
        'title': response.css('h1::text').get(),
        'url': response.url,
        'timestamp': self.crawler.stats.start_time.isoformat()
    }
    
    # 2. yield Request - 返回新的请求
    next_page = response.css('a.next::attr(href)').get()
    if next_page:
        yield scrapy.Request(
            url=response.urljoin(next_page),
            callback=self.parse,
            meta={'page': response.meta.get('page', 1) + 1}
        )
    
    # 3. yield Item - 返回Item对象（需要定义items.py）
    # yield MyItem(title=title, price=price)
    
    # 4. yield 其他可迭代对象
    for link in response.css('a::attr(href)').getall():
        yield {'link': link}

def parse_generator_pattern(self, response):
    """
    使用生成器模式处理复杂逻辑
    """
    # 提取所有商品
    products = response.css('div.product')
    
    for product in products:
        # 提取单个商品信息
        item = {
            'name': product.css('.name::text').get(),
            'price': product.css('.price::text').get(),
            'url': response.urljoin(product.css('a::attr(href)').get())
        }
        
        # 数据验证
        if item['name'] and item['price']:
            yield item
    
    # 提取下一页
    next_page = response.css('.pagination .next::attr(href)').get()
    if next_page:
        yield scrapy.Request(
            url=response.urljoin(next_page),
            callback=self.parse_generator_pattern,
            meta={'depth': response.meta.get('depth', 0) + 1}
        )

def parse_performance_optimized(self, response):
    """
    性能优化的yield使用
    """
    # 批量处理以减少yield调用次数
    items = []
    
    for product in response.css('div.product'):
        item = {
            'name': product.css('.name::text').get(),
            'price': product.css('.price::text').get(),
            'url': response.urljoin(product.css('a::attr(href)').get())
        }
        items.append(item)
    
    # 一次性yield所有有效数据
    for item in items:
        if item['name'] and item['price']:
            yield item
    
    # 条件yield，避免不必要的请求
    if len(items) > 0:  # 只有在有数据时才继续爬取
        next_page = response.css('.next::attr(href)').get()
        if next_page:
            yield scrapy.Request(
                url=response.urljoin(next_page),
                callback=self.parse_performance_optimized
            )

class MultiLevelParsingSpider(scrapy.Spider):
    """
    多层级解析示例：分类 -> 产品列表 -> 产品详情
    """
    name = 'multi_level_parsing'
    
    def start_requests(self):
        # 请求分类页面
        yield scrapy.Request(
            url='http://example.com/categories',
            callback=self.parse_categories
        )
    
    def parse_categories(self, response):
        """
        解析分类页面，提取分类链接
        """
        for category_link in response.css('a.category-link::attr(href)').getall():
            yield response.follow(
                category_link,
                callback=self.parse_products,
                meta={'category': response.css('h1::text').get()}
            )
    
    def parse_products(self, response):
        """
        解析产品列表页面，提取产品链接
        """
        category = response.meta['category']
        
        for product_link in response.css('a.product-link::attr(href)').getall():
            yield response.follow(
                product_link,
                callback=self.parse_product_detail,
                meta={
                    'category': category,
                    'list_page_url': response.url
                }
            )
        
        # 处理翻页
        next_page = response.css('a.next::attr(href)').get()
        if next_page:
            yield response.follow(
                next_page,
                callback=self.parse_products,
                meta={'category': category}
            )
    
    def parse_product_detail(self, response):
        """
        解析产品详情页面，提取完整信息
        """
        yield {
            'category': response.meta['category'],
            'list_page_url': response.meta['list_page_url'],
            'detail_page_url': response.url,
            'name': response.css('h1.product-title::text').get(),
            'price': response.css('.price::text').get(),
            'description': response.css('.description::text').get(),
            'specs': response.css('.specs li::text').getall(),
            'images': response.css('img.product-image::attr(src)').getall()
        }

class DataIntegrationSpider(scrapy.Spider):
    """
    数据关联与整合示例
    """
    name = 'data_integration'
    
    def __init__(self):
        self.temp_storage = {}  # 临时存储跨请求数据
    
    def start_requests(self):
        # 首先获取产品基本信息
        yield scrapy.Request(
            url='http://example.com/products/basic-info',
            callback=self.parse_basic_info,
            meta={'step': 'basic'}
        )
    
    def parse_basic_info(self, response):
        """
        解析基础信息并发起详细信息请求
        """
        basic_data = {}
        for item in response.css('div.product'):
            product_id = item.css('.id::text').get()
            basic_data[product_id] = {
                'name': item.css('.name::text').get(),
                'category': item.css('.category::text').get()
            }
        
        # 存储基础数据
        self.temp_storage['basic'] = basic_data
        
        # 请求详细信息
        yield scrapy.Request(
            url='http://example.com/products/detail-info',
            callback=self.parse_detail_info,
            meta={'step': 'detail'}
        )
    
    def parse_detail_info(self, response):
        """
        解析详细信息并整合数据
        """
        detail_data = {}
        for item in response.css('div.product-detail'):
            product_id = item.css('.product-id::text').get()
            detail_data[product_id] = {
                'price': item.css('.price::text').get(),
                'stock': item.css('.stock::text').get(),
                'rating': item.css('.rating::text').get()
            }
        
        # 整合数据
        for product_id, detail in detail_data.items():
            basic = self.temp_storage['basic'].get(product_id, {})
            integrated_data = {**basic, **detail, 'id': product_id}
            yield integrated_data

def comparison_of_link_following(self, response):
    """
    链接跟随方法比较
    """
    # 方法1: response.follow() - 推荐使用
    next_page = response.css('a.next::attr(href)').get()
    if next_page:
        yield response.follow(
            next_page,  # 可以是相对URL，自动处理
            callback=self.parse,
            meta={'follow_method': 'response.follow'}
        )
    
    # 方法2: scrapy.Request - 需要手动处理URL
    next_page = response.css('a.next::attr(href)').get()
    if next_page:
        yield scrapy.Request(
            url=response.urljoin(next_page),  # 需要手动转换为绝对URL
            callback=self.parse,
            meta={'follow_method': 'scrapy.Request'}
        )

from scrapy.linkextractors import LinkExtractor

class AdvancedLinkExtractionSpider(scrapy.Spider):
    """
    高级链接提取示例
    """
    name = 'advanced_links'
    
    def parse(self, response):
        # 使用LinkExtractor提取链接
        link_extractor = LinkExtractor(
            allow=r'/category/\w+',      # 允许的URL模式
            deny=r'/admin/',             # 拒绝的URL模式
            restrict_css='.main-content', # 限制在特定CSS选择器内
            tags=('a', 'area'),          # 提取的标签类型
            attrs=('href',)              # 提取的属性
        )
        
        links = link_extractor.extract_links(response)
        for link in links:
            yield response.follow(
                link.url,
                callback=self.parse_category,
                meta={'link_text': link.text}
            )
    
    def parse_category(self, response):
        # 提取产品链接
        product_links = response.css('a.product-link::attr(href)').getall()
        for link in product_links:
            yield response.follow(link, callback=self.parse_product)
        
        # 提取下一页
        next_page = response.css('a.next::attr(href)').get()
        if next_page:
            yield response.follow(next_page, callback=self.parse_category)
    
    def parse_product(self, response):
        yield {
            'title': response.css('h1::text').get(),
            'url': response.url
        }

import re
from urllib.parse import urlparse

class SmartLinkFilterSpider(scrapy.Spider):
    """
    智能链接过滤示例
    """
    name = 'smart_links'
    
    def __init__(self):
        # 定义链接过滤规则
        self.exclude_patterns = [
            r'.*\.(jpg|jpeg|png|gif|pdf|doc|docx|zip|rar)$',  # 媒体文件
            r'.*/search\?.*',                                 # 搜索页面
            r'.*/cart.*',                                     # 购物车
            r'.*/login.*',                                    # 登录页面
            r'.*/register.*',                                 # 注册页面
        ]
    
    def parse(self, response):
        # 提取所有链接
        all_links = response.css('a::attr(href)').getall()
        
        for link in all_links:
            absolute_url = response.urljoin(link)
            
            # 应用过滤规则
            if not self._should_exclude(absolute_url):
                yield response.follow(absolute_url, callback=self.parse)
    
    def _should_exclude(self, url):
        """
        判断是否应该排除该URL
        """
        parsed = urlparse(url)
        
        # 检查是否为外部链接
        if parsed.netloc and parsed.netloc != 'example.com':
            return True
        
        # 检查URL模式
        for pattern in self.exclude_patterns:
            if re.match(pattern, url, re.IGNORECASE):
                return True
        
        return False

def advanced_css_selectors(self, response):
    """
    CSS选择器高级用法示例
    """
    # 复杂选择器
    products = response.css('div.product[id*="item"]')  # ID包含"item"的div
    
    # 属性选择器
    external_links = response.css('a[href^="http://"], a[href^="https://"]')  # 外部链接
    
    # 伪类选择器
    first_product = response.css('div.product:first-child .name::text').get()  # 第一个产品的名称
    last_product = response.css('div.product:last-child .name::text').get()    # 最后一个产品的名称
    
    # 组合选择器
    highlighted_prices = response.css('span.price.highlighted, div.price.special')  # 多类选择
    
    # 子选择器
    direct_children = response.css('div.container > p')  # 直接子元素
    
    # 后代选择器
    all_paragraphs = response.css('div.container p')     # 所有后代元素
    
    return {
        'first_product': first_product,
        'last_product': last_product,
        'highlighted_count': len(highlighted_prices)
    }

def advanced_xpath_selectors(self, response):
    """
    XPath选择器高级用法示例
    """
    # 文本包含
    contains_text = response.xpath('//div[contains(@class, "product") and contains(text(), "sale")]')
    
    # 多条件筛选
    specific_products = response.xpath('//div[@class="product" and @data-promo="true"]')
    
    # 函数使用
    total_products = response.xpath('count(//div[@class="product"])').get()
    
    # 轴选择
    following_siblings = response.xpath('//h2/following-sibling::p')  # h2后面的所有p元素
    preceding_siblings = response.xpath('//h2/preceding-sibling::p')  # h2前面的所有p元素
    
    # 位置选择
    third_product = response.xpath('(//div[@class="product"])[3]')
    
    return {
        'total_products': int(float(total_products)) if total_products else 0,
        'specific_products_count': len(specific_products)
    }

import re
from decimal import Decimal

class DataCleaningSpider(scrapy.Spider):
    """
    数据清洗与验证示例
    """
    name = 'data_cleaning'
    
    def parse(self, response):
        for product in response.css('div.product'):
            # 提取原始数据
            raw_title = product.css('.title::text').get()
            raw_price = product.css('.price::text').get()
            raw_description = product.css('.description::text').getall()
            
            # 清洗数据
            cleaned_title = self.clean_text(raw_title)
            cleaned_price = self.clean_price(raw_price)
            cleaned_description = ' '.join([self.clean_text(d) for d in raw_description]).strip()
            
            # 验证数据
            if self.validate_data(cleaned_title, cleaned_price):
                yield {
                    'title': cleaned_title,
                    'price': cleaned_price,
                    'description': cleaned_description,
                    'url': response.url
                }
    
    def clean_text(self, text):
        """
        清洗文本数据
        """
        if not text:
            return ''
        
        # 去除首尾空白
        text = text.strip()
        
        # 去除多余的空白字符
        text = re.sub(r'\s+', ' ', text)
        
        # 去除特殊字符（保留中文、英文、数字、基本标点）
        text = re.sub(r'[^\u4e00-\u9fa5a-zA-Z0-9\s\.,!?;:]', '', text)
        
        return text
    
    def clean_price(self, price_str):
        """
        清洗价格数据
        """
        if not price_str:
            return None
        
        # 提取数字
        numbers = re.findall(r'\d+\.?\d*', price_str.replace(',', ''))
        if numbers:
            try:
                return float(numbers[0])
            except ValueError:
                return None
        
        return None
    
    def validate_data(self, title, price):
        """
        验证数据有效性
        """
        if not title or len(title) < 2:
            return False
        
        if price is None or price <= 0:
            return False
        
        return True

class ErrorHandlingSpider(scrapy.Spider):
    """
    错误处理示例
    """
    name = 'error_handling'
    
    def start_requests(self):
        urls = [
            'http://example.com/valid-page',
            'http://example.com/404-page',
            'http://nonexistent-domain.com/',
        ]
        
        for url in urls:
            yield scrapy.Request(
                url=url,
                callback=self.parse,
                errback=self.handle_error,
                meta={'original_url': url}
            )
    
    def parse(self, response):
        """
        正常响应处理
        """
        if response.status == 200:
            yield {
                'url': response.url,
                'status': response.status,
                'title': response.css('title::text').get(),
                'success': True
            }
        else:
            # 状态码非200的情况
            yield scrapy.Request(
                url=response.url,
                callback=self.parse,
                errback=self.handle_error,
                dont_filter=True,
                meta={'retry_count': response.meta.get('retry_count', 0) + 1}
            )
    
    def handle_error(self, failure):
        """
        错误处理回调
        """
        request = failure.request
        self.logger.error(f"Request failed: {request.url}, Error: {failure.value}")
        
        yield {
            'url': request.meta.get('original_url', request.url),
            'error': str(failure.value),
            'success': False
        }

def robust_data_extraction(self, response):
    """
    健壮的数据提取
    """
    try:
        # 安全的数据提取
        title = self.safe_extract(response, 'h1::text', default='No Title')
        price = self.safe_extract(response, '.price::text', clean_func=self.clean_price)
        images = response.css('img::attr(src)').getall() or []
        
        return {
            'title': title,
            'price': price,
            'image_count': len(images),
            'url': response.url
        }
    except Exception as e:
        self.logger.error(f"Error extracting data from {response.url}: {str(e)}")
        return {
            'url': response.url,
            'error': str(e),
            'success': False
        }

def safe_extract(self, response, css_selector, default=None, clean_func=None):
    """
    安全的数据提取函数
    """
    try:
        result = response.css(css_selector).get()
        if result is None:
            return default
        
        result = result.strip() if result else default
        if clean_func and result:
            result = clean_func(result)
        
        return result
    except Exception:
        return default

class StateManagementSpider(scrapy.Spider):
    """
    状态管理示例
    """
    name = 'state_management'
    
    def __init__(self):
        self.visited_urls = set()  # 记录已访问的URL
        self.extracted_ids = set()  # 记录已提取的ID
        self.stats = {'visited': 0, 'extracted': 0, 'errors': 0}
    
    def parse(self, response):
        # 检查是否已访问
        if response.url in self.visited_urls:
            return
        
        self.visited_urls.add(response.url)
        self.stats['visited'] += 1
        
        # 提取数据
        for item in response.css('div.product'):
            product_id = item.css('.id::text').get()
            
            if product_id and product_id not in self.extracted_ids:
                self.extracted_ids.add(product_id)
                self.stats['extracted'] += 1
                
                yield {
                    'id': product_id,
                    'name': item.css('.name::text').get(),
                    'url': response.url
                }
        
        # 提取新链接
        for link in response.css('a::attr(href)').getall():
            absolute_url = response.urljoin(link)
            if absolute_url not in self.visited_urls:
                yield response.follow(absolute_url, callback=self.parse)
    
    def closed(self, reason):
        """
        爬虫关闭时输出统计信息
        """
        self.logger.info(f"Spider closed. Stats: {self.stats}")
        self.logger.info(f"Unique URLs visited: {len(self.visited_urls)}")
        self.logger.info(f"Unique IDs extracted: {len(self.extracted_ids)}")

class DynamicConfigSpider(scrapy.Spider):
    """
    动态配置示例
    """
    name = 'dynamic_config'
    
    def __init__(self, config_json=None, *args, **kwargs):
        super().__init__(*args, **kwargs)
        
        # 从参数加载配置
        import json
        self.config = json.loads(config_json) if config_json else self._default_config()
        
        # 设置起始URL
        self.start_urls = self.config.get('start_urls', [])
        
        # 设置提取规则
        self.extraction_rules = self.config.get('extraction_rules', {})
    
    def _default_config(self):
        """
        默认配置
        """
        return {
            'start_urls': ['http://example.com'],
            'extraction_rules': {
                'title': 'h1::text',
                'price': '.price::text',
                'description': '.description::text'
            },
            'follow_links': True,
            'max_depth': 3
        }
    
    def parse(self, response):
        # 根据配置提取数据
        data = {}
        for field, selector in self.extraction_rules.items():
            data[field] = response.css(selector).get()
        
        data['url'] = response.url
        yield data
        
        # 根据配置决定是否跟随链接
        if self.config.get('follow_links', True):
            current_depth = response.meta.get('depth', 0)
            max_depth = self.config.get('max_depth', 3)
            
            if current_depth < max_depth:
                for link in response.css('a::attr(href)').getall():
                    yield response.follow(
                        link,
                        callback=self.parse,
                        meta={'depth': current_depth + 1}
                    )

def optimized_batch_processing(self, response):
    """
    批量处理优化示例
    """
    # 预编译选择器以提高性能
    from parsel import Selector
    
    # 提取所有产品数据
    products = []
    product_elements = response.css('div.product')
    
    for elem in product_elements:
        product = {
            'name': elem.css('.name::text').get(),
            'price': elem.css('.price::text').get(),
            'url': response.urljoin(elem.css('a::attr(href)').get())
        }
        
        # 验证并添加到列表
        if product['name'] and product['price']:
            products.append(product)
    
    # 一次性yield所有数据（减少yield调用次数）
    yield from products
    
    # 智能翻页处理
    next_page = response.css('.next::attr(href)').get()
    if next_page and len(products) > 0:  # 只有在有数据时才翻页
        yield response.follow(next_page, callback=self.optimized_batch_processing)

def memory_optimized_parsing(self, response):
    """
    内存优化的解析方法
    """
    # 使用生成器表达式而不是列表
    valid_links = (link for link in response.css('a::attr(href)').getall() 
                   if self.is_valid_link(link))
    
    for link in valid_links:
        yield response.follow(link, callback=self.memory_optimized_parsing)
    
    # 及时释放大对象
    del valid_links  # 删除不再需要的生成器

def is_valid_link(self, link):
    """
    验证链接有效性（内存友好）
    """
    if not link:
        return False
    return not (link.startswith('mailto:') or link.startswith('javascript:'))