#Spider Middleware完全指南 - 数据预处理与后处理技术详解

# settings.py
SPIDER_MIDDLEWARES = {
    'myproject.middlewares.AntiCrawlEncodingFixMiddleware': 400,
    'myproject.middlewares.EnrichCleanMiddleware': 450,
    'myproject.middlewares.ClassifiedExceptionMiddleware': 500,
    'myproject.middlewares.DistributedPreDedupMiddleware': 600,
}

import chardet
import re
from scrapy.exceptions import IgnoreRequest

class AntiCrawlEncodingFixMiddleware:
    """
    输入预处理：
    1. 快速检测反爬特征（仅检查小体积响应）
    2. 使用 chardet 自动修正编码
    """
    def process_spider_input(self, response, spider):
        # ---------- 反爬检测 ----------
        # 只对体积较小的响应做检查，避免浪费资源
        if len(response.body) < 10240:
            anti_keywords = ['验证码', 'blocked', '访问频繁', 'forbidden']
            if any(kw in response.text.lower() for kw in anti_keywords):
                spider.logger.warning(f"疑似反爬页面，跳过：{response.url}")
                raise IgnoreRequest("Anti-crawling detected")

        # ---------- 编码修正 ----------
        # chardet 检测编码，置信度超过 0.9 时才覆盖 header 中的编码
        detect = chardet.detect(response.body)
        if detect['confidence'] > 0.9:
            response._encoding = detect['encoding']
            response.meta['fixed_encoding'] = detect['encoding']  # 记录一下，便于后续查看
        return None

import time
import hashlib
import re
from itemadapter import ItemAdapter
from scrapy.item import Item

class EnrichCleanMiddleware:
    """输出后处理：文本清洗 + 元数据增强"""
    def process_spider_output(self, response, result, spider):
        for obj in result:
            if isinstance(obj, (dict, Item)):
                self._clean_text(obj)
                self._enrich_meta(obj, response)
                yield obj
            else:
                # Request 对象直接通过，不做修改
                yield obj

    def _clean_text(self, obj):
        adapter = ItemAdapter(obj)
        for k, v in adapter.items():
            if isinstance(v, str):
                # 把换行和连续空格替换成一个空格，并去掉首尾空白
                adapter[k] = re.sub(r'\s+', ' ', v.strip())

    def _enrich_meta(self, obj, response):
        adapter = ItemAdapter(obj)
        # 生成稳定的内容 MD5（用于后续去重）
        content_bytes = str(sorted(adapter.asdict().items())).encode()
        adapter['_item_md5'] = hashlib.md5(content_bytes).hexdigest()
        # 添加来源和时间
        adapter['_source_url'] = response.url
        adapter['_crawl_time'] = time.strftime('%Y-%m-%d %H:%M:%S')

from scrapy.http import Request

class ClassifiedExceptionMiddleware:
    """
    异常分类处理：
    - 网络异常自动重试（可配置次数）
    - 解析异常直接跳过
    - 其他异常继续向上抛出
    """
    def __init__(self, max_retry=2):
        self.max_retry = max_retry

    @classmethod
    def from_crawler(cls, crawler):
        return cls(max_retry=crawler.settings.getint('SM_MAX_RETRY', 2))

    def process_spider_exception(self, response, exception, spider):
        exc_name = type(exception).__name__
        spider.logger.error(f"{exc_name} 处理失败：{response.url} | {exception}")

        # ---------- 网络类异常：重试 ----------
        network_errors = ['ConnectionError', 'TimeoutError', 'ConnectTimeout']
        if exc_name in network_errors:
            retry_cnt = response.meta.get('retry_cnt', 0) + 1
            if retry_cnt <= self.max_retry:
                spider.logger.info(f"第{retry_cnt}次重试：{response.url}")
                # 返回新 Request，dont_filter=True 保证被调度
                return [response.request.replace(dont_filter=True, meta={'retry_cnt': retry_cnt})]

        # ---------- 解析类异常：跳过 ----------
        parse_errors = ['ValueError', 'KeyError', 'IndexError']
        if exc_name in parse_errors:
            spider.logger.warning(f"解析失败，跳过页面：{response.url}")
            return []       # 返回空列表表示已处理异常，不再向上传递

        # 其他异常，不做处理，让 Scrapy 按默认逻辑处理
        return None

import redis
from itemadapter import ItemAdapter
from scrapy.item import Item

class DistributedPreDedupMiddleware:
    """分布式前置去重，基于 Redis Set 和 Item MD5"""
    def __init__(self, redis_url, dedup_key, expire_days=7):
        self.redis = redis.from_url(redis_url)
        self.dedup_key = dedup_key
        self.expire_seconds = 86400 * expire_days

    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            redis_url=crawler.settings.get('REDIS_URL', 'redis://localhost:6379'),
            dedup_key=crawler.settings.get('SM_DEDUP_KEY', 'spider:item_dedup:md5'),
            expire_days=crawler.settings.getint('SM_DEDUP_EXPIRE', 7)
        )

    def process_spider_output(self, response, result, spider):
        for obj in result:
            if isinstance(obj, (dict, Item)):
                item_md5 = ItemAdapter(obj).get('_item_md5')
                if item_md5 and self.redis.sismember(self.dedup_key, item_md5):
                    spider.logger.debug(f"已存在，前置去重跳过：{response.url}")
                    continue  # 直接丢弃重复 item
                # 添加到去重集合
                self.redis.sadd(self.dedup_key, item_md5)
                self.redis.expire(self.dedup_key, self.expire_seconds)
            yield obj