#文本清洗与规范化：停用词过滤、正则表达式与词干提取

import re
from html import unescape
from lxml import html

def remove_html(text, use_lxml=False):
    """去除 HTML 标签 + 转义字符（如 &lt; → <）"""
    # 先统一转义
    text = unescape(text)
    if use_lxml:
        # lxml 处理嵌套/残缺更稳定
        try:
            return html.fromstring(text).text_content()
        except Exception:
            pass
    # 回退方案：正则
    return re.sub(r'<[^>]+>', '', text)

text = "<p>这是一段<strong>嵌套HTML</strong>文本，还有<未闭合标签"
print(remove_html(text, use_lxml=True))
# 这是一段嵌套HTML文本，还有

def clean_noise(text):
    """组合清理 URL、邮箱、占位数字（可选）"""
    # 清理 http/https/www 开头的 URL
    text = re.sub(r'https?://\S+|www\.\S+', '', text)
    # 清理标准格式邮箱
    text = re.sub(r'\S+@\S+\.\S+', '', text)
    # 可选：清理占位符类数字（比如工单ID、身份证片段提示）
    # text = re.sub(r'\d{4,}', '', text)
    return text

text = "工单处理进度可戳 https://xxx.com/ticket ，或发邮件至 service@xxx.com"
print(clean_noise(text))
# 工单处理进度可戳 ，或发邮件至 

import unicodedata

def full_to_half(text):
    """全角英数/空格/符号转半角"""
    result = []
    for char in text:
        code = ord(char)
        # 全角空格 → 普通空格
        if code == 12288:
            code = 32
        # 全角字符区间（从！到～）
        elif 65281 <= code <= 65374:
            code -= 65248
        result.append(chr(code))
    return "".join(result)

def normalize_text(text):
    """统一字符编码（NFC最常用），还可以组合消除连笔/装饰符"""
    # NFC：组合字符标准化（比如 "a" + "～" → "ã"）
    text = unicodedata.normalize("NFC", text)
    # 可选：删除非间距装饰符（比如 "ñ" → "n"，慎用，会丢失语言特征）
    # text = "".join([c for c in text if not unicodedata.combining(c)])
    return text

text = "ＨＥＬＬＯ～１２３，我今年２５岁啦"
text = full_to_half(normalize_text(text))
print(text)
# HELLO～123，我今年25岁啦

def clean_duplicate_and_space(text, max_char_repeat=2):
    """压缩连续重复字符 + 合并多余空白"""
    # 压缩重复字符（比如“啊啊啊啊”→“啊啊”）
    text = re.sub(r'(.)\1{%d,}' % max_char_repeat, r'\1' * max_char_repeat, text)
    # 合并换行/制表符/空格，首尾去空
    text = re.sub(r'\s+', ' ', text).strip()
    return text

text = "  啊啊啊啊啊   今天真的！真的！真的！开心到飞起\n\n"
print(clean_duplicate_and_space(text))
# 啊啊 今天真的！真的！开心到飞起

import jieba

# 基础中文停用词表（可根据业务补充，比如“点击、了解”这种通用推广词）
STOPWORDS_ZH = {
    '的', '了', '和', '是', '就', '都', '而', '及', '与', '着',
    '或', '一个', '没有', '我们', '你们', '他们', '这个', '那个',
    '啊', '呀', '呢', '吧', '吗', '哦', '哈', '嗯', '哎',
    '了', '着', '过', '的', '地', '得',
}

def filter_stopwords(tokens, stopwords, min_len=1):
    """过滤停用词 + 过滤过短词（比如单字语气词/标点残留）"""
    return [
        t for t in tokens
        if t not in stopwords and len(t) >= min_len
    ]

# 测试
text = "我今天在省图书馆认真学习自然语言处理技术"
tokens = jieba.lcut(text)
filtered_tokens = filter_stopwords(tokens, STOPWORDS_ZH, min_len=2)
print(filtered_tokens)
# ['今天', '图书馆', '认真', '学习', '自然语言处理', '技术']

def load_custom_stopwords(file_path):
    """从本地UTF-8文件加载停用词（一行一个）"""
    try:
        with open(file_path, 'r', encoding='utf-8') as f:
            return set(line.strip() for line in f if line.strip())
    except FileNotFoundError:
        print(f"⚠️ 停用词文件 {file_path} 不存在，使用默认空表")
        return set()

# 假设 stopwords_zh_ecommerce.txt 存了“亲、包邮、限时”
custom_stopwords = load_custom_stopwords("stopwords_zh_ecommerce.txt")

import nltk
from nltk.stem import PorterStemmer, LancasterStemmer

# 首次使用需下载punkt分词器
nltk.download('punkt', quiet=True)

porter = PorterStemmer()  # 温和型，保留更多语义
lancaster = LancasterStemmer()  # 激进型，压缩更狠

words = ["running", "runs", "ran", "runner", "easily", "fairly"]
print("Porter词干：", [porter.stem(w) for w in words])
print("Lancaster词干：", [lancaster.stem(w) for w in words])

# 输出
# Porter词干： ['run', 'run', 'ran', 'runner', 'easili', 'fairli']
# Lancaster词干： ['run', 'run', 'run', 'run', 'eas', 'fair']

from nltk.stem import WordNetLemmatizer
from nltk.corpus import wordnet

# 首次使用需下载资源
nltk.download('wordnet', quiet=True)
nltk.download('averaged_perceptron_tagger', quiet=True)

lemmatizer = WordNetLemmatizer()

def get_wordnet_pos(tag):
    """把nltk的词性标签转为wordnet可用的"""
    if tag.startswith('J'):
        return wordnet.ADJ
    elif tag.startswith('V'):
        return wordnet.VERB
    elif tag.startswith('N'):
        return wordnet.NOUN
    elif tag.startswith('R'):
        return wordnet.ADV
    else:
        return wordnet.NOUN  # 默认名词

text = "He is running faster than me because he runs every day and ran yesterday"
tokens = nltk.word_tokenize(text)
pos_tags = nltk.pos_tag(tokens)

lemmas = [
    lemmatizer.lemmatize(token, get_wordnet_pos(tag))
    for token, tag in pos_tags
]
print(lemmas)
# ['He', 'be', 'run', 'faster', 'than', 'me', 'because', 'he', 'run', 'every', 'day', 'and', 'run', 'yesterday']

from functools import reduce

class SimpleTextPreprocessor:
    def __init__(self, stopwords=None, lang='zh'):
        self.stopwords = stopwords or set()
        self.lang = lang

    # 中文/英文通用清洗
    def _clean_general(self, text):
        steps = [
            lambda x: remove_html(x, use_lxml=True),
            clean_noise,
            normalize_text,
            lambda x: full_to_half(x) if self.lang == 'zh' else x,
            clean_duplicate_and_space,
        ]
        return reduce(lambda t, fn: fn(t), steps, text)

    # 中文分词过滤
    def _process_zh(self, text):
        tokens = jieba.lcut(text)
        return filter_stopwords(tokens, self.stopwords, min_len=2)

    # 英文分词还原（可选词干）
    def _process_en(self, text, use_lemmatizer=True):
        tokens = nltk.word_tokenize(text.lower())  # 英文一般转小写
        if use_lemmatizer:
            pos_tags = nltk.pos_tag(tokens)
            tokens = [
                lemmatizer.lemmatize(t, get_wordnet_pos(tag))
                for t, tag in pos_tags
            ]
        else:
            tokens = [porter.stem(t) for t in tokens]
        # 英文也有基础停用词，这里简化省略
        return [t for t in tokens if len(t) >= 2]

    # 主入口
    def process(self, text, **kwargs):
        cleaned = self._clean_general(text)
        if self.lang == 'zh':
            return self._process_zh(cleaned)
        else:
            return self._process_en(cleaned, **kwargs)

# 测试中文
zh_processor = SimpleTextPreprocessor(stopwords=STOPWORDS_ZH, lang='zh')
zh_raw = "<p>🔥🔥🔥 年度大促！！！快来 https://xxx.com 抢！抢！抢！亲，仅限新粉哦～</p>"
print(zh_processor.process(zh_raw))
# ['年度', '大促', '快来', '仅限', '新粉']