#文本特征工程：TF-IDF 与相似度度量

"""
TF(t,d) = 词 t 在文档 d 中出现的次数
IDF(t) = log(总文档数 / 包含词 t 的文档数)

TF-IDF(t,d) = TF(t,d) × IDF(t)

示例：
文档1："机器学习 机器 机器"      TF(机器)=3, TF(学习)=1
文档2："深度学习 深度"           TF(深度)=1, TF(学习)=1

假设总文档数=2：
IDF(机器) = log(2/1) = 0.693
IDF(深度) = log(2/1) = 0.693
IDF(学习) = log(2/2) = 0

TF-IDF("机器", 文档1) = 3 × 0.693 = 2.079  ← 关键词
TF-IDF("学习", 文档2) = 1 × 0 = 0         ← 不重要
"""

from sklearn.feature_extraction.text import TfidfVectorizer
import jieba

# 示例语料
documents = [
    "自然语言处理是人工智能的重要分支",
    "机器学习和深度学习是人工智能的核心技术",
    "自然语言处理和机器学习有着密切的关系",
]

# 中文需要自定义分词器
def chinese_tokenizer(text):
    return jieba.lcut(text)

# 创建 TF-IDF 向量化器
vectorizer = TfidfVectorizer(
    tokenizer=chinese_tokenizer,
    max_features=5000,         # 最大特征数
    min_df=1,                   # 最小文档频率
    max_df=0.8,                # 最大文档频率（过滤太常见的词）
    norm="l2",                  # L2 归一化
    use_idf=True,              # 使用 IDF
    smooth_idf=True,            # 平滑 IDF
)

# 拟合并转换
tfidf_matrix = vectorizer.fit_transform(documents)

print("词汇表:", vectorizer.get_feature_names_out())
# ['人工智能', '核心', '技术', '机器', ...]

print("TF-IDF 矩阵形状:", tfidf_matrix.shape)
# (3, N)

print("第一篇文档的 TF-IDF 向量:")
print(tfidf_matrix[0].toarray())

import numpy as np

def extract_keywords(tfidf_vector, feature_names, top_k=5):
    """提取 TF-IDF 最高的词"""
    # tfidf_vector 是稀疏向量
    sorted_indices = np.argsort(tfidf_vector.toarray().flatten())[::-1]
    top_keywords = []
    for i in sorted_indices[:top_k]:
        if tfidf_vector[0, i] > 0:
            top_keywords.append((feature_names[i], tfidf_vector[0, i]))
    return top_keywords

feature_names = vectorizer.get_feature_names_out()
keywords = extract_keywords(tfidf_matrix[0], feature_names)
print("第一篇文档的关键词:", keywords)
# [('自然语言处理', 0.7), ('人工智能', 0.5), ...]

from sklearn.metrics.pairwise import cosine_similarity
import numpy as np

def cosine_sim(a, b):
    """计算两个向量的余弦相似度"""
    a = np.array(a).reshape(1, -1)
    b = np.array(b).reshape(1, -1)
    return cosine_similarity(a, b)[0][0]

# TF-IDF 向量
doc1_vec = tfidf_matrix[0].toarray().flatten()
doc2_vec = tfidf_matrix[1].toarray().flatten()

sim = cosine_sim(doc1_vec, doc2_vec)
print(f"文档1 vs 文档2 相似度: {sim:.4f}")
# 0.6253

# 批量计算所有文档对的相似度
similarity_matrix = cosine_similarity(tfidf_matrix)
print("相似度矩阵:\n", similarity_matrix.round(4))

from sklearn.metrics.pairwise import euclidean_distances

dist = euclidean_distances(tfidf_matrix[0], tfidf_matrix[1])[0][0]
print(f"欧氏距离: {dist:.4f}")

# 注意：欧氏距离对向量长度敏感，余弦相似度对长度不敏感
# 文本相似度通常用余弦相似度

def jaccard_similarity(set1, set2):
    """杰卡德相似度 = |交集| / |并集|"""
    intersection = len(set1 & set2)
    union = len(set1 | set2)
    return intersection / union if union > 0 else 0

# 文本转为词集合
def text_to_set(text):
    return set(jieba.lcut(text))

s1 = text_to_set("自然语言处理是人工智能")
s2 = text_to_set("人工智能技术很重要")
print(f"杰卡德相似度: {jaccard_similarity(s1, s2):.4f}")
# 0.4

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
import jieba

# 数据准备
texts = [
    ("这部电影太棒了，推荐大家看", "positive"),
    ("非常无聊，浪费时间", "negative"),
    ("演员演技在线，值得一看", "positive"),
    ("剧情拖沓差评", "negative"),
    # ... 更多数据
]

X = [t[0] for t in texts]
y = [t[1] for t in texts]

# TF-IDF 特征
vectorizer = TfidfVectorizer(tokenizer=lambda x: jieba.lcut(x))
X_tfidf = vectorizer.fit_transform(X)

# 划分训练/测试集
X_train, X_test, y_train, y_test = train_test_split(
    X_tfidf, y, test_size=0.2
)

# 训练朴素贝叶斯
clf = MultinomialNB()
clf.fit(X_train, y_train)

# 评估
y_pred = clf.predict(X_test)
print(classification_report(y_test, y_pred))

# TF-IDF 速查

from sklearn.feature_extraction.text import TfidfVectorizer

vectorizer = TfidfVectorizer(tokenizer=chinese_tokenizer)
X = vectorizer.fit_transform(documents)

# 词汇表
vectorizer.get_feature_names_out()

# 计算相似度
from sklearn.metrics.pairwise import cosine_similarity
cosine_similarity(X[0], X[1])