#词向量空间详解：从One-Hot到Word2Vec、GloVe及现代嵌入技术原理与PyTorch实现

import numpy as np

def one_hot_encode(word, vocab):
    """
    One-Hot编码实现
    """
    vocab_size = len(vocab)
    word_index = vocab.index(word) if word in vocab else -1
    
    if word_index == -1:
        raise ValueError(f"词 '{word}' 不在词汇表中")
    
    # 创建One-Hot向量
    one_hot = np.zeros(vocab_size)
    one_hot[word_index] = 1
    return one_hot

# 示例
vocab = ["我", "爱", "机器", "学习", "深度", "人工智能"]
machine_vec = one_hot_encode("机器", vocab)
print(f"'机器'的One-Hot向量: {machine_vec}")
# [0. 0. 1. 0. 0. 0.]

# 计算两个词的相似度（总是0，无法表示语义关系）
learning_vec = one_hot_encode("学习", vocab)
similarity = np.dot(machine_vec, learning_vec)  # 0
print(f"'机器'和'学习'的相似度: {similarity}")

# One-Hot编码的问题演示
def analyze_onehot_issues():
    """
    分析One-Hot编码的主要问题
    """
    vocab = ["国王", "王后", "男人", "女人", "巴黎", "法国", "柏林", "德国"]
    
    # 任意两个词的相似度都是0
    king_vec = np.array([1 if i == 0 else 0 for i in range(len(vocab))])  # 国王
    queen_vec = np.array([1 if i == 1 else 0 for i in range(len(vocab))])  # 王后
    
    similarity = np.dot(king_vec, queen_vec)
    print(f"国王和王后的相似度: {similarity}")  # 0，但实际上它们有很强的语义联系
    
    print(f"向量维度: {len(vocab)}")
    print(f"稀疏度: {1 - 1/len(vocab):.4f}")  # 接近100%稀疏

analyze_onehot_issues()

import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
import random

class SkipGramModel(nn.Module):
    """
    Skip-Gram模型实现
    """
    def __init__(self, vocab_size, embed_dim=100):
        super(SkipGramModel, self).__init__()
        # 中心词嵌入
        self.center_embed = nn.Embedding(vocab_size, embed_dim)
        # 上下文词嵌入（负采样用）
        self.context_embed = nn.Embedding(vocab_size, embed_dim)
        
        # 初始化权重
        initrange = 0.5 / embed_dim
        self.center_embed.weight.data.uniform_(-initrange, initrange)
        self.context_embed.weight.data.uniform_(-initrange, initrange)
    
    def forward(self, center_words, context_words, neg_words):
        """
        前向传播
        """
        # 中心词嵌入
        center_embeds = self.center_embed(center_words)  # (batch_size, embed_dim)
        
        # 正样本（真实上下文）得分
        context_embeds = self.context_embed(context_words)  # (batch_size, embed_dim)
        pos_scores = torch.sum(center_embeds * context_embeds, dim=1)  # (batch_size,)
        
        # 负样本得分
        neg_embeds = self.context_embed(neg_words)  # (batch_size, k, embed_dim)
        neg_scores = torch.bmm(neg_embeds, center_embeds.unsqueeze(2)).squeeze(2)  # (batch_size, k)
        
        # 使用log-sigmoid作为损失函数
        pos_loss = F.logsigmoid(pos_scores)
        neg_loss = torch.sum(F.logsigmoid(-neg_scores), dim=1)
        
        return -(pos_loss + neg_loss).mean()

def generate_training_data(sentences, window_size=2):
    """
    生成训练数据
    """
    pairs = []
    for sentence in sentences:
        for i, center_word in enumerate(sentence):
            # 定义上下文窗口
            start = max(0, i - window_size)
            end = min(len(sentence), i + window_size + 1)
            
            # 收集上下文词
            for j in range(start, end):
                if i != j:  # 不包括自己
                    pairs.append((center_word, sentence[j]))
    return pairs

# 示例数据
sentences = [
    ["我", "爱", "机器", "学习"],
    ["深度", "学习", "是", "人工智能", "的重要", "分支"],
    ["自然", "语言", "处理", "是", "有趣的", "领域"]
]

# 构建词汇表
word_to_idx = {}
idx_to_word = {}
all_words = []
for sentence in sentences:
    all_words.extend(sentence)

vocab = list(set(all_words))
for i, word in enumerate(vocab):
    word_to_idx[word] = i
    idx_to_word[i] = word

print(f"词汇表大小: {len(vocab)}")

class CBOWModel(nn.Module):
    """
    CBOW模型实现
    """
    def __init__(self, vocab_size, embed_dim=100):
        super(CBOWModel, self).__init__()
        self.embeddings = nn.Embedding(vocab_size, embed_dim)
        self.linear = nn.Linear(embed_dim, vocab_size)
        
        # 初始化权重
        initrange = 0.5 / embed_dim
        self.embeddings.weight.data.uniform_(-initrange, initrange)
        self.linear.weight.data.uniform_(-initrange, initrange)
        self.linear.bias.data.zero_()
    
    def forward(self, context_words):
        """
        context_words: (batch_size, context_size)
        """
        embeds = self.embeddings(context_words)  # (batch_size, context_size, embed_dim)
        # 平均上下文词向量
        avg_embeds = torch.mean(embeds, dim=1)  # (batch_size, embed_dim)
        scores = self.linear(avg_embeds)  # (batch_size, vocab_size)
        return F.log_softmax(scores, dim=1)

# 模拟预训练词向量的使用
def demonstrate_word2vec_usage():
    """
    演示如何使用预训练Word2Vec
    """
    # 模拟词向量
    class MockWord2Vec:
        def most_similar(self, positive=None, negative=None, topn=5):
            if positive and "机器学习" in positive:
                return [("深度学习", 0.85), ("人工智能", 0.82), ("自然语言处理", 0.78)]
            else:
                return [("王后", 0.92)]
    
    mock_model = MockWord2Vec()
    
    # 查找相似词
    similar = mock_model.most_similar(positive=["机器学习"], topn=3)
    print(f"与'机器学习'相似的词: {similar}")
    
    # 词语类比
    analogy = mock_model.most_similar(positive=["国王", "女人"], negative=["男人"], topn=1)
    print(f"国王 - 男人 + 女人 = {analogy[0][0]}")

demonstrate_word2vec_usage()

def simulate_glove_training():
    """
    模拟GloVe训练过程
    """
    # 构建共现矩阵（简化版）
    vocab = ["我", "爱", "机器", "学习"]
    
    # 示例共现矩阵（实际会很大）
    cooccurrence = np.array([
        [0, 1, 1, 0],  # 我
        [1, 0, 0, 1],  # 爱
        [1, 0, 0, 1],  # 机器
        [0, 1, 1, 0]   # 学习
    ])
    
    print("共现矩阵示例:")
    print(cooccurrence)

simulate_glove_training()

def explain_fasttext():
    """
    解释FastText的特点
    """
    print("FastText特点:")
    print("1. 将词分解为字符n-gram")
    print("2. 可以处理未登录词（OOV）")
    print("3. 对形态丰富的语言特别有效")
    print("4. 词向量是其字符n-gram向量的平均")
    
    # 示例：单词"where"的字符n-gram
    word = "where"
    n = 3  # trigram
    ngrams = []
    padded_word = "<" + word + ">"  # 添加边界符号
    for i in range(len(padded_word) - n + 1):
        ngram = padded_word[i:i+n]
        ngrams.append(ngram)
    
    print(f"\n'{word}'的{3}-gram: {ngrams}")
    # ['<wh', 'whe', 'her', 'ere', 're>']

explain_fasttext()

def modern_embeddings_comparison():
    """
    现代词嵌入技术对比
    """
    print("词嵌入技术演进:")
    print("1. One-Hot: 稀疏、高维、无语义")
    print("2. Word2Vec/GloVe: 稠密、低维、静态语义")
    print("3. FastText: 处理未登录词")
    print("4. ELMo/BERT: 上下文相关、动态词向量")
    print("5. GPT系列: 生成式预训练")
    
    print("\n现代最佳实践:")
    print("- 简单任务: 使用预训练词向量（Word2Vec, GloVe）")
    print("- 复杂任务: 使用Transformer模型的隐藏状态")
    print("- 资源受限: 使用轻量级模型（DistilBERT, TinyBERT）")

modern_embeddings_comparison()

def demonstrate_modern_embeddings():
    """
    演示现代词嵌入使用方法
    """
    print("现代嵌入特点: 上下文相关，同一词在不同语境下有不同表示")
    print("想象一下:")
    print("- '苹果'在'我吃了一个苹果'里的向量")
    print("- '苹果'在'我买了一台苹果电脑'里的向量")
    print("这两个向量在BERT里是不一样的！")

demonstrate_modern_embeddings()

def text_classification_with_embeddings():
    """
    使用词向量进行文本分类的示例
    """
    import numpy as np
    from sklearn.linear_model import LogisticRegression
    
    # 模拟词向量
    vocab = {"很好", "优秀", "糟糕", "差劲", "喜欢", "讨厌", "推荐", "不推荐"}
    word_vectors = {}
    for word in vocab:
        word_vectors[word] = np.random.rand(50)
    
    def sentence_to_vector(sentence, word_vectors, dim=50):
        """
        将句子转换为向量（词向量平均）
        """
        words = sentence.split()
        vectors = []
        for word in words:
            if word in word_vectors:
                vectors.append(word_vectors[word])
        
        if vectors:
            return np.mean(vectors, axis=0)
        else:
            return np.zeros(dim)
    
    # 示例数据
    texts = [
        "这个产品很好很优秀",
        "质量很不错推荐购买", 
        "很糟糕差劲不推荐",
        "质量太差劲了讨厌"
    ]
    labels = [1, 1, 0, 0]  # 1表示正面，0表示负面
    
    # 转换为向量
    X = np.array([sentence_to_vector(text, word_vectors) for text in texts])
    y = np.array(labels)
    
    print(f"特征矩阵形状: {X.shape}")
    
    # 训练分类器
    clf = LogisticRegression()
    clf.fit(X, y)
    
    # 预测新文本
    new_text = "产品质量优秀值得推荐"
    new_vec = sentence_to_vector(new_text, word_vectors)
    prediction = clf.predict([new_vec])
    
    print(f"\n新文本: {new_text}")
    print(f"预测类别: {'正面' if prediction[0] == 1 else '负面'}")

text_classification_with_embeddings()

def word_similarity_demo():
    """
    词向量相似度计算示例
    """
    from sklearn.metrics.pairwise import cosine_similarity
    
    # 模拟词向量
    words = ["机器", "学习", "深度", "人工智能", "计算机", "科学"]
    vectors = np.random.rand(len(words), 100)  # 随机向量
    
    # 计算相似度矩阵
    sim_matrix = cosine_similarity(vectors)
    
    print("词向量相似度矩阵:")
    print(f"{'':<8}", end="")
    for word in words:
        print(f"{word:<8}", end="")
    print()
    
    for i, word in enumerate(words):
        print(f"{word:<8}", end="")
        for j in range(len(words)):
            print(f"{sim_matrix[i][j]:<8.3f}", end="")
        print()

word_similarity_demo()