#词向量空间详解：从One-Hot到Word2Vec、GloVe及现代嵌入技术原理与PyTorch实现

import numpy as np

def one_hot_encode(word, vocab):
    """
    One-Hot编码实现
    """
    vocab_size = len(vocab)
    word_index = vocab.index(word) if word in vocab else -1
    
    if word_index == -1:
        raise ValueError(f"词 '{word}' 不在词汇表中")
    
    # 创建One-Hot向量
    one_hot = np.zeros(vocab_size)
    one_hot[word_index] = 1
    return one_hot

# 示例
vocab = ["我", "爱", "机器", "学习", "深度", "人工智能"]
machine_vec = one_hot_encode("机器", vocab)
print(f"'机器'的One-Hot向量: {machine_vec}")
# [0. 0. 1. 0. 0. 0.]

# 计算两个词的相似度（总是0，无法表示语义关系）
learning_vec = one_hot_encode("学习", vocab)
similarity = np.dot(machine_vec, learning_vec)  # 0
print(f"'机器'和'学习'的相似度: {similarity}")

# One-Hot编码的问题演示
def analyze_onehot_issues():
    """
    分析One-Hot编码的主要问题
    """
    vocab = ["国王", "王后", "男人", "女人", "巴黎", "法国", "柏林", "德国"]
    
    # 任意两个词的相似度都是0
    king_vec = np.array([1 if i == 0 else 0 for i in range(len(vocab))])  # 国王
    queen_vec = np.array([1 if i == 1 else 0 for i in range(len(vocab))])  # 王后
    
    similarity = np.dot(king_vec, queen_vec)
    print(f"国王和王后的相似度: {similarity}")  # 0，但实际上它们有很强的语义联系
    
    print(f"向量维度: {len(vocab)}")
    print(f"稀疏度: {1 - 1/len(vocab):.4f}")  # 接近100%稀疏

analyze_onehot_issues()

import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
import random

class SkipGramModel(nn.Module):
    """
    Skip-Gram模型实现
    """
    def __init__(self, vocab_size, embed_dim=100):
        super(SkipGramModel, self).__init__()
        # 中心词嵌入
        self.center_embed = nn.Embedding(vocab_size, embed_dim)
        # 上下文词嵌入（负采样用）
        self.context_embed = nn.Embedding(vocab_size, embed_dim)
        
        # 初始化权重
        initrange = 0.5 / embed_dim
        self.center_embed.weight.data.uniform_(-initrange, initrange)
        self.context_embed.weight.data.uniform_(-initrange, initrange)
    
    def forward(self, center_words, context_words, neg_words):
        """
        前向传播
        """
        # 中心词嵌入
        center_embeds = self.center_embed(center_words)  # (batch_size, embed_dim)
        
        # 正样本（真实上下文）得分
        context_embeds = self.context_embed(context_words)  # (batch_size, embed_dim)
        pos_scores = torch.sum(center_embeds * context_embeds, dim=1)  # (batch_size,)
        
        # 负样本得分
        neg_embeds = self.context_embed(neg_words)  # (batch_size, k, embed_dim)
        neg_scores = torch.bmm(neg_embeds, center_embeds.unsqueeze(2)).squeeze(2)  # (batch_size, k)
        
        # 使用log-sigmoid作为损失函数
        pos_loss = F.logsigmoid(pos_scores)
        neg_loss = torch.sum(F.logsigmoid(-neg_scores), dim=1)
        
        return -(pos_loss + neg_loss).mean()

def generate_training_data(sentences, window_size=2):
    """
    生成训练数据
    """
    pairs = []
    for sentence in sentences:
        for i, center_word in enumerate(sentence):
            # 定义上下文窗口
            start = max(0, i - window_size)
            end = min(len(sentence), i + window_size + 1)
            
            # 收集上下文词
            for j in range(start, end):
                if i != j:  # 不包括自己
                    pairs.append((center_word, sentence[j]))
    return pairs

# 示例数据
sentences = [
    ["我", "爱", "机器", "学习"],
    ["深度", "学习", "是", "人工智能", "的重要", "分支"],
    ["自然", "语言", "处理", "是", "有趣的", "领域"]
]

# 构建词汇表
word_to_idx = {}
idx_to_word = {}
all_words = []
for sentence in sentences:
    all_words.extend(sentence)

vocab = list(set(all_words))
for i, word in enumerate(vocab):
    word_to_idx[word] = i
    idx_to_word[i] = word

print(f"词汇表大小: {len(vocab)}")
print(f"示例训练对: {generate_training_data([[word_to_idx[word] for word in sent] for sent in sentences])[:5]}")

class CBOWModel(nn.Module):
    """
    CBOW模型实现
    """
    def __init__(self, vocab_size, embed_dim=100):
        super(CBOWModel, self).__init__()
        self.embeddings = nn.Embedding(vocab_size, embed_dim)
        self.linear = nn.Linear(embed_dim, vocab_size)
        
        # 初始化权重
        initrange = 0.5 / embed_dim
        self.embeddings.weight.data.uniform_(-initrange, initrange)
        self.linear.weight.data.uniform_(-initrange, initrange)
        self.linear.bias.data.zero_()
    
    def forward(self, context_words):
        """
        context_words: (batch_size, context_size)
        """
        embeds = self.embeddings(context_words)  # (batch_size, context_size, embed_dim)
        # 平均上下文词向量
        avg_embeds = torch.mean(embeds, dim=1)  # (batch_size, embed_dim)
        scores = self.linear(avg_embeds)  # (batch_size, vocab_size)
        return F.log_softmax(scores, dim=1)

# 使用预训练词向量的示例
def demonstrate_word2vec_usage():
    """
    演示如何使用预训练Word2Vec
    """
    try:
        from gensim.models import KeyedVectors
        
        # 加载预训练词向量（示例）
        # model = KeyedVectors.load_word2vec_format('path/to/vector.bin', binary=True)
        
        # 模拟词向量
        class MockWord2Vec:
            def __init__(self):
                self.vocab = {"机器学习": 0, "深度学习": 1, "人工智能": 2, "自然语言处理": 3}
                
                # 随机生成词向量（实际使用预训练模型）
                self.vectors = np.random.rand(len(self.vocab), 100)
                
            def most_similar(self, positive=None, negative=None, topn=5):
                # 简化的相似词查找
                if positive and isinstance(positive[0], str):
                    # 查找语义相似词
                    return [("深度学习", 0.85), ("人工智能", 0.82), ("自然语言处理", 0.78)]
                else:
                    # 词语类比: positive=["国王", "女人"], negative=["男人"] -> "王后"
                    return [("王后", 0.92)]
        
        mock_model = MockWord2Vec()
        
        # 查找相似词
        similar = mock_model.most_similar(positive=["机器学习"], topn=3)
        print(f"与'机器学习'相似的词: {similar}")
        
        # 词语类比
        analogy = mock_model.most_similar(
            positive=["国王", "女人"], 
            negative=["男人"], 
            topn=1
        )
        print(f"国王 - 男人 + 女人 = {analogy[0][0]}")
        
        return mock_model
    except ImportError:
        print("gensim未安装，跳过Word2Vec示例")
        return None

demonstrate_word2vec_usage()

def train_word2vec_simulation():
    """
    模拟Word2Vec训练流程
    """
    print("Word2Vec训练流程:")
    print("1. 准备训练语料（大量文本数据）")
    print("2. 构建词汇表（去除低频词）")
    print("3. 生成训练样本（中心词-上下文对）")
    print("4. 初始化词向量（随机）")
    print("5. 使用SGD优化目标函数")
    print("6. 保存训练好的词向量")
    
    # 使用gensim训练Word2Vec的示例
    sample_sentences = [
        ['自然', '语言', '处理', '是', '人工智能', '的', '重要', '分支'],
        ['机器', '学习', '和', '深度', '学习', '是', 'NLP', '的', '基础'],
        ['词向量', '能够', '表示', '词汇', '的', '语义', '信息']
    ]
    
    print(f"\n示例句子: {sample_sentences[:2]}")
    
    # 实际训练代码（需要gensim）
    # from gensim.models import Word2Vec
    # model = Word2Vec(
    #     sentences=sample_sentences,
    #     vector_size=100,
    #     window=5,
    #     min_count=1,
    #     sg=1,  # 1 for skip-gram, 0 for CBOW
    #     workers=4
    # )

train_word2vec_simulation()

def explain_glove_principle():
    """
    解释GloVe的核心原理
    """
    print("GloVe核心思想:")
    print("1. 基于全局词共现统计（统计整个语料库中词对的共现频率）")
    print("2. 结合全局统计和局部上下文信息")
    print("3. 通过向量运算表示词间关系")
    
    print("\nGloVe vs Word2Vec:")
    print("- Word2Vec: 基于局部窗口的预测模型")
    print("- GloVe: 基于全局共现矩阵的计数模型")
    print("- GloVe通常训练更快，效果稳定")

explain_glove_principle()

# GloVe训练示例（概念性）
def simulate_glove_training():
    """
    模拟GloVe训练过程
    """
    # 构建共现矩阵（简化版）
    vocab = ["我", "爱", "机器", "学习"]
    
    # 示例共现矩阵（实际会很大）
    cooccurrence = np.array([
        [0, 1, 1, 0],  # 我
        [1, 0, 0, 1],  # 爱
        [1, 0, 0, 1],  # 机器
        [0, 1, 1, 0]   # 学习
    ])
    
    print("共现矩阵示例:")
    print(cooccurrence)
    
    # 实际GloVe会使用加权最小二乘法训练
    # loss = sum(weight * (dot(u_i, v_j) - log(X_ij))^2)
    # 其中X_ij是共现频次，weight是加权函数

simulate_glove_training()

def explain_fasttext():
    """
    解释FastText的特点
    """
    print("FastText特点:")
    print("1. 将词分解为字符n-gram")
    print("2. 可以处理未登录词（OOV）")
    print("3. 对形态丰富的语言特别有效")
    print("4. 词向量是其字符n-gram向量的平均")
    
    # 示例：单词"where"的字符n-gram
    word = "where"
    n = 3  # trigram
    ngrams = []
    padded_word = "<" + word + ">"  # 添加边界符号
    for i in range(len(padded_word) - n + 1):
        ngram = padded_word[i:i+n]
        ngrams.append(ngram)
    
    print(f"\n'{word}'的{3}-gram: {ngrams}")
    # ['<wh', 'whe', 'her', 'ere', 're>']

explain_fasttext()

def modern_embeddings_comparison():
    """
    现代词嵌入技术对比
    """
    print("词嵌入技术演进:")
    print("1. One-Hot: 稀疏、高维、无语义")
    print("2. Word2Vec/GloVe: 稠密、低维、静态语义")
    print("3. FastText: 处理未登录词")
    print("4. ELMo/BERT: 上下文相关、动态词向量")
    print("5. GPT系列: 生成式预训练")
    
    print("\n现代最佳实践:")
    print("- 简单任务: 使用预训练词向量（Word2Vec, GloVe）")
    print("- 复杂任务: 使用Transformer模型的隐藏状态")
    print("- 资源受限: 使用轻量级模型（DistilBERT, TinyBERT）")

modern_embeddings_embeddings_comparison()

# 使用HuggingFace的词嵌入
def demonstrate_modern_embeddings():
    """
    演示现代词嵌入使用方法
    """
    try:
        from transformers import AutoTokenizer, AutoModel
        import torch
        
        # 加载预训练模型
        tokenizer = AutoTokenizer.from_pretrained("bert-base-chinese")
        model = AutoModel.from_pretrained("bert-base-chinese")
        
        text = "自然语言处理是人工智能的重要分支"
        inputs = tokenizer(text, return_tensors="pt", padding=True, truncation=True)
        
        with torch.no_grad():
            outputs = model(**inputs)
            # 获取最后一层隐藏状态作为词嵌入
            embeddings = outputs.last_hidden_state
        
        print(f"输入文本: {text}")
        print(f"嵌入形状: {embeddings.shape}")  # [batch_size, seq_len, hidden_size]
        print("现代嵌入特点: 上下文相关，同一词在不同语境下有不同表示")
        
    except ImportError:
        print("transformers未安装，跳过现代嵌入示例")

demonstrate_modern_embeddings()

def text_classification_with_embeddings():
    """
    使用词向量进行文本分类的示例
    """
    import numpy as np
    from sklearn.linear_model import LogisticRegression
    from sklearn.model_selection import train_test_split
    
    # 模拟词向量
    vocab = {"很好", "优秀", "糟糕", "差劲", "喜欢", "讨厌", "推荐", "不推荐"}
    word_vectors = {}
    for word in vocab:
        # 随机初始化词向量（实际使用预训练向量）
        word_vectors[word] = np.random.rand(50)
    
    def sentence_to_vector(sentence, word_vectors, dim=50):
        """
        将句子转换为向量（词向量平均）
        """
        words = sentence.split()
        vectors = []
        for word in words:
            if word in word_vectors:
                vectors.append(word_vectors[word])
        
        if vectors:
            return np.mean(vectors, axis=0)
        else:
            return np.zeros(dim)
    
    # 示例数据
    texts = [
        "这个产品很好很优秀",
        "质量很不错推荐购买", 
        "很糟糕差劲不推荐",
        "质量太差劲了讨厌"
    ]
    labels = [1, 1, 0, 0]  # 1表示正面，0表示负面
    
    # 转换为向量
    X = np.array([sentence_to_vector(text, word_vectors) for text in texts])
    y = np.array(labels)
    
    print(f"特征矩阵形状: {X.shape}")
    print(f"标签: {y}")
    
    # 训练分类器
    clf = LogisticRegression()
    clf.fit(X, y)
    
    # 预测新文本
    new_text = "产品质量优秀值得推荐"
    new_vec = sentence_to_vector(new_text, word_vectors)
    prediction = clf.predict([new_vec])
    probability = clf.predict_proba([new_vec])
    
    print(f"\n新文本: {new_text}")
    print(f"预测类别: {prediction[0]}")
    print(f"预测概率: {probability[0]}")

text_classification_with_embeddings()

def word_similarity_demo():
    """
    词向量相似度计算示例
    """
    from sklearn.metrics.pairwise import cosine_similarity
    
    # 模拟词向量（实际使用预训练模型）
    words = ["机器", "学习", "深度", "人工智能", "计算机", "科学"]
    vectors = np.random.rand(len(words), 100)  # 随机向量
    
    # 计算相似度矩阵
    sim_matrix = cosine_similarity(vectors)
    
    print("词向量相似度矩阵:")
    print(f"{'':<8}", end="")
    for word in words:
        print(f"{word:<8}", end="")
    print()
    
    for i, word in enumerate(words):
        print(f"{word:<8}", end="")
        for j in range(len(words)):
            print(f"{sim_matrix[i][j]:<8.3f}", end="")
        print()
    
    # 找最相似的词
    def find_most_similar(target_idx, topk=3):
        similarities = sim_matrix[target_idx]
        # 排除自己
        similarities[target_idx] = -1  # 设为最小值
        top_indices = np.argsort(similarities)[-topk:][::-1]
        return [(words[i], similarities[i]) for i in top_indices]
    
    print(f"\n与'机器'最相似的词:")
    most_similar = find_most_similar(0, topk=2)
    for word, sim in most_similar:
        print(f"  {word}: {sim:.3f}")

word_similarity_demo()