#BERT家族详解：从双向编码器到ALBERT、RoBERTa等变体及Hugging Face实战

def bert_vs_gpt_comparison():
    """
    BERT与GPT的架构对比
    """
    print("BERT (Bidirectional Encoder Representations from Transformers):")
    print("- 架构：仅编码器（Encoder-only）")
    print("- 上下文：双向（左右两边都能看到）")
    print("- 训练目标：MLM（掩码语言模型）+ NSP（下一句预测）")
    print("- 应用：理解任务（分类、问答、NER等）")
    print("- 代表token：[CLS]用于分类，[SEP]分隔句子")
    
    print("\nGPT (Generative Pre-trained Transformer):")
    print("- 架构：仅解码器（Decoder-only）")
    print("- 上下文：单向（只能看到左边）")
    print("- 训练目标：LM（语言模型，预测下一个词）")
    print("- 应用：生成任务（文本生成、对话等）")
    print("- 生成方式：自回归生成")

bert_vs_gpt_comparison()

import random
import torch

def create_mlm_data(text, mask_ratio=0.15, mask_token="[MASK]", vocab_list=None):
    """
    创建MLM训练数据
    """
    tokens = text.split()  # 简化的分词
    masked_tokens = tokens.copy()
    num_mask = int(len(tokens) * mask_ratio)
    
    # 随机选择要掩码的位置
    mask_indices = random.sample(range(len(tokens)), num_mask)
    
    for idx in mask_indices:
        rand = random.random()
        if rand < 0.8:  # 80%替换为[MASK]
            masked_tokens[idx] = mask_token
        elif rand < 0.9:  # 10%保持原词
            pass
        else:  # 10%随机替换
            if vocab_list:
                masked_tokens[idx] = random.choice(vocab_list)
    
    return " ".join(masked_tokens), mask_indices

def mlm_example():
    """
    MLM任务示例
    """
    original_text = "自然语言处理是人工智能的重要分支"
    masked_text, mask_positions = create_mlm_data(original_text)
    
    print("原始文本:", original_text)
    print("掩码文本:", masked_text)
    print("掩码位置:", mask_positions)
    print("训练目标：根据上下文预测被掩码的词")

mlm_example()

def create_nsp_data(sentence_a, sentence_b, is_next=True):
    """
    创建NSP训练数据
    """
    prompt = f"[CLS] {sentence_a} [SEP] {sentence_b} [SEP]"
    label = 1 if is_next else 0
    return prompt, label

def nsp_example():
    """
    NSP任务示例
    """
    # 正例（下一句）
    sent_a = "今天天气很好"
    sent_b = "我决定出去散步"
    prompt_pos, label_pos = create_nsp_data(sent_a, sent_b, is_next=True)
    
    # 负例（非下一句）
    sent_c = "昨天吃了火锅"
    prompt_neg, label_neg = create_nsp_data(sent_a, sent_c, is_next=False)
    
    print("正例（IsNext）:")
    print(f"  输入: {prompt_pos}")
    print(f"  标签: {label_pos}")
    
    print("\n负例（NotNext）:")
    print(f"  输入: {prompt_neg}")
    print(f"  标签: {label_neg}")

nsp_example()

from transformers import BertTokenizer, BertForSequenceClassification, Trainer, TrainingArguments
import torch

def text_classification_example():
    """
    文本分类微调示例
    """
    # 加载预训练模型和分词器
    model_name = "bert-base-chinese"
    tokenizer = BertTokenizer.from_pretrained(model_name)
    model = BertForSequenceClassification.from_pretrained(
        model_name, 
        num_labels=2  # 二分类
    )
    
    # 示例数据
    texts = [
        "这个产品很好用，推荐购买",
        "质量太差了，不建议购买"
    ]
    labels = [1, 0]  # 1: 正面, 0: 负面
    
    # 编码输入
    inputs = tokenizer(
        texts,
        padding=True,
        truncation=True,
        max_length=128,
        return_tensors="pt"
    )
    inputs['labels'] = torch.tensor(labels)
    
    print("文本分类输入格式:")
    print(f"  input_ids shape: {inputs['input_ids'].shape}")
    print(f"  attention_mask shape: {inputs['attention_mask'].shape}")
    print(f"  labels: {inputs['labels']}")
    
    # 模型前向传播
    outputs = model(**inputs)
    print(f"  logits shape: {outputs.logits.shape}")
    print(f"  loss: {outputs.loss}")

text_classification_example()

from transformers import BertForTokenClassification

def ner_example():
    """
    NER微调示例
    """
    # 加载NER模型
    model = BertForTokenClassification.from_pretrained(
        "bert-base-chinese",
        num_labels=9  # PER, LOC, ORG等标签数量
    )
    
    # 示例：对分词后的序列进行标注
    text = "小明在北京大学读书"
    tokens = ["[CLS]", "小", "明", "在", "北京", "大学", "读书", "[SEP]"]
    
    # 真实标签（简化版）
    labels = [0, 1, 1, 0, 2, 2, 0, 0]  # 0: O, 1: B-PER, 2: B-ORG
    
    print("NER任务特点:")
    print("- 每个token都有对应的标签")
    print("- 使用BIO标注体系")
    print("- [CLS]和[SEP]通常标记为O")

ner_example()

from transformers import BertForQuestionAnswering

def qa_example():
    """
    问答任务示例
    """
    model = BertForQuestionAnswering.from_pretrained("bert-base-chinese")
    
    # 问题和上下文
    question = "谁是北京大学的学生？"
    context = "小明在北京大学读书，他是一名优秀的学生。"
    
    # 编码输入
    tokenizer = BertTokenizer.from_pretrained("bert-base-chinese")
    inputs = tokenizer(
        question,
        context,
        max_length=512,
        truncation=True,
        padding=True,
        return_tensors="pt"
    )
    
    # 模型预测
    with torch.no_grad():
        outputs = model(**inputs)
        start_scores = outputs.start_logits
        end_scores = outputs.end_logits
    
    print("问答任务特点:")
    print("- 输入：问题 + 上下文")
    print("- 输出：答案起始和结束位置")
    print("- 模型预测答案在上下文中的span")

qa_example()

def model_performance_comparison():
    """
    不同BERT变体的性能对比
    """
    print("BERT变体性能特点:")
    print("\n1. BERT-Base/Large:")
    print("   - 性能：高精度基准")
    print("   - 速度：相对较慢")
    print("   - 内存：占用较多")
    print("   - 适用：精度优先的任务")
    
    print("\n2. RoBERTa:")
    print("   - 改进：去掉NSP，动态mask，更多数据")
    print("   - 性能：通常优于BERT")
    print("   - 训练：更充分的预训练")
    
    print("\n3. ALBERT:")
    print("   - 轻量化：参数共享，因子分解")
    print("   - 内存：大幅减少内存占用")
    print("   - 性能：保持较好性能")
    
    print("\n4. DistilBERT/TinyBERT:")
    print("   - 速度：显著提升推理速度")
    print("   - 部署：适合边缘设备")
    print("   - 精度：轻微性能损失")

model_performance_comparison()

def albert_innovations():
    """
    ALBERT的主要创新点
    """
    print("ALBERT三大创新:")
    
    print("\n1. 跨层参数共享:")
    print("   - 传统BERT：每层独立参数")
    print("   - ALBERT：所有层共享参数")
    print("   - 效果：参数量大幅减少")
    
    print("\n2. 嵌入层参数分解:")
    print("   - 大词汇表分解为小矩阵")
    print("   - 减少嵌入层参数")
    
    print("\n3. 句子顺序预测(SOP):")
    print("   - 替代NSP任务")
    print("   - 更好的句子关系建模")

albert_innovations()

class ALBERTLayerSharing:
    """
    ALBERT参数共享概念示例
    """
    def __init__(self, shared_layer, num_layers):
        self.shared_layer = shared_layer  # 共享的层参数
        self.num_layers = num_layers      # 层数
        
    def forward(self, x):
        """
        通过共享层多次
        """
        for i in range(self.num_layers):
            x = self.shared_layer(x)  # 使用相同的参数
        return x

def parameter_sharing_benefit():
    """
    参数共享的优势
    """
    print("参数共享的优势:")
    print("- 参数量：O(n_layers * n_params) → O(n_params)")
    print("- 内存：减少模型大小")
    print("- 泛化：参数共享可能提升泛化能力")
    print("- 训练：更稳定的梯度更新")

parameter_sharing_benefit()

def roberta_improvements():
    """
    RoBERTa相对于BERT的改进
    """
    print("RoBERTa的四大改进:")
    
    print("\n1. 去掉NSP任务:")
    print("   - BERT: [CLS] sent_A [SEP] sent_B [SEP]")
    print("   - RoBERTa: [CLS] sent_A [SEP] 或 [CLS] sent_A sent_B [SEP]")
    
    print("\n2. 动态掩码:")
    print("   - BERT: 预先固定掩码")
    print("   - RoBERTa: 每次epoch重新掩码")
    
    print("\n3. 更大数据集:")
    print("   - 使用BookCorpus + English Wikipedia + CC-News + OpenWebText + Stories")
    
    print("\n4. 更长训练时间:")
    print("   - 更充分的预训练")

roberta_improvements()

def training_strategies_comparison():
    """
    不同模型的训练策略对比
    """
    strategies = {
        "BERT": {
            "masking": "静态掩码",
            "nsp": "Yes",
            "dataset_size": "16GB",
            "training_steps": "1M"
        },
        "RoBERTa": {
            "masking": "动态掩码",
            "nsp": "No",
            "dataset_size": "160GB",
            "training_steps": "500K"
        }
    }
    
    print("训练策略对比:")
    for model, strategy in strategies.items():
        print(f"\n{model}:")
        for key, value in strategy.items():
            print(f"  {key}: {value}")

training_strategies_comparison()

def distilbert_characteristics():
    """
    DistilBERT特点
    """
    print("DistilBERT (知识蒸馏):")
    print("- 方法：从大模型蒸馏小模型")
    print("- 参数：66M (BERT-base的40%)")
    print("- 速度：2倍加速")
    print("- 性能：保持95%原始性能")
    
    print("\n蒸馏过程:")
    print("1. 训练教师模型（BERT-base）")
    print("2. 使用软标签训练学生模型")
    print("3. 保持主要性能，大幅减少参数")

distilbert_characteristics()

def tinybert_features():
    """
    TinyBERT特点
    """
    print("TinyBERT (两阶段蒸馏):")
    print("- 参数：14M (BERT-base的12%)")
    print("- 层数：4层transformer")
    print("- 两阶段：BERT → Student → Teacher蒸馏")
    
    print("\n两阶段蒸馏:")
    print("第一阶段：通用蒸馏（学习通用表示）")
    print("第二阶段：任务特定蒸馏（针对特定任务）")

tinybert_features()

def model_selection_guide():
    """
    不同场景的模型选择指南
    """
    print("模型选择指南:")
    
    print("\n精度优先:")
    print("- 选择：BERT-Large, RoBERTa-Large")
    print("- 场景：竞赛、研究、高精度要求")
    
    print("\n速度优先:")
    print("- 选择：DistilBERT, TinyBERT")
    print("- 场景：在线服务、移动应用")
    
    print("\n平衡考虑:")
    print("- 选择：BERT-Base, RoBERTa-Base")
    print("- 场景：大多数实际应用")
    
    print("\n资源受限:")
    print("- 选择：MobileBERT, MiniLM")
    print("- 场景：边缘设备、IoT应用")

model_selection_guide()

from transformers import pipeline, AutoTokenizer, AutoModel

def huggingface_bert_usage():
    """
    Hugging Face BERT使用示例
    """
    print("1. 使用pipeline（最简单）:")
    
    # 文本分类
    classifier = pipeline("sentiment-analysis", model="uer/roberta-base-finetuned-chinanews-chinese")
    result = classifier("这个产品很好用")
    print(f"   情感分析结果: {result}")
    
    # 命名实体识别
    ner = pipeline("ner", model="ckiplab/bert-base-chinese-ner")
    result = ner("小明在北京大学读书")
    print(f"   NER结果: {result}")
    
    print("\n2. 使用AutoModel（更灵活）:")
    
    # 加载模型和分词器
    tokenizer = AutoTokenizer.from_pretrained("bert-base-chinese")
    model = AutoModel.from_pretrained("bert-base-chinese")
    
    # 编码文本
    text = "自然语言处理很有趣"
    inputs = tokenizer(text, return_tensors="pt", padding=True, truncation=True)
    
    # 获取嵌入
    with torch.no_grad():
        outputs = model(**inputs)
        embeddings = outputs.last_hidden_state  # [batch, seq_len, hidden_size]
    
    print(f"   嵌入形状: {embeddings.shape}")
    print(f"   [CLS]嵌入: {embeddings[:, 0, :].shape}")

huggingface_bert_usage()

def fine_tuning_example():
    """
    微调示例
    """
    from transformers import (
        BertTokenizer, 
        BertForSequenceClassification,
        Trainer,
        TrainingArguments
    )
    from torch.utils.data import Dataset
    
    class TextDataset(Dataset):
        def __init__(self, texts, labels, tokenizer, max_length=128):
            self.texts = texts
            self.labels = labels
            self.tokenizer = tokenizer
            self.max_length = max_length
        
        def __len__(self):
            return len(self.texts)
        
        def __getitem__(self, idx):
            text = str(self.texts[idx])
            label = self.labels[idx]
            
            encoding = self.tokenizer(
                text,
                truncation=True,
                padding='max_length',
                max_length=self.max_length,
                return_tensors='pt'
            )
            
            return {
                'input_ids': encoding['input_ids'].flatten(),
                'attention_mask': encoding['attention_mask'].flatten(),
                'labels': torch.tensor(label, dtype=torch.long)
            }
    
    print("微调步骤:")
    print("1. 准备数据集")
    print("2. 加载预训练模型")
    print("3. 设置训练参数")
    print("4. 创建Trainer")
    print("5. 开始训练")
    
    # 示例数据
    texts = ["很好的产品", "质量很差", "非常满意", "不推荐"]
    labels = [1, 0, 1, 0]
    
    print(f"\n示例数据: {len(texts)} 条")

fine_tuning_example()

def text_similarity_case():
    """
    使用BERT进行文本相似度计算
    """
    from transformers import AutoTokenizer, AutoModel
    from sklearn.metrics.pairwise import cosine_similarity
    import torch
    
    # 加载模型
    tokenizer = AutoTokenizer.from_pretrained("bert-base-chinese")
    model = AutoModel.from_pretrained("bert-base-chinese")
    
    def get_sentence_embedding(text):
        inputs = tokenizer(text, return_tensors="pt", padding=True, truncation=True)
        with torch.no_grad():
            outputs = model(**inputs)
            # 使用[CLS] token的嵌入作为句子表示
            sentence_embedding = outputs.last_hidden_state[:, 0, :]
        return sentence_embedding.numpy()
    
    # 计算相似度
    text1 = "自然语言处理是人工智能的重要分支"
    text2 = "AI领域中NLP是很重要的方向"
    
    emb1 = get_sentence_embedding(text1)
    emb2 = get_sentence_embedding(text2)
    
    similarity = cosine_similarity(emb1, emb2)[0][0]
    
    print(f"文本1: {text1}")
    print(f"文本2: {text2}")
    print(f"相似度: {similarity:.4f}")

text_similarity_case()

def sentiment_analysis_application():
    """
    情感分析实际应用
    """
    from transformers import pipeline
    
    # 使用预训练的情感分析模型
    classifier = pipeline(
        "sentiment-analysis",
        model="uer/roberta-base-finetuned-chinanews-chinese"
    )
    
    # 实际应用场景
    reviews = [
        "产品质量很好，物流也很快，非常满意！",
        "不太满意，质量一般，客服态度也不好。",
        "还行吧，中规中矩，没有特别惊喜的地方。"
    ]
    
    print("情感分析结果:")
    for i, review in enumerate(reviews, 1):
        result = classifier(review)[0]
        sentiment = result['label']
        confidence = result['score']
        print(f"{i}. {review}")
        print(f"   情感: {sentiment}, 置信度: {confidence:.4f}\n")

sentiment_analysis_application()