#Vision-Language多模态：CLIP模型与图文对齐详解

import torch
import torch.nn as nn
import torch.nn.functional as F

# -------------------------- 图像编码器（ViT简化版） --------------------------
class ImageEncoder(nn.Module):
    def __init__(self, embed_dim=512, img_size=224, patch_size=16, 
                 vision_width=768, vision_layers=12):
        super().__init__()
        # 基础参数
        num_patches = (img_size // patch_size) ** 2
        
        # ViT核心组件
        self.patch_embed = nn.Conv2d(3, vision_width, patch_size, patch_size, bias=False)
        self.cls_token = nn.Parameter(torch.randn(1, 1, vision_width))
        self.pos_embed = nn.Parameter(torch.randn(1, num_patches + 1, vision_width))
        self.ln_pre = nn.LayerNorm(vision_width)
        self.transformer = nn.TransformerEncoder(
            nn.TransformerEncoderLayer(vision_width, nhead=12, batch_first=True),
            num_layers=vision_layers
        )
        self.ln_post = nn.LayerNorm(vision_width)
        self.proj = nn.Parameter(torch.randn(vision_width, embed_dim))

    def forward(self, x):
        # 分块嵌入
        x = self.patch_embed(x).flatten(2).transpose(1, 2)  # [B, num_patches, C]
        # 加CLS Token和位置编码
        x = torch.cat([self.cls_token.expand(x.shape[0], -1, -1), x], dim=1)
        x = x + self.pos_embed
        x = self.ln_pre(x)
        # Transformer编码
        x = self.transformer(x)
        # 取CLS Token输出并投影
        x = self.ln_post(x[:, 0, :]) @ self.proj
        return x

# -------------------------- 文本编码器（简化版） --------------------------
class TextEncoder(nn.Module):
    def __init__(self, embed_dim=512, context_len=77, vocab_size=49408,
                 text_width=512, text_layers=12):
        super().__init__()
        # 基础组件
        self.token_embed = nn.Embedding(vocab_size, text_width)
        self.pos_embed = nn.Parameter(torch.randn(1, context_len, text_width))
        self.transformer = nn.TransformerEncoder(
            nn.TransformerEncoderLayer(text_width, nhead=8, batch_first=True),
            num_layers=text_layers
        )
        self.ln_final = nn.LayerNorm(text_width)
        self.proj = nn.Parameter(torch.randn(text_width, embed_dim))

    def forward(self, text):
        # text是[B, 77]的token序列，eot_token为每行最大值
        x = self.token_embed(text) + self.pos_embed
        x = self.transformer(x)
        x = self.ln_final(x)
        # 取eot_token输出并投影
        x = x[torch.arange(x.shape[0]), text.argmax(dim=-1)] @ self.proj
        return x

class CLIPLoss(nn.Module):
    def __init__(self, temperature=0.07):
        super().__init__()
        # 温度参数：控制相似度分布的锐度，0.07是OpenAI预训练的默认值
        self.temperature = nn.Parameter(torch.tensor(temperature).log())

    def forward(self, img_feat, text_feat):
        # 1. L2归一化（必须！确保嵌入在单位球面上）
        img_feat = F.normalize(img_feat, dim=-1)
        text_feat = F.normalize(text_feat, dim=-1)
        
        # 2. 计算双向相似度矩阵
        logits_per_img = img_feat @ text_feat.t() * self.temperature.exp()
        logits_per_text = logits_per_img.t()
        
        # 3. 创建标签：Batch内同一索引为正样本
        batch_size = img_feat.shape[0]
        labels = torch.arange(batch_size, device=img_feat.device)
        
        # 4. 双向交叉熵损失
        loss_img = F.cross_entropy(logits_per_img, labels)
        loss_text = F.cross_entropy(logits_per_text, labels)
        
        return (loss_img + loss_text) / 2

from transformers import CLIPProcessor, CLIPModel
from PIL import Image
import torch

# 1. 加载预训练模型和处理器（ViT-B/32速度快，ViT-L/14精度高）
device = "cuda" if torch.cuda.is_available() else "cpu"
model_name = "openai/clip-vit-base-patch32"
processor = CLIPProcessor.from_pretrained(model_name)
model = CLIPModel.from_pretrained(model_name).to(device)

# 2. 准备待分类图像和类别（带自然语言模板）
image = Image.open("cat_dog.jpg").convert("RGB")
class_names = ["cat", "dog", "bird", "car"]
templates = [
    "a photo of a {}",
    "a photo of the {}",
    "a blurry photo of a {}",
    "a close-up of a {}"
]
texts = [template.format(cls) for cls in class_names for template in templates]

# 3. 预处理并推理
inputs = processor(text=texts, images=image, return_tensors="pt", padding=True).to(device)
with torch.no_grad():
    outputs = model(**inputs)

# 4. 计算平均相似度并输出预测
logits_per_img = outputs.logits_per_image  # [1, len(texts)]
class_logits = logits_per_img.view(len(class_names), len(templates)).mean(dim=-1)
probs = class_logits.softmax(dim=-1).cpu().numpy()[0]

# 打印结果
for cls, prob in zip(class_names, probs):
    print(f"{cls}: {prob:.2%}")