#Vision-Language多模态：CLIP模型与图文对齐详解

import torch
import torch.nn as nn
import torch.nn.functional as F

class ImageEncoder(nn.Module):
    def __init__(self, embed_dim=512, img_size=224, patch_size=16,
                 vision_width=768, vision_layers=12):
        super().__init__()
        num_patches = (img_size // patch_size) ** 2

        # 把图像切成小块，并用卷积将每个小块投影到 vision_width 维度
        self.patch_embed = nn.Conv2d(3, vision_width, patch_size, patch_size, bias=False)
        # 学习一个类别 token（类似 BERT 的 [CLS]）
        self.cls_token = nn.Parameter(torch.randn(1, 1, vision_width))
        # 位置编码
        self.pos_embed = nn.Parameter(torch.randn(1, num_patches + 1, vision_width))
        self.ln_pre = nn.LayerNorm(vision_width)

        # Transformer 编码器
        self.transformer = nn.TransformerEncoder(
            nn.TransformerEncoderLayer(vision_width, nhead=12, batch_first=True),
            num_layers=vision_layers
        )
        self.ln_post = nn.LayerNorm(vision_width)
        # 投影到统一的嵌入维度
        self.proj = nn.Parameter(torch.randn(vision_width, embed_dim))

    def forward(self, x):
        # x: (B, 3, H, W)
        x = self.patch_embed(x).flatten(2).transpose(1, 2)  # (B, num_patches, vision_width)
        # 在前面拼接 class token
        x = torch.cat([self.cls_token.expand(x.shape[0], -1, -1), x], dim=1)
        x = x + self.pos_embed
        x = self.ln_pre(x)
        x = self.transformer(x)
        # 取出 class token 对应的输出，再通过投影矩阵映射
        x = self.ln_post(x[:, 0, :]) @ self.proj
        return x

class TextEncoder(nn.Module):
    def __init__(self, embed_dim=512, context_len=77, vocab_size=49408,
                 text_width=512, text_layers=12):
        super().__init__()
        self.token_embed = nn.Embedding(vocab_size, text_width)
        self.pos_embed = nn.Parameter(torch.randn(1, context_len, text_width))
        self.transformer = nn.TransformerEncoder(
            nn.TransformerEncoderLayer(text_width, nhead=8, batch_first=True),
            num_layers=text_layers
        )
        self.ln_final = nn.LayerNorm(text_width)
        self.proj = nn.Parameter(torch.randn(text_width, embed_dim))

    def forward(self, text):
        # text: (B, 77)，每个序列的结束符是最后出现的特殊 token（eot_token）
        x = self.token_embed(text) + self.pos_embed
        x = self.transformer(x)
        x = self.ln_final(x)
        # 取 eot_token 位置的输出进行投影
        x = x[torch.arange(x.shape[0]), text.argmax(dim=-1)] @ self.proj
        return x

class CLIPLoss(nn.Module):
    def __init__(self, temperature=0.07):
        super().__init__()
        # 温度参数：控制相似度矩阵的“锐度”
        # 设为一个可学习的 log 值，训练过程中也会调整
        self.temperature = nn.Parameter(torch.tensor(temperature).log())

    def forward(self, img_feat, text_feat):
        # 1. L2 归一化，确保向量长度都为 1
        img_feat = F.normalize(img_feat, dim=-1)
        text_feat = F.normalize(text_feat, dim=-1)

        # 2. 计算相似度矩阵，并乘以温度系数的倒数（即放缩）
        logits_per_img = img_feat @ text_feat.t() * self.temperature.exp()
        logits_per_text = logits_per_img.t()

        # 3. 构造标签：第 i 张图的正确答案就是文本 i
        batch_size = img_feat.shape[0]
        labels = torch.arange(batch_size, device=img_feat.device)

        # 4. 两个方向的交叉熵损失，取平均
        loss_img = F.cross_entropy(logits_per_img, labels)
        loss_text = F.cross_entropy(logits_per_text, labels)

        return (loss_img + loss_text) / 2

from transformers import CLIPProcessor, CLIPModel
from PIL import Image
import torch

# 1. 加载模型和处理器（ViT‑B/32 速度快，ViT‑L/14 精度更高）
device = "cuda" if torch.cuda.is_available() else "cpu"
model_name = "openai/clip-vit-base-patch32"
processor = CLIPProcessor.from_pretrained(model_name)
model = CLIPModel.from_pretrained(model_name).to(device)

# 2. 准备图像和类别描述（配合提示模板）
image = Image.open("cat_dog.jpg").convert("RGB")
class_names = ["cat", "dog", "bird", "car"]
templates = [
    "a photo of a {}",
    "a photo of the {}",
    "a blurry photo of a {}",
    "a close-up of a {}"
]
# 将所有模板组合成完整的句子列表
texts = [template.format(cls) for cls in class_names for template in templates]

# 3. 预处理并推理
inputs = processor(text=texts, images=image, return_tensors="pt", padding=True).to(device)
with torch.no_grad():
    outputs = model(**inputs)

# 4. 计算每个类别的平均相似度，并转换为概率
logits_per_img = outputs.logits_per_image  # shape: [1, num_texts]
# 按类别重新分组，取平均
class_logits = logits_per_img.view(len(class_names), len(templates)).mean(dim=-1)
probs = class_logits.softmax(dim=-1).cpu().numpy()[0]

for cls, prob in zip(class_names, probs):
    print(f"{cls}: {prob:.2%}")