Transformer——多头注意力机制(Pytorch)

  1. 原理图

  2. 代码

python 复制代码
import torch
import torch.nn as nn


class Multi_Head_Self_Attention(nn.Module):
    def __init__(self, embed_size, heads):
        super(Multi_Head_Self_Attention, self).__init__()
        self.embed_size = embed_size
        self.heads = heads
        self.head_dim = embed_size // heads

        self.queries = nn.Linear(self.embed_size, self.embed_size, bias=False)
        self.keys = nn.Linear(self.embed_size, self.embed_size, bias=False)
        self.values = nn.Linear(self.embed_size, self.embed_size, bias=False)
        self.fc_out = nn.Linear(self.embed_size, self.embed_size, bias=False)

    def forward(self,queries, keys, values, mask):
        N = queries.shape[0]  # batch_size
        query_len = queries.shape[1]  # sequence_length
        key_len = keys.shape[1]  # sequence_length 
        value_len = values.shape[1]  # sequence_length

        queries = self.queries(queries)
        keys = self.keys(keys)
        values = self.values(values)

        # Split the embedding into self.heads pieces
        # batch_size, sequence_length, embed_size(512) --> 
        # batch_size, sequence_length, heads(8), head_dim(64)
        queries = queries.reshape(N, query_len, self.heads, self.head_dim)
        keys = keys.reshape(N, key_len, self.heads, self.head_dim)
        values = values.reshape(N, value_len, self.heads, self.head_dim)

        # batch_size, sequence_length, heads(8), head_dim(64) --> 
        # batch_size, heads(8), sequence_length, head_dim(64)
        queries = queries.transpose(1, 2)
        keys = keys.transpose(1, 2)
        values = values.transpose(1, 2)

        # Scaled dot-product attention
        score = torch.matmul(queries, keys.transpose(-2, -1)) / (self.head_dim ** (1/2))

        if mask is not None:
            score = score.masked_fill(mask == 0, float("-inf"))
        # batch_size, heads(8), sequence_length, sequence_length
        attention = torch.softmax(score, dim=-1)

        out = torch.matmul(attention, values)
        # batch_size, heads(8), sequence_length, head_dim(64) -->
        # batch_size, sequence_length, heads(8), head_dim(64) -->
        # batch_size, sequence_length, embed_size(512)
        # 为了方便送入后面的网络
        out = out.transpose(1, 2).contiguous().reshape(N, query_len, self.embed_size)
        out = self.fc_out(out)

        return out
    

batch_size = 64
sequence_length = 10
embed_size = 512
heads = 8
mask = None

Q = torch.randn(batch_size, sequence_length, embed_size)  
K = torch.randn(batch_size, sequence_length, embed_size)  
V = torch.randn(batch_size, sequence_length, embed_size)  

model = Multi_Head_Self_Attention(embed_size, heads)
output = model(Q, K, V, mask)
print(output.shape)
相关推荐
天下弈星~2 小时前
GANs生成对抗网络生成手写数字的Pytorch实现
人工智能·pytorch·深度学习·神经网络·生成对抗网络·gans
暮小暮3 小时前
从ChatGPT到智能助手:Agent智能体如何颠覆AI应用
人工智能·深度学习·神经网络·ai·语言模型·chatgpt
七元权3 小时前
论文阅读-Gated CRF Loss for Weakly Supervised Semantic Image Segmentation
论文阅读·深度学习·计算机视觉·语义分割·弱监督
人类发明了工具3 小时前
【深度学习-基础知识】单机多卡和多机多卡训练
人工智能·深度学习
CoovallyAIHub3 小时前
方案 | 动车底部零部件检测实时流水线检测算法改进
深度学习·算法·计算机视觉
CoovallyAIHub3 小时前
方案 | 光伏清洁机器人系统详细技术实施方案
深度学习·算法·计算机视觉
大千AI助手5 小时前
SWE-bench:真实世界软件工程任务的“试金石”
人工智能·深度学习·大模型·llm·软件工程·代码生成·swe-bench
盼小辉丶7 小时前
PyTorch生成式人工智能——使用MusicGen生成音乐
pytorch·python·深度学习·生成模型
Tiger Z8 小时前
《动手学深度学习v2》学习笔记 | 1. 引言
pytorch·深度学习·ai编程
胡耀超1 天前
DataOceanAI Dolphin(ffmpeg音频转化教程) 多语言(中国方言)语音识别系统部署与应用指南
python·深度学习·ffmpeg·音视频·语音识别·多模态·asr