Python / PyTorch 工业标准参考实现答案 (含测试验证与边界断言)
import torch
import math
def flash_attention_tiled(Q: torch.Tensor, K: torch.Tensor, V: torch.Tensor, block_size: int = 16) -> torch.Tensor:
"""
Tiled FlashAttention implementation simulating fast SRAM tiling and online softmax rescaling.
Input shapes: Q, K, V: (B, S, D)
"""
B, S, D = Q.shape
scale = 1.0 / math.sqrt(D)
O = torch.zeros_like(Q)
l = torch.zeros(B, S, device=Q.device) # running denominator sum
m = torch.full((B, S), -float('inf'), device=Q.device) # running max
# Iterate over key/value column tiles (Br)
for j in range(0, S, block_size):
K_j = K[:, j:j + block_size, :] # (B, Br, D)
V_j = V[:, j:j + block_size, :] # (B, Br, D)
# Iterate over query row tiles (Bc)
for i in range(0, S, block_size):
Q_i = Q[:, i:i + block_size, :] # (B, Bc, D)
m_prev = m[:, i:i + block_size] # (B, Bc)
l_prev = l[:, i:i + block_size] # (B, Bc)
O_prev = O[:, i:i + block_size, :]# (B, Bc, D)
# Compute tile scores: (B, Bc, D) @ (B, D, Br) -> (B, Bc, Br)
S_ij = torch.bmm(Q_i, K_j.transpose(1, 2)) * scale
# Local block maximum
m_ij = torch.max(S_ij, dim=-1).values # (B, Bc)
m_new = torch.maximum(m_prev, m_ij)
# Compute exponentials
P_ij = torch.exp(S_ij - m_new.unsqueeze(-1)) # (B, Bc, Br)
l_new = l_prev * torch.exp(m_prev - m_new) + torch.sum(P_ij, dim=-1)
# Rescale previous accumulated output and add new block contribution
scale_prev = torch.exp(m_prev - m_new) / (l_new / (l_prev + 1e-12)).clamp(min=1e-12)
# Safe rescaling formulation:
O_i = (O_prev * torch.exp(m_prev - m_new).unsqueeze(-1) * l_prev.unsqueeze(-1) +
torch.bmm(P_ij, V_j)) / l_new.unsqueeze(-1)
# Update running stats
m[:, i:i + block_size] = m_new
l[:, i:i + block_size] = l_new
O[:, i:i + block_size, :] = O_i
return O
# --- Verification & Test Case ---
if __name__ == "__main__":
B, S, D = 2, 32, 16
Q = torch.randn(B, S, D)
K = torch.randn(B, S, D)
V = torch.randn(B, S, D)
flash_out = flash_attention_tiled(Q, K, V, block_size=8)
scores = torch.bmm(Q, K.transpose(1, 2)) / math.sqrt(D)
ref_out = torch.bmm(torch.softmax(scores, dim=-1), V)
assert torch.allclose(flash_out, ref_out, atol=1e-4)
print("FlashAttention tiled kernel matches standard PyTorch reference!")