Python / PyTorch 工业标准参考实现答案 (含测试验证与边界断言)
import torch
import math
def attention_backward(
grad_out: torch.Tensor,
Q: torch.Tensor,
K: torch.Tensor,
V: torch.Tensor,
attn_probs: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Computes exact analytical gradients dQ, dK, dV for Scaled Dot-Product Attention.
All inputs have shape (B, S, D) or (B, S, S) for attn_probs.
"""
scale = 1.0 / math.sqrt(Q.shape[-1])
# 1. Gradient w.r.t V: dV = P^T @ grad_out
# (B, S, S)^T @ (B, S, D) -> (B, S, D)
dV = torch.bmm(attn_probs.transpose(1, 2), grad_out)
# 2. Gradient w.r.t P (attention weights): dP = grad_out @ V^T
dP = torch.bmm(grad_out, V.transpose(1, 2))
# 3. Gradient through Softmax: dS = P * (dP - sum(dP * P, dim=-1, keepdim=True))
sum_dP_P = torch.sum(dP * attn_probs, dim=-1, keepdim=True)
dS = attn_probs * (dP - sum_dP_P) * scale
# 4. Gradient w.r.t Q: dQ = dS @ K
dQ = torch.bmm(dS, K)
# 5. Gradient w.r.t K: dK = dS^T @ Q
dK = torch.bmm(dS.transpose(1, 2), Q)
return dQ, dK, dV
# --- Verification & Test Case ---
if __name__ == "__main__":
B, S, D = 1, 4, 8
Q = torch.randn(B, S, D, requires_grad=True)
K = torch.randn(B, S, D, requires_grad=True)
V = torch.randn(B, S, D, requires_grad=True)
scores = torch.bmm(Q, K.transpose(1, 2)) / math.sqrt(D)
P = torch.softmax(scores, dim=-1)
out = torch.bmm(P, V)
grad_out = torch.randn_like(out)
out.backward(grad_out)
dQ_analytic, dK_analytic, dV_analytic = attention_backward(grad_out, Q.detach(), K.detach(), V.detach(), P.detach())
assert torch.allclose(Q.grad, dQ_analytic, atol=1e-4)
assert torch.allclose(K.grad, dK_analytic, atol=1e-4)
assert torch.allclose(V.grad, dV_analytic, atol=1e-4)
print("Analytical Attention Backward exactly matches PyTorch autograd!")