Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fixed masked flash attention #2589

Merged
merged 4 commits into from
Jun 27, 2024
Merged

fixed masked flash attention #2589

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2c4eded
fixed masked flash attention
l-k-11235 May 17, 2024
e9edb12
some code cleaning
l-k-11235 May 21, 2024
506b355
cleaned the scaled-dot attention path
l-k-11235 May 28, 2024
4fa9721
further code cleaning
l-k-11235 May 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 10 additions & 20 deletions onmt/modules/multi_headed_attn.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -439,7 +439,6 @@ def forward(
"""
# 1) Project key, value, and query.
# as a reminder at training layer_cache[0] remains False
key_pad_mask = self.layer_cache[1].get("key_pad_mask", None)
if self.layer_cache[0]:
# Retrieve keys and values from the KV cache (decoding mode only).
if self.attn_type == "self":
Expand Down Expand Up @@ -484,6 +483,16 @@ def forward(
key = key[:, :, 1:, :]
value = value[:, :, 1:, :]

if step == 0:
key_pad_mask = self.layer_cache[1].get("key_pad_mask", None)
if key_pad_mask is not None:
x = key_pad_mask.expand(
-1, self.head_count // self.parallel_gpu, -1
)
x = x.unsqueeze(3)
x = x.expand(-1, -1, -1, value.size(3))
value = value.masked_fill(x, 0)

self.layer_cache[1]["keys"] = key
self.layer_cache[1]["values"] = value

Expand Down Expand Up @@ -565,19 +574,6 @@ def forward(
self.layer_cache[1]["keys"] = key
self.layer_cache[1]["values"] = value

if key_pad_mask is not None:
# Increase the cached key pad mask by concatenation.
# For decoding only.
if step > 0:
y = torch.zeros(
(key_pad_mask.size(0), key_pad_mask.size(1), 1),
dtype=torch.bool,
device=key_pad_mask.device,
)
self.layer_cache[1]["key_pad_mask"] = torch.cat(
(key_pad_mask, y), 2
)
key_pad_mask = self.layer_cache[1]["key_pad_mask"]
else:
# Retrieve keys and values from linear layers (training mode).
key = self.maybe_ckpt(self.linear_keys, key)
Expand Down Expand Up @@ -706,8 +702,6 @@ def forward(
scores = self.alibi(scores)

scores = scores.float()
if key_pad_mask is not None and mask is None:
mask = key_pad_mask.unsqueeze(1)

if mask is not None:
# not 100% necessary but expand to nb of heads
Expand All @@ -727,10 +721,6 @@ def forward(
attn_output.add_(relative_matmul(drop_attn, relations_values, False))

context = unshape(attn_output)
if key_pad_mask is not None:
if key_pad_mask.size(0) > 1 and context.size(1) > 1:
x = key_pad_mask.squeeze(1).unsqueeze(2).expand(-1, -1, context.size(2))
context = context.masked_fill(x, 0)

if self.layer_cache[0]:
attn_output = self.final_linear(context)
Expand Down
Loading