This commit is contained in:
lunar committed 2022-12-09 11:06:44 +00:00
1 parent b99d18f7b3
commit ab500dd288
41 files changed
+17 -7

No files matched your search

Executable → Regular
+12 -2
View File
@@ -16,6 +16,9 @@ try:
except:
XFORMERS_IS_AVAILBLE = False
# CrossAttn precision handling
import os
_ATTN_PRECISION = os.environ.get("ATTN_PRECISION", "fp32")
def exists(val):
return val is not None
@@ -167,9 +170,16 @@ class CrossAttention(nn.Module):
q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> (b h) n d', h=h), (q, k, v))
sim = einsum('b i d, b j d -> b i j', q, k) * self.scale
# force cast to fp32 to avoid overflowing
if _ATTN_PRECISION =="fp32":
with torch.autocast(enabled=False, device_type = 'cuda'):
q, k = q.float(), k.float()
sim = einsum('b i d, b j d -> b i j', q, k) * self.scale
else:
sim = einsum('b i d, b j d -> b i j', q, k) * self.scale
del q, k
if exists(mask):
mask = rearrange(mask, 'b ... -> b (...)')
max_neg_value = -torch.finfo(sim.dtype).max
View File
File mode changed.
View File
File mode changed.
View File
File mode changed.
View File
File mode changed.
View File
File mode changed.
View File
File mode changed.
View File
File mode changed.
Executable → Regular
View File
File mode changed.
Executable → Regular
View File
File mode changed.
Executable → Regular
View File
File mode changed.
View File
File mode changed.
View File
File mode changed.
View File
File mode changed.
View File
File mode changed.

Before

Width:  |  Height:  |  Size: 431 KiB

After

Width:  |  Height:  |  Size: 431 KiB

View File
File mode changed.
Executable → Regular
View File
File mode changed.
Executable → Regular
View File
File mode changed.
View File
File mode changed.
View File
File mode changed.
View File
File mode changed.
View File
File mode changed.
View File
File mode changed.
View File
File mode changed.
View File
File mode changed.
Executable → Regular
View File
File mode changed.
Executable → Regular
View File
File mode changed.