Quantumbraid's picture
download
raw
2.12 kB
"""Varlen attention shim for Mage-Flow.
The reference implementation (github.com/microsoft/Mage) runs every attention
call through flash-attn's ``flash_attn_varlen_func`` so that several
variable-length samples can be packed into one sequence and kept isolated via
``cu_seqlens``. This shim exposes the same function with the FA2 calling
convention, falling back to a per-sequence ``torch.scaled_dot_product_attention``
loop when flash-attn is not installed (functionally equivalent for the dense /
non-causal path the DiT uses, just slower).
"""
from __future__ import annotations
from typing import Any, Callable
import torch
import torch.nn.functional as F
_RESOLVED_FN: Callable[..., Any] | None = None
def _sdpa_varlen(
q,
k,
v,
cu_seqlens_q=None,
cu_seqlens_k=None,
max_seqlen_q=None,
max_seqlen_k=None,
dropout_p: float = 0.0,
softmax_scale=None,
causal: bool = False,
**_unused: Any,
):
"""FA2-varlen-compatible SDPA fallback: one SDPA dispatch per sequence."""
cu_q = cu_seqlens_q.tolist()
cu_k = cu_seqlens_k.tolist()
outs = []
for qs, qe, ks, ke in zip(cu_q[:-1], cu_q[1:], cu_k[:-1], cu_k[1:]):
# (s, h, d) -> (1, h, s, d)
q_i = q[qs:qe].transpose(0, 1).unsqueeze(0)
k_i = k[ks:ke].transpose(0, 1).unsqueeze(0)
v_i = v[ks:ke].transpose(0, 1).unsqueeze(0)
out_i = F.scaled_dot_product_attention(
q_i,
k_i,
v_i,
attn_mask=None,
dropout_p=dropout_p,
is_causal=causal,
scale=softmax_scale,
)
outs.append(out_i.squeeze(0).transpose(0, 1))
return torch.cat(outs, dim=0).contiguous()
def _resolve() -> Callable[..., Any]:
global _RESOLVED_FN
if _RESOLVED_FN is None:
try:
from flash_attn import flash_attn_varlen_func as _fn
_RESOLVED_FN = _fn
except ImportError:
_RESOLVED_FN = _sdpa_varlen
return _RESOLVED_FN
def flash_attn_varlen_func(*args, **kwargs):
return _resolve()(*args, **kwargs)
__all__ = ["flash_attn_varlen_func"]

Xet Storage Details

Size:
2.12 kB
·
Xet hash:
3d44e51f68f97a1db992cb7a008aa8b58df0e9e3afc00c17a84a2f59c4f10bf1

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.