Buckets:
| """Mage-Flow NR-MMDiT (dual-stream, native-resolution packed) for ai-toolkit. | |
| Vendored from the reference implementation | |
| (github.com/microsoft/Mage, mage_flow/models/mage_flow.py + modules/mage_layers.py) | |
| with these deviations: | |
| - loguru / pydantic / torch._dynamo decorators dropped, | |
| - gradient checkpointing is gated on ``self.gradient_checkpointing and | |
| torch.is_grad_enabled()`` (instead of the reference's ``self.training and | |
| self.checkpoint``) and exposed via ``enable_gradient_checkpointing`` / | |
| ``disable_gradient_checkpointing`` so ai-toolkit's trainer can toggle it, | |
| - flash-attn is imported through the local shim (``.attn``) which falls back | |
| to a per-sequence SDPA loop when flash-attn is unavailable. | |
| State-dict keys are identical to the reference checkpoints | |
| (``transformer/diffusion_pytorch_model.safetensors`` in | |
| microsoft/Mage-Flow-Base and microsoft/Mage-Flow-Edit-Base). | |
| Sequence layout: images and text are packed as variable-length sequences of | |
| batch dim 1 (``[1, sum_len, C]``) with per-sample ``cu_seqlens``; joint text+image | |
| attention runs in a single varlen kernel. 2D multi-scale RoPE is applied to | |
| image tokens only (text tokens are not rotated). ``timesteps`` is the flow | |
| sigma in [0, 1] (1 = pure noise), one entry per packed sample. | |
| """ | |
| import math | |
| from dataclasses import dataclass | |
| from typing import Any | |
| import torch | |
| from toolkit.models.v2._mixin import OstrisModelMixin | |
| import torch.nn as nn | |
| from torch import Tensor | |
| from torch.utils.checkpoint import checkpoint | |
| from diffusers.models.attention import FeedForward | |
| from diffusers.models.embeddings import TimestepEmbedding | |
| from diffusers.models.normalization import RMSNorm | |
| from .attn import flash_attn_varlen_func | |
| def apply_rotary_emb_mageflow(x: torch.Tensor, freqs_cis: torch.Tensor) -> torch.Tensor: | |
| """Apply complex rotary embeddings to `x` ([S, H, D]) using `freqs_cis` | |
| (the MageFlowEmbedRope 2D multi-scale RoPE, adjacent-pair complex convention).""" | |
| x_rotated = torch.view_as_complex(x.float().reshape(*x.shape[:-1], -1, 2)) | |
| freqs_cis = freqs_cis.unsqueeze(1) | |
| x_out = torch.view_as_real(x_rotated * freqs_cis).flatten(-2) | |
| return x_out.type_as(x) | |
| def get_timestep_embedding( | |
| timesteps: torch.Tensor, | |
| embedding_dim: int, | |
| flip_sin_to_cos: bool = False, | |
| downscale_freq_shift: float = 1, | |
| scale: float = 1, | |
| max_period: int = 10000, | |
| ) -> torch.Tensor: | |
| """Sinusoidal timestep embeddings (DDPM convention). | |
| NOTE: kept vendored (not diffusers') because the frequency table is | |
| downcast to ``timesteps.dtype`` (bf16) here — the model was trained with | |
| this exact bf16 rounding, so diffusers' fp32 variant produces a slightly | |
| different embedding and degrades outputs. | |
| """ | |
| assert len(timesteps.shape) == 1, "Timesteps should be a 1d-array" | |
| half_dim = embedding_dim // 2 | |
| exponent = -math.log(max_period) * torch.arange( | |
| start=0, end=half_dim, dtype=torch.float32, device=timesteps.device | |
| ) | |
| exponent = exponent / (half_dim - downscale_freq_shift) | |
| emb = torch.exp(exponent).to(timesteps.dtype) | |
| emb = timesteps[:, None].float() * emb[None, :] | |
| # scale embeddings | |
| emb = scale * emb | |
| # concat sine and cosine embeddings | |
| emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=-1) | |
| # flip sine and cosine embeddings | |
| if flip_sin_to_cos: | |
| emb = torch.cat([emb[:, half_dim:], emb[:, :half_dim]], dim=-1) | |
| # zero pad | |
| if embedding_dim % 2 == 1: | |
| emb = torch.nn.functional.pad(emb, (0, 1, 0, 0)) | |
| return emb | |
| class Timesteps(nn.Module): | |
| def __init__( | |
| self, | |
| num_channels: int, | |
| flip_sin_to_cos: bool, | |
| downscale_freq_shift: float, | |
| scale: int = 1, | |
| ): | |
| super().__init__() | |
| self.num_channels = num_channels | |
| self.flip_sin_to_cos = flip_sin_to_cos | |
| self.downscale_freq_shift = downscale_freq_shift | |
| self.scale = scale | |
| def forward(self, timesteps: torch.Tensor) -> torch.Tensor: | |
| return get_timestep_embedding( | |
| timesteps, | |
| self.num_channels, | |
| flip_sin_to_cos=self.flip_sin_to_cos, | |
| downscale_freq_shift=self.downscale_freq_shift, | |
| scale=self.scale, | |
| ) | |
| class MageFlowTimestepProjEmbeddings(nn.Module): | |
| def __init__(self, embedding_dim): | |
| super().__init__() | |
| self.time_proj = Timesteps( | |
| num_channels=256, flip_sin_to_cos=True, downscale_freq_shift=0, scale=1000 | |
| ) | |
| self.timestep_embedder = TimestepEmbedding( | |
| in_channels=256, time_embed_dim=embedding_dim | |
| ) | |
| def forward(self, timestep, hidden_states): | |
| timesteps_proj = self.time_proj(timestep) | |
| timesteps_emb = self.timestep_embedder( | |
| timesteps_proj.to(dtype=hidden_states.dtype) | |
| ) # (N, D) | |
| return timesteps_emb | |
| class MageFlowEmbedRope(nn.Module): | |
| def __init__(self, theta: int, axes_dim: list, scale_rope=False): | |
| super().__init__() | |
| self.theta = theta | |
| self.axes_dim = axes_dim | |
| pos_index = torch.arange(4096) | |
| neg_index = torch.arange(4096).flip(0) * -1 - 1 | |
| self.pos_freqs = torch.cat( | |
| [ | |
| self.rope_params(pos_index, self.axes_dim[0], self.theta), | |
| self.rope_params(pos_index, self.axes_dim[1], self.theta), | |
| self.rope_params(pos_index, self.axes_dim[2], self.theta), | |
| ], | |
| dim=1, | |
| ) | |
| self.neg_freqs = torch.cat( | |
| [ | |
| self.rope_params(neg_index, self.axes_dim[0], self.theta), | |
| self.rope_params(neg_index, self.axes_dim[1], self.theta), | |
| self.rope_params(neg_index, self.axes_dim[2], self.theta), | |
| ], | |
| dim=1, | |
| ) | |
| # DO NOT USING REGISTER BUFFER HERE, IT WILL CAUSE COMPLEX NUMBERS LOSE ITS IMAGINARY PART | |
| self.scale_rope = scale_rope | |
| self.video_freq_cache = {} | |
| def rope_params(self, index, dim, theta=10000): | |
| assert dim % 2 == 0 | |
| freqs = torch.outer( | |
| index, | |
| 1.0 / torch.pow(theta, torch.arange(0, dim, 2).to(torch.float32).div(dim)), | |
| ) | |
| freqs = torch.polar(torch.ones_like(freqs), freqs) | |
| return freqs | |
| def forward( | |
| self, video_fhw, device: torch.device, max_img_len: int = None | |
| ) -> torch.Tensor: | |
| """Compute the vision RoPE frequencies for the packed image tokens. | |
| Text tokens are NOT rotated, so no text RoPE is computed. | |
| ``video_fhw`` is a list of (frame, height, width) tuples, one per image | |
| segment in the packed sequence; segment index doubles as the "frame" | |
| RoPE coordinate (edit reference images land on later frame indices). | |
| """ | |
| if self.pos_freqs.device != device: | |
| self.pos_freqs = self.pos_freqs.to(device) | |
| self.neg_freqs = self.neg_freqs.to(device) | |
| if isinstance(video_fhw, list): | |
| video_fhw = video_fhw[0] | |
| if not isinstance(video_fhw, list): | |
| video_fhw = [video_fhw] | |
| vid_freqs = [] | |
| for idx, fhw in enumerate(video_fhw): | |
| frame, height, width = fhw | |
| key = (frame, height, width, idx) | |
| if key not in self.video_freq_cache: | |
| self.video_freq_cache[key] = self._compute_video_freqs( | |
| frame, height, width, idx | |
| ) | |
| vid_freqs.append(self.video_freq_cache[key].to(device)) | |
| vid_freqs = torch.cat(vid_freqs, dim=0) | |
| if max_img_len is not None and vid_freqs.shape[0] < max_img_len: | |
| pad_len = max_img_len - vid_freqs.shape[0] | |
| vid_freqs = torch.nn.functional.pad(vid_freqs, (0, 0, 0, pad_len)) | |
| return vid_freqs | |
| def _compute_video_freqs( | |
| self, frame: int, height: int, width: int, idx: int = 0 | |
| ) -> torch.Tensor: | |
| seq_lens = frame * height * width | |
| freqs_pos = self.pos_freqs.split([x // 2 for x in self.axes_dim], dim=1) | |
| freqs_neg = self.neg_freqs.split([x // 2 for x in self.axes_dim], dim=1) | |
| freqs_frame = ( | |
| freqs_pos[0][idx : idx + frame] | |
| .view(frame, 1, 1, -1) | |
| .expand(frame, height, width, -1) | |
| ) | |
| if self.scale_rope: | |
| freqs_height = torch.cat( | |
| [freqs_neg[1][-(height - height // 2) :], freqs_pos[1][: height // 2]], | |
| dim=0, | |
| ) | |
| freqs_height = freqs_height.view(1, height, 1, -1).expand( | |
| frame, height, width, -1 | |
| ) | |
| freqs_width = torch.cat( | |
| [freqs_neg[2][-(width - width // 2) :], freqs_pos[2][: width // 2]], | |
| dim=0, | |
| ) | |
| freqs_width = freqs_width.view(1, 1, width, -1).expand( | |
| frame, height, width, -1 | |
| ) | |
| else: | |
| freqs_height = ( | |
| freqs_pos[1][:height] | |
| .view(1, height, 1, -1) | |
| .expand(frame, height, width, -1) | |
| ) | |
| freqs_width = ( | |
| freqs_pos[2][:width] | |
| .view(1, 1, width, -1) | |
| .expand(frame, height, width, -1) | |
| ) | |
| freqs = torch.cat([freqs_frame, freqs_height, freqs_width], dim=-1).reshape( | |
| seq_lens, -1 | |
| ) | |
| return freqs.clone().contiguous() | |
| class Attention(nn.Module): | |
| """Joint text+image attention (reference subset: the double-stream config | |
| MageFlow actually instantiates — qk rms_norm, added kv projections).""" | |
| def __init__( | |
| self, | |
| query_dim: int, | |
| heads: int = 8, | |
| dim_head: int = 64, | |
| dropout: float = 0.0, | |
| bias: bool = False, | |
| added_kv_proj_dim: int = None, | |
| added_proj_bias: bool = True, | |
| out_bias: bool = True, | |
| eps: float = 1e-5, | |
| processor=None, | |
| out_dim: int = None, | |
| out_context_dim: int = None, | |
| elementwise_affine: bool = True, | |
| ): | |
| super().__init__() | |
| self.inner_dim = out_dim if out_dim is not None else dim_head * heads | |
| self.inner_kv_dim = self.inner_dim | |
| self.query_dim = query_dim | |
| self.use_bias = bias | |
| self.cross_attention_dim = query_dim | |
| self.out_dim = out_dim if out_dim is not None else query_dim | |
| self.out_context_dim = ( | |
| out_context_dim if out_context_dim is not None else query_dim | |
| ) | |
| self.heads = out_dim // dim_head if out_dim is not None else heads | |
| self.added_kv_proj_dim = added_kv_proj_dim | |
| self.added_proj_bias = added_proj_bias | |
| # qk_norm is always "rms_norm" for MageFlow. | |
| self.norm_q = RMSNorm(dim_head, eps=eps, elementwise_affine=elementwise_affine) | |
| self.norm_k = RMSNorm(dim_head, eps=eps, elementwise_affine=elementwise_affine) | |
| self.to_q = nn.Linear(query_dim, self.inner_dim, bias=bias) | |
| self.to_k = nn.Linear(self.cross_attention_dim, self.inner_kv_dim, bias=bias) | |
| self.to_v = nn.Linear(self.cross_attention_dim, self.inner_kv_dim, bias=bias) | |
| self.add_k_proj = nn.Linear( | |
| added_kv_proj_dim, self.inner_kv_dim, bias=added_proj_bias | |
| ) | |
| self.add_v_proj = nn.Linear( | |
| added_kv_proj_dim, self.inner_kv_dim, bias=added_proj_bias | |
| ) | |
| self.add_q_proj = nn.Linear( | |
| added_kv_proj_dim, self.inner_dim, bias=added_proj_bias | |
| ) | |
| self.norm_added_q = RMSNorm(dim_head, eps=eps) | |
| self.norm_added_k = RMSNorm(dim_head, eps=eps) | |
| self.to_out = nn.ModuleList([]) | |
| self.to_out.append(nn.Linear(self.inner_dim, self.out_dim, bias=out_bias)) | |
| self.to_out.append(nn.Dropout(dropout)) | |
| self.to_add_out = nn.Linear(self.inner_dim, self.out_context_dim, bias=out_bias) | |
| self.processor = processor | |
| def forward( | |
| self, | |
| hidden_states: torch.Tensor, | |
| txt_cu_lens: torch.Tensor = None, | |
| img_cu_lens: torch.Tensor = None, | |
| image_rotary_emb: torch.Tensor = None, | |
| **attention_kwargs, | |
| ) -> torch.Tensor: | |
| return self.processor( | |
| self, | |
| hidden_states, | |
| txt_cu_lens=txt_cu_lens, | |
| img_cu_lens=img_cu_lens, | |
| image_rotary_emb=image_rotary_emb, | |
| **attention_kwargs, | |
| ) | |
| class MageDoubleStreamAttnProcessor: | |
| """Attention processor for the Mage double-stream architecture. Implements | |
| joint attention where the packed text and image streams are concatenated | |
| per sample (order: [text, image]) and run through one varlen kernel.""" | |
| def __call__( | |
| self, | |
| attn: Attention, | |
| hidden_states: torch.FloatTensor, # Image stream [1, sum_img, D] | |
| img_cu_lens: torch.LongTensor, | |
| encoder_hidden_states: torch.FloatTensor = None, # Text stream [1, sum_txt, D] | |
| txt_cu_lens: torch.LongTensor = None, | |
| image_rotary_emb: torch.Tensor = None, | |
| **kwargs, | |
| ) -> torch.FloatTensor: | |
| if encoder_hidden_states is None: | |
| raise ValueError( | |
| "MageDoubleStreamAttnProcessor requires encoder_hidden_states (text stream)" | |
| ) | |
| # Compute QKV for image stream (sample projections) | |
| img_query = attn.to_q(hidden_states) | |
| img_key = attn.to_k(hidden_states) | |
| img_value = attn.to_v(hidden_states) | |
| # Compute QKV for text stream (context projections) | |
| txt_query = attn.add_q_proj(encoder_hidden_states) | |
| txt_key = attn.add_k_proj(encoder_hidden_states) | |
| txt_value = attn.add_v_proj(encoder_hidden_states) | |
| # Reshape for multi-head attention | |
| img_query = img_query.unflatten(-1, (attn.heads, -1)) | |
| img_key = img_key.unflatten(-1, (attn.heads, -1)) | |
| img_value = img_value.unflatten(-1, (attn.heads, -1)) | |
| txt_query = txt_query.unflatten(-1, (attn.heads, -1)) | |
| txt_key = txt_key.unflatten(-1, (attn.heads, -1)) | |
| txt_value = txt_value.unflatten(-1, (attn.heads, -1)) | |
| if img_query.ndim == 4: | |
| img_query = img_query.flatten(0, 1) | |
| img_key = img_key.flatten(0, 1) | |
| img_value = img_value.flatten(0, 1) | |
| if txt_query.ndim == 4: | |
| txt_query = txt_query.flatten(0, 1) | |
| txt_key = txt_key.flatten(0, 1) | |
| txt_value = txt_value.flatten(0, 1) | |
| # Apply QK normalization | |
| img_query = attn.norm_q(img_query) | |
| img_key = attn.norm_k(img_key) | |
| txt_query = attn.norm_added_q(txt_query) | |
| txt_key = attn.norm_added_k(txt_key) | |
| # Apply 2D multi-scale RoPE (MageFlowEmbedRope) to image tokens | |
| img_freqs = image_rotary_emb | |
| img_query = apply_rotary_emb_mageflow(img_query, img_freqs) | |
| img_key = apply_rotary_emb_mageflow(img_key, img_freqs) | |
| # Calculate lengths | |
| img_lens = img_cu_lens[1:] - img_cu_lens[:-1] | |
| txt_lens = txt_cu_lens[1:] - txt_cu_lens[:-1] | |
| # Calculate joint cu_seqlens | |
| joint_lens = txt_lens + img_lens | |
| joint_cu_lens = torch.cat( | |
| [ | |
| torch.zeros(1, dtype=torch.int32, device=joint_lens.device), | |
| torch.cumsum(joint_lens, dim=0, dtype=torch.int32), | |
| ], | |
| dim=0, | |
| ) | |
| device = joint_lens.device | |
| batch_size = len(txt_lens) | |
| sample_indices = torch.arange(batch_size, device=device) | |
| txt_sample_ids = torch.repeat_interleave(sample_indices, txt_lens) | |
| img_sample_ids = torch.repeat_interleave(sample_indices, img_lens) | |
| txt_intra_pos = ( | |
| torch.arange(txt_query.shape[0], device=device) | |
| - txt_cu_lens[txt_sample_ids] | |
| ) | |
| img_intra_pos = ( | |
| torch.arange(img_query.shape[0], device=device) | |
| - img_cu_lens[img_sample_ids] | |
| ) | |
| txt_dest_indices = joint_cu_lens[txt_sample_ids] + txt_intra_pos | |
| img_dest_indices = ( | |
| joint_cu_lens[img_sample_ids] + txt_lens[img_sample_ids] + img_intra_pos | |
| ) | |
| total_tokens = joint_cu_lens[-1] | |
| joint_query = torch.empty( | |
| (total_tokens, *txt_query.shape[1:]), dtype=txt_query.dtype, device=device | |
| ) | |
| joint_key = torch.empty( | |
| (total_tokens, *txt_key.shape[1:]), dtype=txt_key.dtype, device=device | |
| ) | |
| joint_value = torch.empty( | |
| (total_tokens, *txt_value.shape[1:]), dtype=txt_value.dtype, device=device | |
| ) | |
| joint_query[txt_dest_indices] = txt_query | |
| joint_query[img_dest_indices] = img_query | |
| joint_key[txt_dest_indices] = txt_key | |
| joint_key[img_dest_indices] = img_key | |
| joint_value[txt_dest_indices] = txt_value | |
| joint_value[img_dest_indices] = img_value | |
| max_seqlen = joint_lens.max().item() | |
| joint_attn_output = flash_attn_varlen_func( | |
| joint_query, | |
| joint_key, | |
| joint_value, | |
| cu_seqlens_q=joint_cu_lens, | |
| cu_seqlens_k=joint_cu_lens, | |
| max_seqlen_q=max_seqlen, | |
| max_seqlen_k=max_seqlen, | |
| dropout_p=0.0, | |
| softmax_scale=None, | |
| causal=False, | |
| ) | |
| txt_attn_output = joint_attn_output[txt_dest_indices] | |
| img_attn_output = joint_attn_output[img_dest_indices] | |
| img_attn_output = img_attn_output.flatten(1, 2) # (N, H, D) -> (N, H*D) | |
| img_attn_output = img_attn_output.to(joint_query.dtype) | |
| txt_attn_output = txt_attn_output.flatten(1, 2) # (N, H, D) -> (N, H*D) | |
| txt_attn_output = txt_attn_output.to(joint_query.dtype) | |
| img_attn_output = attn.to_out[0](img_attn_output) | |
| img_attn_output = attn.to_out[1](img_attn_output) # dropout | |
| txt_attn_output = attn.to_add_out(txt_attn_output) | |
| txt_attn_output = txt_attn_output.view( | |
| encoder_hidden_states.shape[0], | |
| encoder_hidden_states.shape[1], | |
| txt_attn_output.shape[-1], | |
| ) | |
| return img_attn_output, txt_attn_output | |
| class MageFlowTransformerBlock(nn.Module): | |
| def __init__( | |
| self, | |
| dim: int, | |
| num_attention_heads: int, | |
| attention_head_dim: int, | |
| eps: float = 1e-6, | |
| ): | |
| super().__init__() | |
| self.dim = dim | |
| self.num_attention_heads = num_attention_heads | |
| self.attention_head_dim = attention_head_dim | |
| # Image processing modules | |
| self.img_mod = nn.Sequential( | |
| nn.SiLU(), | |
| nn.Linear( | |
| dim, 6 * dim, bias=True | |
| ), # For scale, shift, gate for norm1 and norm2 | |
| ) | |
| self.img_norm1 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps) | |
| self.attn = Attention( | |
| query_dim=dim, | |
| added_kv_proj_dim=dim, # Enable added KV projections for text stream | |
| dim_head=attention_head_dim, | |
| heads=num_attention_heads, | |
| out_dim=dim, | |
| bias=True, | |
| processor=MageDoubleStreamAttnProcessor(), | |
| eps=eps, | |
| ) | |
| self.img_norm2 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps) | |
| self.img_mlp = FeedForward( | |
| dim=dim, dim_out=dim, activation_fn="gelu-approximate" | |
| ) | |
| # Text processing modules | |
| self.txt_mod = nn.Sequential( | |
| nn.SiLU(), | |
| nn.Linear( | |
| dim, 6 * dim, bias=True | |
| ), # For scale, shift, gate for norm1 and norm2 | |
| ) | |
| self.txt_norm1 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps) | |
| # Text doesn't need separate attention - it's handled by img attn joint computation | |
| self.txt_norm2 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps) | |
| self.txt_mlp = FeedForward( | |
| dim=dim, dim_out=dim, activation_fn="gelu-approximate" | |
| ) | |
| def _modulate(self, x, mod_params, cu_lens=None): | |
| """Apply modulation to input tensor""" | |
| shift, scale, gate = mod_params.chunk(3, dim=-1) | |
| if cu_lens is not None: | |
| assert x.shape[0] == 1, "x must be of shape (1, *) when cu_lens is not None" | |
| x_flattened = x.view(-1, x.shape[-1]) | |
| lengths = cu_lens[1:] - cu_lens[:-1] | |
| shift_t = shift.repeat_interleave(lengths, dim=0) | |
| scale_t = scale.repeat_interleave(lengths, dim=0) | |
| gate_t = gate.repeat_interleave(lengths, dim=0) | |
| x_flattened = x_flattened * (1 + scale_t) + shift_t | |
| x = x_flattened.view(x.shape) | |
| return x, gate_t | |
| else: | |
| return x * (1 + scale) + shift, gate | |
| def forward( | |
| self, | |
| hidden_states: torch.Tensor, | |
| encoder_hidden_states: torch.Tensor, | |
| temb: torch.Tensor, | |
| image_rotary_emb: torch.Tensor, | |
| txt_cu_lens: torch.Tensor, | |
| img_cu_lens: torch.Tensor, | |
| joint_attention_kwargs: dict = None, | |
| ): | |
| # Get modulation parameters for both streams | |
| img_mod_params = self.img_mod(temb) # [B, 6*dim] | |
| txt_mod_params = self.txt_mod(temb) # [B, 6*dim] | |
| # Split modulation parameters for norm1 and norm2 | |
| img_mod1, img_mod2 = img_mod_params.chunk(2, dim=-1) # Each [B, 3*dim] | |
| txt_mod1, txt_mod2 = txt_mod_params.chunk(2, dim=-1) # Each [B, 3*dim] | |
| # Process image stream - norm1 + modulation | |
| img_normed = self.img_norm1(hidden_states) | |
| img_modulated, img_gate1 = self._modulate( | |
| img_normed, img_mod1, cu_lens=img_cu_lens | |
| ) | |
| # Process text stream - norm1 + modulation | |
| txt_normed = self.txt_norm1(encoder_hidden_states) | |
| txt_modulated, txt_gate1 = self._modulate( | |
| txt_normed, txt_mod1, cu_lens=txt_cu_lens | |
| ) | |
| # Joint attention: computes QKV for both streams, applies QK norm and | |
| # RoPE, concatenates per sample and splits the results back. | |
| joint_attention_kwargs = joint_attention_kwargs or {} | |
| attn_output = self.attn( | |
| hidden_states=img_modulated, | |
| encoder_hidden_states=txt_modulated, | |
| image_rotary_emb=image_rotary_emb, | |
| txt_cu_lens=txt_cu_lens, | |
| img_cu_lens=img_cu_lens, | |
| **joint_attention_kwargs, | |
| ) | |
| img_attn_output, txt_attn_output = attn_output | |
| # Apply attention gates and add residual | |
| hidden_states = hidden_states + img_gate1 * img_attn_output | |
| encoder_hidden_states = encoder_hidden_states + txt_gate1 * txt_attn_output | |
| # Process image stream - norm2 + MLP | |
| img_normed2 = self.img_norm2(hidden_states) | |
| img_modulated2, img_gate2 = self._modulate( | |
| img_normed2, img_mod2, cu_lens=img_cu_lens | |
| ) | |
| img_mlp_output = self.img_mlp(img_modulated2) | |
| hidden_states = hidden_states + img_gate2 * img_mlp_output | |
| # Process text stream - norm2 + MLP | |
| txt_normed2 = self.txt_norm2(encoder_hidden_states) | |
| txt_modulated2, txt_gate2 = self._modulate( | |
| txt_normed2, txt_mod2, cu_lens=txt_cu_lens | |
| ) | |
| txt_mlp_output = self.txt_mlp(txt_modulated2) | |
| encoder_hidden_states = encoder_hidden_states + txt_gate2 * txt_mlp_output | |
| # Clip to prevent overflow for fp16 | |
| if encoder_hidden_states.dtype == torch.float16: | |
| encoder_hidden_states = encoder_hidden_states.clip(-65504, 65504) | |
| if hidden_states.dtype == torch.float16: | |
| hidden_states = hidden_states.clip(-65504, 65504) | |
| return encoder_hidden_states, hidden_states | |
| class AdaLayerNormContinuous(nn.Module): | |
| """Adaptive output norm; supports packed sequences via ``cu_seqlens``.""" | |
| def __init__( | |
| self, | |
| embedding_dim: int, | |
| conditioning_embedding_dim: int, | |
| elementwise_affine=True, | |
| eps=1e-5, | |
| bias=True, | |
| norm_type="layer_norm", | |
| ): | |
| super().__init__() | |
| self.silu = nn.SiLU() | |
| self.linear = nn.Linear( | |
| conditioning_embedding_dim, embedding_dim * 2, bias=bias | |
| ) | |
| if norm_type == "layer_norm": | |
| self.norm = nn.LayerNorm(embedding_dim, eps, elementwise_affine, bias) | |
| elif norm_type == "rms_norm": | |
| self.norm = RMSNorm(embedding_dim, eps, elementwise_affine) | |
| else: | |
| raise ValueError(f"unknown norm_type {norm_type}") | |
| def forward( | |
| self, | |
| x: torch.Tensor, | |
| conditioning_embedding: torch.Tensor, | |
| cu_seqlens: torch.Tensor = None, | |
| ) -> torch.Tensor: | |
| emb = self.linear(self.silu(conditioning_embedding).to(x.dtype)) | |
| if cu_seqlens is None: | |
| scale, shift = torch.chunk(emb, 2, dim=-1) | |
| x = self.norm(x) * (1 + scale) + shift | |
| else: | |
| sample_lens = cu_seqlens[1:] - cu_seqlens[:-1] | |
| flattened_x = x.view(-1, x.shape[-1]) | |
| scale, shift = torch.chunk(emb, 2, dim=-1) | |
| scale_t = torch.repeat_interleave(scale, sample_lens, dim=0) | |
| shift_t = torch.repeat_interleave(shift, sample_lens, dim=0) | |
| flattened_x = self.norm(flattened_x) * (1 + scale_t) + shift_t | |
| x = flattened_x.view(x.shape) | |
| return x | |
| class MageFlowParams: | |
| in_channels: int | |
| out_channels: int | |
| context_in_dim: int | |
| hidden_size: int | |
| num_heads: int | |
| depth: int | |
| axes_dim: list | |
| checkpoint: bool | |
| patch_size: int = 1 | |
| class MageFlow(nn.Module, OstrisModelMixin): | |
| def get_transformer_block_names(cls): | |
| return ["transformer_blocks"] | |
| def get_quantization_exclude_modules(cls): | |
| # sensitive modules kept in full precision (fnmatch patterns): | |
| # img_in / txt_in / txt_norm - input projections | |
| # time_text_embed* - timestep embedder feeding every | |
| # block's modulation | |
| # norm_out* / proj_out - final adaptive norm / output projection | |
| return [ | |
| "img_in", | |
| "txt_in", | |
| "txt_norm", | |
| "time_text_embed*", | |
| "norm_out*", | |
| "proj_out", | |
| ] | |
| def __init__(self, params: MageFlowParams): | |
| super().__init__() | |
| self.params = params | |
| self.gradient_checkpointing = bool(params.checkpoint) | |
| self.in_channels = params.in_channels | |
| self.out_channels = params.out_channels | |
| self.inner_dim = params.hidden_size # num_attention_heads * attention_head_dim | |
| self.axes_dim = params.axes_dim | |
| self.num_attention_heads = params.num_heads | |
| self.attention_head_dim = self.inner_dim // self.num_attention_heads | |
| self.patch_size = params.patch_size | |
| assert sum(self.axes_dim) == self.attention_head_dim | |
| self.pos_embed = MageFlowEmbedRope( | |
| theta=10000, axes_dim=self.axes_dim, scale_rope=True | |
| ) | |
| self.img_in = nn.Linear(self.in_channels, self.inner_dim) | |
| self.txt_norm = RMSNorm(params.context_in_dim, eps=1e-6) | |
| self.txt_in = nn.Linear(params.context_in_dim, self.inner_dim) | |
| self.time_text_embed = MageFlowTimestepProjEmbeddings( | |
| embedding_dim=self.inner_dim | |
| ) | |
| self.transformer_blocks = nn.ModuleList( | |
| [ | |
| MageFlowTransformerBlock( | |
| dim=self.inner_dim, | |
| num_attention_heads=self.num_attention_heads, | |
| attention_head_dim=self.attention_head_dim, | |
| ) | |
| for _ in range(params.depth) | |
| ] | |
| ) | |
| self.norm_out = AdaLayerNormContinuous( | |
| self.inner_dim, self.inner_dim, elementwise_affine=False, eps=1e-6 | |
| ) | |
| self.proj_out = nn.Linear( | |
| self.inner_dim, | |
| self.patch_size * self.patch_size * self.out_channels, | |
| bias=True, | |
| ) | |
| def enable_gradient_checkpointing(self, enable: bool = True): | |
| self.gradient_checkpointing = enable | |
| def disable_gradient_checkpointing(self): | |
| self.gradient_checkpointing = False | |
| def reset_rope(self): | |
| """Rebuild the RoPE frequency tables. Needed after a meta-device init | |
| (the tables are plain tensors, not buffers, so ``load_state_dict`` / | |
| ``assign=True`` never materializes them).""" | |
| self.pos_embed = MageFlowEmbedRope( | |
| theta=10000, axes_dim=self.axes_dim, scale_rope=True | |
| ) | |
| def device(self): | |
| return next(self.parameters()).device | |
| def dtype(self): | |
| return next(self.parameters()).dtype | |
| def forward( | |
| self, | |
| img: Tensor, | |
| txt: Tensor, | |
| timesteps: Tensor, | |
| img_shapes=None, | |
| img_cu_seqlens: Tensor = None, | |
| txt_cu_seqlens: Tensor = None, | |
| attention_kwargs: dict = None, | |
| ) -> Tensor: | |
| if img.ndim != 3 or txt.ndim != 3: | |
| raise ValueError("Input img and txt tensors must have 3 dimensions.") | |
| # Prepare vision RoPE (msrope); text tokens are not rotated. | |
| ms_pe = self.pos_embed(img_shapes, device=img.device) | |
| img = self.img_in(img) | |
| txt = self.txt_norm(txt) | |
| timesteps = timesteps.to(img.dtype) | |
| temb = self.time_text_embed(timesteps, img) | |
| txt = self.txt_in(txt) | |
| # The reference adds a zero "text vector" to temb (vec_type is null in | |
| # the released config); kept for parity with the reference forward. | |
| txt_vec = torch.zeros( | |
| txt.shape[0], self.inner_dim, dtype=txt.dtype, device=txt.device | |
| ) | |
| temb = temb + txt_vec | |
| attention_kwargs = attention_kwargs or {} | |
| for block in self.transformer_blocks: | |
| if self.gradient_checkpointing and torch.is_grad_enabled(): | |
| txt, img = checkpoint( | |
| block, | |
| img, # hidden_states | |
| txt, # encoder_hidden_states | |
| temb, # temb | |
| ms_pe, # image_rotary_emb | |
| txt_cu_seqlens, # txt_cu_lens | |
| img_cu_seqlens, # img_cu_lens | |
| use_reentrant=False, | |
| ) | |
| else: | |
| txt, img = block( | |
| hidden_states=img, | |
| encoder_hidden_states=txt, | |
| txt_cu_lens=txt_cu_seqlens, | |
| img_cu_lens=img_cu_seqlens, | |
| temb=temb, | |
| image_rotary_emb=ms_pe, | |
| joint_attention_kwargs=attention_kwargs, | |
| ) | |
| # Use only the image part (hidden_states) from the dual-stream blocks | |
| img = self.norm_out(img, temb, cu_seqlens=img_cu_seqlens) | |
| img = self.proj_out(img) | |
| return img | |
Xet Storage Details
- Size:
- 30.7 kB
- Xet hash:
- b56da82ae6a453ed92ffae622eca110a6b97525156469a8b3a24bda2ef0908b9
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.