configuration_lfm3_vl.py

# Vendored from transformers' LFM3 code (not in a release yet) at 69da1b47a11e362fbb15bfe5d5c8da0790c688c5, src/transformers/models/lfm3_vl/configuration_lfm3_vl.py:
# its relative imports made absolute, nothing else changed.
#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
#           This file was automatically generated from src/transformers/models/lfm3_vl/modular_lfm3_vl.py.
#               Do NOT edit this file manually as any edits will be overwritten by the generation of
#             the file from the modular. If any change should be done, please apply the change to the
#                          modular_lfm3_vl.py file directly. One of our CI enforces this.
#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
# Copyright 2026 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from huggingface_hub.dataclasses import strict

from transformers.configuration_utils import PreTrainedConfig
from transformers.utils import auto_docstring
from transformers.models.auto import CONFIG_MAPPING, AutoConfig


@auto_docstring(checkpoint="LiquidAI/LFM3-VL-1.9B")
@strict
class Lfm3VlVisionConfig(PreTrainedConfig):
    r"""
    layer_types (`list[str]`, *optional*):
        Per-layer operator type, each one of `"full_attention"` or `"conv"`. Defaults to two gated convolution
        layers followed by alternating attention and convolution layers.
    patch_size (`int`, *optional*, defaults to 16):
        The spatial size of each image patch.
    temporal_patch_size (`int`, *optional*, defaults to 2):
        The number of video frames packed into one patch.
    spatial_merge_size (`int`, *optional*, defaults to 2):
        The side of the square block of patches merged into one language token.
    conv_kernel_size (`int`, *optional*, defaults to 3):
        The kernel size of the depthwise 2D convolution layers.
    norm_eps (`float`, *optional*, defaults to 1e-05):
        The epsilon used by the RMS normalization layers.
    """

    model_type = "lfm3_vl_vision"
    base_config_key = "vision_config"
    default_theta = 1_000_000.0
    default_rope_type = "axial"
    _attention_layer_types = ("full_attention",)

    hidden_size: int = 1024
    intermediate_size: int = 2560
    num_hidden_layers: int = 14
    num_attention_heads: int = 16
    num_key_value_heads: int = 8
    layer_types: list[str] | None = None
    patch_size: int = 16
    temporal_patch_size: int = 2
    in_channels: int = 3
    spatial_merge_size: int = 2
    conv_kernel_size: int = 3
    norm_eps: float = 1e-5
    initializer_range: float = 0.02
    rope_parameters: dict | None = None

    def __post_init__(self, **kwargs):
        if self.layer_types is None:
            self.layer_types = ["conv" if i < 2 or i % 2 else "full_attention" for i in range(self.num_hidden_layers)]
        super().__post_init__(**kwargs)

    def validate_architecture(self):
        """Part of ``@strict``-powered validation."""
        if len(self.layer_types) != self.num_hidden_layers:
            raise ValueError(
                f"`layer_types` has {len(self.layer_types)} entries but `num_hidden_layers` is {self.num_hidden_layers}."
            )
        allowed_layer_types = (*self._attention_layer_types, "conv")
        invalid_layer_types = [layer_type for layer_type in self.layer_types if layer_type not in allowed_layer_types]
        if invalid_layer_types:
            raise ValueError(f"`layer_types` entries must be one of {allowed_layer_types}; got {invalid_layer_types}.")
        if self.hidden_size % self.num_attention_heads:
            raise ValueError("`hidden_size` must be a multiple of `num_attention_heads`.")


# The first checkpoints spell every text and `vision_*` field at the top level of the config
_LEGACY_TEXT_KEYS = (
    "vocab_size",
    "hidden_size",
    "intermediate_size",
    "num_hidden_layers",
    "num_attention_heads",
    "num_key_value_heads",
    "head_dim",
    "layer_types",
    "conv_bias",
    "conv_L_cache",
    "sliding_window",
    "max_position_embeddings",
    "use_sandwich_norm",
    "use_cache",
)
_LEGACY_VISION_KEYS = {
    "vision_hidden_size": "hidden_size",
    "vision_intermediate_size": "intermediate_size",
    "vision_num_hidden_layers": "num_hidden_layers",
    "vision_num_attention_heads": "num_attention_heads",
    "vision_num_key_value_heads": "num_key_value_heads",
    "vision_layer_types": "layer_types",
    "vision_patch_size": "patch_size",
    "vision_temporal_patch_size": "temporal_patch_size",
    "vision_num_channels": "in_channels",
    "vision_spatial_merge_size": "spatial_merge_size",
}
_LEGACY_DROPPED_KEYS = ("vision_patch_dim", "vision_max_position_embeddings")


@auto_docstring(checkpoint="LiquidAI/LFM3-VL-1.9B")
@strict
class Lfm3VlConfig(PreTrainedConfig):
    r"""
    video_token_id (`int`, *optional*, defaults to 125029):
        The id of the placeholder token for video patches.
    vision_start_token_id (`int`, *optional*, defaults to 125009):
        The id of the token opening an image or a video frame.
    vision_end_token_id (`int`, *optional*, defaults to 125010):
        The id of the token closing an image or a video frame.
    projector_layer_norm_eps (`float`, *optional*, defaults to 1e-06):
        The epsilon of the layer normalization applied to the vision features before projection.
    downsample_factor (`int`, *optional*, defaults to 2):
        The side of the square block of vision tokens merged by the projector, equal to the vision encoder's
        `spatial_merge_size`.
    """

    model_type = "lfm3_vl"
    sub_configs = {"text_config": AutoConfig, "vision_config": Lfm3VlVisionConfig}

    text_config: dict | PreTrainedConfig | None = None
    vision_config: dict | PreTrainedConfig | None = None
    image_token_id: int = 125_030
    video_token_id: int = 125_029
    vision_start_token_id: int = 125_009
    vision_end_token_id: int = 125_010
    projector_hidden_act: str = "gelu"
    projector_hidden_size: int = 4096
    projector_bias: bool = True
    projector_layer_norm_eps: float = 1e-6
    downsample_factor: int = 2
    tie_word_embeddings: bool = True

    def __post_init__(self, **kwargs):
        if "vision_hidden_size" in kwargs:
            kwargs = self._convert_legacy_flat_config(kwargs)

        if isinstance(self.vision_config, dict):
            self.vision_config = self.sub_configs["vision_config"](**self.vision_config)
        elif self.vision_config is None:
            self.vision_config = self.sub_configs["vision_config"]()

        if isinstance(self.text_config, dict):
            self.text_config = {"model_type": "lfm3", **self.text_config}
            self.text_config = CONFIG_MAPPING[self.text_config["model_type"]](**self.text_config)
        elif self.text_config is None:
            self.text_config = CONFIG_MAPPING["lfm3"]()
        self.validate_vision_fields()

        super().__post_init__(**kwargs)

    def validate_vision_fields(self):
        """Part of `@strict`-powered validation, so `validate()` and `save_pretrained` refuse later edits too."""
        if self.downsample_factor != self.vision_config.spatial_merge_size:
            raise ValueError("downsample_factor must match vision_config.spatial_merge_size")
        # an `inputs_embeds` forward embeds both placeholder ids to find them
        vocab_size = self.text_config.vocab_size
        for name in ("image_token_id", "video_token_id"):
            if not 0 <= getattr(self, name) < vocab_size:
                raise ValueError(f"{name}={getattr(self, name)} must lie in [0, text_config.vocab_size={vocab_size})")

    def _convert_legacy_flat_config(self, kwargs: dict) -> dict:
        text_config = {key: kwargs.pop(key) for key in _LEGACY_TEXT_KEYS if key in kwargs}
        text_config["norm_eps"] = kwargs.pop("rms_norm_eps", 1e-5)
        text_config["rope_parameters"] = {
            "full_attention": {
                "rope_type": "default",
                "rope_theta": kwargs.pop("full_attention_rope_theta", 1_000_000.0),
                "partial_rotary_factor": kwargs.pop("partial_rotary_factor", 0.5),
            },
            "sliding_attention": {
                "rope_type": "default",
                "rope_theta": kwargs.pop("sliding_attention_rope_theta", 10_000.0),
            },
        }
        for key in ("pad_token_id", "bos_token_id", "eos_token_id"):
            if kwargs.get(key) is not None:
                text_config[key] = kwargs[key]
        text_config["tie_word_embeddings"] = self.tie_word_embeddings

        vision_config = {new: kwargs.pop(old) for old, new in _LEGACY_VISION_KEYS.items() if old in kwargs}
        vision_config["norm_eps"] = text_config["norm_eps"]
        vision_config["rope_parameters"] = {"rope_type": "axial", "rope_theta": kwargs.pop("vision_rope_theta", 1e6)}
        for key in _LEGACY_DROPPED_KEYS:
            kwargs.pop(key, None)

        self.text_config = text_config
        self.vision_config = vision_config
        return kwargs


__all__ = ["Lfm3VlConfig", "Lfm3VlVisionConfig"]