| from transformers import CONFIG_MAPPING, AutoConfig |
| from transformers.configuration_utils import PretrainedConfig |
|
|
|
|
| class LlavaOnevision2VisionConfig(PretrainedConfig): |
| model_type = "llava_onevision2" |
| base_config_key = "vision_config" |
|
|
| def __init__( |
| self, |
| hidden_size=1024, |
| intermediate_size=4096, |
| num_hidden_layers=24, |
| num_attention_heads=16, |
| num_channels=3, |
| image_size=448, |
| patch_size=14, |
| hidden_act="gelu", |
| layer_norm_eps=1e-6, |
| layer_norm_type="layer_norm", |
| attention_dropout=0.0, |
| initializer_range=0.02, |
| rope_theta=10000.0, |
| use_head=False, |
| out_hidden_size=1024, |
| spatial_merge_size=2, |
| tokens_per_second=1, |
| temporal_patch_size=1, |
| frame_windows_size=4, |
| use_patch_position_encoding=False, |
| patch_position_encoding_type="absolute", |
| max_position_embeddings=8192, |
| **kwargs, |
| ): |
| super().__init__(**kwargs) |
| self.hidden_size = hidden_size |
| self.intermediate_size = intermediate_size |
| self.num_hidden_layers = num_hidden_layers |
| self.num_attention_heads = num_attention_heads |
| self.num_channels = num_channels |
| self.image_size = image_size |
| self.patch_size = patch_size |
| self.hidden_act = hidden_act |
| self.layer_norm_eps = layer_norm_eps |
| self.layer_norm_type = layer_norm_type |
| self.attention_dropout = attention_dropout |
| self.initializer_range = initializer_range |
| self.rope_theta = rope_theta |
| self.use_head = use_head |
| self.out_hidden_size = out_hidden_size |
| self.spatial_merge_size = spatial_merge_size |
| self.tokens_per_second = tokens_per_second |
| self.temporal_patch_size = temporal_patch_size |
| self.frame_windows_size = frame_windows_size |
| self.use_patch_position_encoding = use_patch_position_encoding |
| self.patch_position_encoding_type = patch_position_encoding_type |
| self.max_position_embeddings = max_position_embeddings |
|
|
|
|
| class LlavaOnevision2Config(PretrainedConfig): |
| r""" |
| This is the configuration class to store the configuration of a [`LlavaOnevision2Model`] for `microsoft/Mage-VL-Cowork`. It is used to instantiate a |
| LlavaOnevision2Model model according to the specified arguments, defining the model architecture. |
| |
| Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the |
| documentation from [`PretrainedConfig`] for more information. |
| |
| Args: |
| text_config (`Union[PreTrainedConfig, dict]`, *optional*, defaults to `Qwen3Config`): |
| The config object or dictionary of the text backbone. |
| vision_config (`Union[PreTrainedConfig, dict]`, *optional*, defaults to `LlavaOnevision2VisionConfig`): |
| The config object or dictionary of the vision backbone. |
| image_token_id (`int`, *optional*, defaults to 151655): |
| The image token index to encode the image prompt. |
| video_token_id (`int`, *optional*, defaults to 151656): |
| The video token index to encode the image prompt. |
| vision_start_token_id (`int`, *optional*, defaults to 151652): |
| The token index to denote start of vision input. |
| vision_end_token_id (`int`, *optional*, defaults to 151653): |
| The token index to denote end of vision input. |
| |
| ```python |
| >>> from transformers import LlavaOnevision2Model, LlavaOnevision2Config |
| |
| >>> # Initializing a LlavaOnevision2 style configuration |
| >>> configuration = LlavaOnevision2Config() |
| |
| >>> # Initializing a model from the Llava-Onevision-1.5-8B style configuration |
| >>> model = LlavaOnevision2Model(configuration) |
| |
| >>> # Accessing the model configuration |
| >>> configuration = model.config |
| ```""" |
|
|
| model_type = "llava_onevision2" |
| sub_configs = {"vision_config": LlavaOnevision2VisionConfig, "text_config": AutoConfig} |
| keys_to_ignore_at_inference = ["past_key_values"] |
|
|
| def __init__( |
| self, |
| text_config=None, |
| vision_config=None, |
| image_token_id=151655, |
| video_token_id=151656, |
| vision_start_token_id=151652, |
| vision_end_token_id=151653, |
| **kwargs, |
| ): |
| |
| |
| |
| super().__init__(**kwargs) |
| if isinstance(text_config, dict): |
| text_config["model_type"] = text_config.get("model_type", "qwen3") |
| self.sub_configs["text_config"] = CONFIG_MAPPING[text_config["model_type"]] |
| elif text_config is None: |
| self.sub_configs["text_config"] = CONFIG_MAPPING["qwen3"] |
|
|
| if isinstance(vision_config, dict): |
| self.vision_config = self.sub_configs["vision_config"](**vision_config) |
| elif vision_config is None: |
| self.vision_config = self.sub_configs["vision_config"]() |
|
|
| if isinstance(text_config, dict): |
| self.text_config = self.sub_configs["text_config"](**text_config) |
| elif text_config is None: |
| |
| self.text_config = self.sub_configs["text_config"](**kwargs) |
|
|
| self.image_token_id = image_token_id |
| self.video_token_id = video_token_id |
| self.vision_start_token_id = vision_start_token_id |
| self.vision_end_token_id = vision_end_token_id |
|
|
| |
| self._attn_implementation = kwargs.pop("attn_implementation", None) |
|
|
| def __setattr__(self, key, value): |
| if ( |
| (text_config := super().__getattribute__("__dict__").get("text_config")) is not None |
| and key not in ["dtype", "_attn_implementation_internal"] |
| and key in text_config.__dict__ |
| ): |
| setattr(text_config, key, value) |
| else: |
| super().__setattr__(key, value) |
|
|
| def __getattribute__(self, key): |
| if "text_config" in super().__getattribute__("__dict__") and key not in [ |
| "_name_or_path", |
| "model_type", |
| "dtype", |
| "_attn_implementation_internal", |
| ]: |
| text_config = super().__getattribute__("text_config") |
| if key in text_config.__dict__: |
| return getattr(text_config, key) |
|
|
| return super().__getattribute__(key) |
|
|
|
|
| __all__ = ["LlavaOnevision2Config"] |
|
|
