Add Ovis2 model and processor implementation

docs/source/en/_toctree.yml

@@ -977,6 +977,8 @@
         title: OneFormer
       - local: model_doc/owlvit
         title: OWL-ViT
+      - local: model_doc/ovis2
+        title: Ovis2
       - local: model_doc/owlv2
         title: OWLv2
       - local: model_doc/paligemma

docs/source/en/model_doc/ovis2.md

@@ -0,0 +1,55 @@
+# Ovis2
+
+## Overview
+
+The [Ovis2](https://github.com/AIDC-AI/Ovis) is an updated version of the [Ovis](https://arxiv.org/abs/2405.20797) model developed by the AIDC-AI team at Alibaba International Digital Commerce Group. 
+
+The abstract from this update is the following:
+
+*It brings major improvements, including better performance for small models, stronger reasoning ability, advanced video and multi-image processing, wider multilingual OCR support, and improved handling of high-resolution images.*
+
+
+```python
+
+from PIL import Image
+import requests
+import torch
+from torchvision import io
+from typing import Dict
+from transformers.image_utils import load_images, load_video
+from transformers import AutoModelForVision2Seq, AutoTokenizer, AutoProcessor
+
+model = AutoModelForVision2Seq.from_pretrained(
+    "thisisiron/Ovis2-2B-hf",
+    torch_dtype=torch.bfloat16,
+).eval().to("cuda:0")
+processor = AutoProcessor.from_pretrained("thisisiron/Ovis2-2B-hf")
+
+messages = [
+    {
+        "role": "user",
+        "content": [
+            {"type": "image"},
+            {"type": "text", "text": "Describe the image."},
+        ],
+    },
+]
+url = "http://images.cocodataset.org/val2014/COCO_val2014_000000537955.jpg"
+image = Image.open(requests.get(url, stream=True).raw)
+messages = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+print(messages)
+
+inputs = processor(
+    images=[image],
+    text=messages,
+    return_tensors="pt",
+)
+inputs = inputs.to("cuda:0")
+inputs['pixel_values'] = inputs['pixel_values'].to(torch.bfloat16)
+
+with torch.inference_mode():
+    output_ids = model.generate(**inputs, max_new_tokens=128, do_sample=False)
+    generated_ids = [output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs.input_ids, output_ids)]
+    output_text = processor.batch_decode(generated_ids, skip_special_tokens=True)
+    print(output_text)
+```

src/transformers/__init__.py

@@ -673,6 +673,11 @@
         "OpenAIGPTTokenizer",
     ],
     "models.opt": ["OPTConfig"],
+    "models.ovis2": [
+        "Ovis2Config",
+        "Ovis2Processor",
+        "Ovis2VisionConfig",
+    ],
     "models.owlv2": [
         "Owlv2Config",
         "Owlv2Processor",
@@ -1303,6 +1308,7 @@
     _import_structure["models.mobilevit"].extend(["MobileViTFeatureExtractor", "MobileViTImageProcessor"])
     _import_structure["models.nougat"].append("NougatImageProcessor")
     _import_structure["models.oneformer"].extend(["OneFormerImageProcessor"])
+    _import_structure["models.ovis2"].extend(["Ovis2ImageProcessor"])
     _import_structure["models.owlv2"].append("Owlv2ImageProcessor")
     _import_structure["models.owlvit"].extend(["OwlViTFeatureExtractor", "OwlViTImageProcessor"])
     _import_structure["models.perceiver"].extend(["PerceiverFeatureExtractor", "PerceiverImageProcessor"])
@@ -3231,6 +3237,13 @@
             "OPTPreTrainedModel",
         ]
     )
+    _import_structure["models.ovis2"].extend(
+        [
+            "Ovis2ForConditionalGeneration",
+            "Ovis2VisionModel",
+            "Ovis2PreTrainedModel",
+        ]
+    )
     _import_structure["models.owlv2"].extend(
         [
             "Owlv2ForObjectDetection",
@@ -5933,6 +5946,11 @@
         OpenAIGPTTokenizer,
     )
     from .models.opt import OPTConfig
+    from .models.ovis2 import (
+        Ovis2Config,
+        Ovis2Processor,
+        Ovis2VisionConfig,
+    )
     from .models.owlv2 import (
         Owlv2Config,
         Owlv2Processor,
@@ -8144,6 +8162,9 @@
             OPTModel,
             OPTPreTrainedModel,
         )
+        from .models.ovis2 import (
+            Ovis2ForConditionalGeneration,
+        )
         from .models.owlv2 import (
             Owlv2ForObjectDetection,
             Owlv2Model,

src/transformers/models/__init__.py

@@ -202,6 +202,7 @@
     oneformer,
     openai,
     opt,
+    ovis2,
     owlv2,
     owlvit,
     paligemma,

src/transformers/models/auto/configuration_auto.py

@@ -225,6 +225,7 @@
         ("open-llama", "OpenLlamaConfig"),
         ("openai-gpt", "OpenAIGPTConfig"),
         ("opt", "OPTConfig"),
+        ("ovis2", "Ovis2Config"),
         ("owlv2", "Owlv2Config"),
         ("owlvit", "OwlViTConfig"),
         ("paligemma", "PaliGemmaConfig"),
@@ -582,6 +583,7 @@
         ("open-llama", "OpenLlama"),
         ("openai-gpt", "OpenAI GPT"),
         ("opt", "OPT"),
+        ("ovis2", "Ovis2"),
         ("owlv2", "OWLv2"),
         ("owlvit", "OWL-ViT"),
         ("paligemma", "PaliGemma"),

src/transformers/models/auto/image_processing_auto.py

@@ -120,6 +120,7 @@
             ("nat", ("ViTImageProcessor", "ViTImageProcessorFast")),
             ("nougat", ("NougatImageProcessor",)),
             ("oneformer", ("OneFormerImageProcessor",)),
+            ("ovis2", "Ovis2ImageProcessor"),
             ("owlv2", ("Owlv2ImageProcessor",)),
             ("owlvit", ("OwlViTImageProcessor",)),
             ("paligemma", ("SiglipImageProcessor", "SiglipImageProcessorFast")),

src/transformers/models/auto/modeling_auto.py

@@ -816,6 +816,7 @@
         ("llava_onevision", "LlavaOnevisionForConditionalGeneration"),
         ("mistral3", "Mistral3ForConditionalGeneration"),
         ("mllama", "MllamaForConditionalGeneration"),
+        ("ovis2", "Ovis2ForConditionalGeneration"),
         ("paligemma", "PaliGemmaForConditionalGeneration"),
         ("pix2struct", "Pix2StructForConditionalGeneration"),
         ("qwen2_5_vl", "Qwen2_5_VLForConditionalGeneration"),
@@ -854,6 +855,7 @@
         ("llava_onevision", "LlavaOnevisionForConditionalGeneration"),
         ("mistral3", "Mistral3ForConditionalGeneration"),
         ("mllama", "MllamaForConditionalGeneration"),
+        ("ovis2", "Ovis2ForConditionalGeneration"),
         ("paligemma", "PaliGemmaForConditionalGeneration"),
         ("pix2struct", "Pix2StructForConditionalGeneration"),
         ("pixtral", "LlavaForConditionalGeneration"),

src/transformers/models/auto/processing_auto.py

@@ -88,6 +88,7 @@
         ("mllama", "MllamaProcessor"),
         ("moonshine", "Wav2Vec2Processor"),
         ("oneformer", "OneFormerProcessor"),
+        ("ovis2", "Ovis2Processor"),
         ("owlv2", "Owlv2Processor"),
         ("owlvit", "OwlViTProcessor"),
         ("paligemma", "PaliGemmaProcessor"),

src/transformers/models/auto/tokenization_auto.py

@@ -396,6 +396,7 @@
                 "openai-gpt",
                 ("OpenAIGPTTokenizer", "OpenAIGPTTokenizerFast" if is_tokenizers_available() else None),
             ),
+            ("ovis2",  ("Qwen2Tokenizer", "Qwen2TokenizerFast" if is_tokenizers_available() else None)),
             ("opt", ("GPT2Tokenizer", "GPT2TokenizerFast" if is_tokenizers_available() else None)),
             ("owlv2", ("CLIPTokenizer", "CLIPTokenizerFast" if is_tokenizers_available() else None)),
             ("owlvit", ("CLIPTokenizer", "CLIPTokenizerFast" if is_tokenizers_available() else None)),

src/transformers/models/ovis2/__init__.py

@@ -0,0 +1,18 @@
+
+# limitations under the License.
+from typing import TYPE_CHECKING
+
+from ...utils import _LazyModule
+from ...utils.import_utils import define_import_structure
+
+
+if TYPE_CHECKING:
+    from .configuration_ovis2 import *
+    from .image_processing_ovis2 import *
+    from .modeling_ovis2 import *
+    from .processing_ovis2 import *
+else:
+    import sys
+
+    _file = globals()["__file__"]
+    sys.modules[__name__] = _LazyModule(__name__, _file, define_import_structure(_file), module_spec=__spec__)

src/transformers/models/ovis2/configuration_ovis2.py

@@ -0,0 +1,132 @@
+from ...configuration_utils import PretrainedConfig
+from ..qwen2.configuration_qwen2 import Qwen2Config
+
+
+class Ovis2VisionConfig(PretrainedConfig):
+    r"""
+
+    Args:
+        hidden_size (`int`, *optional*, defaults to 768):
+            Dimensionality of the encoder layers and the pooler layer.
+        intermediate_size (`int`, *optional*, defaults to 3072):
+            Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
+        num_hidden_layers (`int`, *optional*, defaults to 12):
+            Number of hidden layers in the Transformer encoder.
+        num_attention_heads (`int`, *optional*, defaults to 12):
+            Number of attention heads for each attention layer in the Transformer encoder.
+        num_channels (`int`, *optional*, defaults to 3):
+            Number of channels in the input images.
+        image_size (`int`, *optional*, defaults to 224):
+            The size (resolution) of each image.
+        patch_size (`int`, *optional*, defaults to 16):
+            The size (resolution) of each patch.
+        hidden_act (`str` or `function`, *optional*, defaults to `"gelu_pytorch_tanh"`):
+            The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+            `"relu"`, `"selu"` and `"gelu_new"` `"quick_gelu"` are supported.
+        layer_norm_eps (`float`, *optional*, defaults to 1e-06):
+            The epsilon used by the layer normalization layers.
+        attention_dropout (`float`, *optional*, defaults to 0.0):
+            The dropout ratio for the attention probabilities.
+
+    Example:
+
+    ```python
+    >>> from transformers import Ovis2VisionConfig, Ovis2VisionModel
+
+    >>> # Initializing a Ovis2VisionConfig with google/ovis2-base-patch16-224 style configuration
+    >>> configuration = Ovis2VisionConfig()
+
+    >>> # Initializing a Ovis2VisionModel (with random weights) from the google/ovis2-base-patch16-224 style configuration
+    >>> model = Ovis2VisionModel(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "ovis2_vision_model"
+    base_config_key = "vision_config"
+
+    def __init__(
+        self,
+        hidden_size: int = 1024,
+        intermediate_size: int = 2816,
+        num_hidden_layers: int = 24,
+        num_attention_heads: int = 8,
+        num_channels: int = 3,
+        image_size: int = 224,
+        patch_size: int = 14,
+        rms_norm_eps: float = 1e-5,
+        attention_dropout: float = 0.0,
+        projection_dropout: float = 0.0,
+        qkv_bias: bool = False,
+        use_bias: bool = False,
+        hidden_act="silu",
+        vocab_size=16384,
+        hidden_stride=1,
+        vision_feature_select_strategy="full",
+        num_visual_indicator_tokens=5,
+        tokenize_function="softmax",
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.num_channels = num_channels
+        self.patch_size = patch_size
+        self.image_size = image_size
+
+        self.attention_dropout = attention_dropout
+        self.hidden_act = hidden_act
+        self.use_bias = use_bias
+        self.qkv_bias = qkv_bias
+        self.rms_norm_eps = rms_norm_eps
+        self.projection_dropout = projection_dropout
+        self.vocab_size = vocab_size
+        self.hidden_stride = hidden_stride
+        self.vision_feature_select_strategy = vision_feature_select_strategy
+        self.num_visual_indicator_tokens = num_visual_indicator_tokens
+        self.tokenize_function = tokenize_function
+
+
+class Ovis2Config(PretrainedConfig):
+    model_type = "ovis2"
+    sub_configs = {"text_config": Qwen2Config, "vision_config": Ovis2VisionConfig}
+
+    def __init__(
+        self,
+        vision_config=None,
+        text_config=None,
+        image_token_id=151665,
+        visual_indicator_token_ids=[151666, 151667, 151668, 151669, 151670],
+        vocab_size=151643,
+        sliding_window=32768,
+        hidden_size=1536,
+        **kwargs,
+    ):
+        if isinstance(vision_config, dict):
+            self.vision_config = Ovis2VisionConfig(**vision_config)
+        elif isinstance(vision_config, Ovis2VisionConfig):
+            self.vision_config = vision_config
+        if vision_config is None:
+            self.vision_config = Ovis2VisionConfig(num_visual_indicator_tokens=len(visual_indicator_token_ids))
+
+        if isinstance(text_config, dict):
+            self.text_config = Qwen2Config(**text_config)
+        elif isinstance(text_config, Qwen2Config):
+            self.text_config = text_config
+        elif text_config is None:
+            self.text_config = Qwen2Config()
+
+        self.vocab_size = vocab_size
+        self.sliding_window = sliding_window
+        self.hidden_size = hidden_size
+
+        self.image_token_id = image_token_id
+        self.visual_indicator_token_ids = visual_indicator_token_ids
+        super().__init__(**kwargs)
+
+
+__all__ = ["Ovis2VisionConfig", "Ovis2Config"]