Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+images/gme_logo.png filter=lfs diff=lfs merge=lfs -text

1_Pooling/config.json

@@ -0,0 +1,10 @@
+{
+    "word_embedding_dimension": 1536,
+    "pooling_mode_cls_token": false,
+    "pooling_mode_mean_tokens": false,
+    "pooling_mode_max_tokens": false,
+    "pooling_mode_mean_sqrt_len_tokens": false,
+    "pooling_mode_weightedmean_tokens": false,
+    "pooling_mode_lasttoken": true,
+    "include_prompt": true
+  }

added_tokens.json

@@ -0,0 +1,16 @@
+{
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

chat_template.json

@@ -0,0 +1,3 @@
+{
+    "chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}"
+}

config.json

@@ -0,0 +1,48 @@
+{
+  "_name_or_path": "Alibaba-NLP/gme-Qwen2-VL-2B-Instruct",
+  "architectures": [
+    "Qwen2VLForConditionalGeneration",
+    "GmeQwen2VL"
+  ],
+  "auto_map": {
+    "AutoConfig": "modeling_gme_qwen2vl.GmeQwen2VLConfig",
+    "AutoModel": "modeling_gme_qwen2vl.GmeQwen2VL"
+  },
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 1536,
+  "image_token_id": 151655,
+  "initializer_range": 0.02,
+  "intermediate_size": 8960,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "model_type": "qwen2_vl",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-6,
+  "rope_scaling": {
+    "mrope_section": [16, 24, 24],
+    "type": "mrope"
+  },
+  "rope_theta": 1000000.0,
+  "sliding_window": 32768,
+  "tie_word_embeddings": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.45.0.dev0",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "video_token_id": 151656,
+  "vision_config": {
+    "hidden_size": 1536,
+    "in_chans": 3,
+    "model_type": "qwen2_vl",
+    "spatial_patch_size": 14
+  },
+  "vision_end_token_id": 151653,
+  "vision_start_token_id": 151652,
+  "vision_token_id": 151654,
+  "vocab_size": 151936
+}

config_sentence_transformers.json

@@ -0,0 +1,7 @@
+{
+  "prompts": {
+     "query": "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
+  },
+  "default_prompt_name": null,
+  "similarity_fn_name": null
+}

custom_st.py

@@ -0,0 +1,223 @@
+import math
+import logging
+from io import BytesIO
+from typing import Any, Dict, Optional, List
+import torch
+from PIL import Image
+from sentence_transformers.models import Transformer as BaseTransformer
+from transformers import AutoModelForVision2Seq, AutoProcessor
+
+
+class MultiModalTransformer(BaseTransformer):
+    def __init__(
+        self,
+        model_name_or_path: str,
+        cache_dir: Optional[str] = None,
+        tokenizer_args: Optional[Dict[str, Any]] = None,
+        min_image_tokens: int = 256,
+        max_image_tokens: int = 1280,
+        max_length: int = 1800,
+        **kwargs,
+    ):
+        super().__init__(model_name_or_path, **kwargs)
+        if tokenizer_args is None:
+            tokenizer_args = {}
+        tokenizer_args.pop("trust_remote_code", None)
+
+        # Initialize processor
+        min_pixels = min_image_tokens * 28 * 28
+        max_pixels = max_image_tokens * 28 * 28
+        self.processor = AutoProcessor.from_pretrained(
+            model_name_or_path, min_pixels=min_pixels, max_pixels=max_pixels, **kwargs
+        )
+        self.processor.tokenizer.padding_side = 'right'
+        self.sep = ' '
+        self.max_length = max_length
+        self.normalize = True
+
+    def _load_model(
+            self,
+            model_name_or_path: str,
+            config,
+            cache_dir: str,
+            backend: str,
+            is_peft_model: bool,
+            **model_args,
+    ) -> None:
+        model_args.pop("trust_remote_code", None)
+        self.auto_model = AutoModelForVision2Seq.from_pretrained(
+            model_name_or_path, torch_dtype=torch.float16, **model_args
+        )
+
+    def forward(
+        self, features: Dict[str, torch.Tensor], **kwargs
+    ) -> Dict[str, torch.Tensor]:       
+        if features.get("inputs_embeds", None) is None:
+            features["inputs_embeds"] = self.auto_model.base_model.get_input_embeddings()(features["input_ids"])
+            if features.get("pixel_values", None) is not None:
+                features["pixel_values"] = features["pixel_values"].type(self.auto_model.visual.get_dtype())
+                image_embeds = self.auto_model.visual(
+                    features["pixel_values"], grid_thw=features["image_grid_thw"]
+                )
+                image_mask = features["input_ids"] == self.auto_model.config.image_token_id
+                features["inputs_embeds"][image_mask] = image_embeds
+                # features.pop("pixel_values")
+                # features.pop("image_grid_thw")
+        # features.pop("input_ids")
+        inputs = {k: v for k, v in features.items() if k in 'position_ids,attention_mask,inputs_embeds'}
+        outputs = self.auto_model.model(
+            **inputs,
+            return_dict=True,
+            output_hidden_states=True,
+            # **kwargs
+        )
+        # pooling_mask = features["attention_mask"] if features.get("pooling_mask", None) is None else features["pooling_mask"]
+        # left_padding = (pooling_mask[:, -1].sum() == pooling_mask.shape[0])  # TODO
+        # if left_padding:
+        #     embeddings = outputs.last_hidden_state
+        # else:
+        #     sequence_lengths = pooling_mask.sum(dim=1) - 1
+        #     embeddings = outputs.last_hidden_state[torch.arange(
+        #         outputs.last_hidden_state.shape[0], device=outputs.last_hidden_state.device
+        #     ), sequence_lengths]
+        features.update({"token_embeddings": outputs.last_hidden_state})
+        return features 
+
+    def tokenize(self, texts: List[List[Dict[str, Any]]] | List[str]) -> Dict[str, torch.Tensor]:
+        default_instruction = 'You are a helpful assistant.'
+
+        all_texts, all_images = list(), list()
+        for item in texts:
+            if isinstance(item, str):
+                txt, img, inst = item, None, default_instruction
+            elif isinstance(item, dict):
+                txt = item.get('text', None)
+                img = item.get('image', None)
+                inst = item.get('prompt', default_instruction)
+            else:
+                raise RuntimeError(f'Input format not supported! {item=}')
+
+            input_str = ''
+            if img is None:
+                all_images = None  # All examples in the same batch are consistent
+                # or will have ValueError: Could not make a flat list of images from xxxx
+            else:
+                input_str += '<|vision_start|><|image_pad|><|vision_end|>'
+                img = fetch_image(img)
+                all_images.append(img)
+            if txt is not None:
+                input_str += txt
+            msg = f'<|im_start|>system\n{inst}<|im_end|>\n<|im_start|>user\n{input_str}<|im_end|>\n<|im_start|>assistant\n<|endoftext|>'
+            all_texts.append(msg)
+
+        inputs = self.processor(
+            text=all_texts,
+            images=all_images,
+            padding="longest",
+            truncation=True,
+            max_length=self.max_seq_length,
+            return_tensors='pt'
+        )
+        return inputs
+
+
+### Copied from qwen_vl_utils.vision_process.py
+import base64
+from io import BytesIO
+import requests
+
+IMAGE_FACTOR = 28
+MIN_PIXELS = 4 * 28 * 28
+MAX_PIXELS = 16384 * 28 * 28
+MAX_RATIO = 200
+
+
+def round_by_factor(number: int, factor: int) -> int:
+    """Returns the closest integer to 'number' that is divisible by 'factor'."""
+    return round(number / factor) * factor
+
+
+def ceil_by_factor(number: int, factor: int) -> int:
+    """Returns the smallest integer greater than or equal to 'number' that is divisible by 'factor'."""
+    return math.ceil(number / factor) * factor
+
+
+def floor_by_factor(number: int, factor: int) -> int:
+    """Returns the largest integer less than or equal to 'number' that is divisible by 'factor'."""
+    return math.floor(number / factor) * factor
+
+
+def smart_resize(
+    height: int, width: int, factor: int = IMAGE_FACTOR, min_pixels: int = MIN_PIXELS, max_pixels: int = MAX_PIXELS
+) -> tuple[int, int]:
+    """
+    Rescales the image so that the following conditions are met:
+
+    1. Both dimensions (height and width) are divisible by 'factor'.
+
+    2. The total number of pixels is within the range ['min_pixels', 'max_pixels'].
+
+    3. The aspect ratio of the image is maintained as closely as possible.
+    """
+    h_bar = max(factor, round_by_factor(height, factor))
+    w_bar = max(factor, round_by_factor(width, factor))
+    if h_bar * w_bar > max_pixels:
+        beta = math.sqrt((height * width) / max_pixels)
+        h_bar = floor_by_factor(height / beta, factor)
+        w_bar = floor_by_factor(width / beta, factor)
+    elif h_bar * w_bar < min_pixels:
+        beta = math.sqrt(min_pixels / (height * width))
+        h_bar = ceil_by_factor(height * beta, factor)
+        w_bar = ceil_by_factor(width * beta, factor)
+
+    if max(h_bar, w_bar) / min(h_bar, w_bar) > MAX_RATIO:
+        logging.warning(
+            f"Absolute aspect ratio must be smaller than {MAX_RATIO}, got {max(h_bar, w_bar) / min(h_bar, w_bar)}"
+        )
+        if h_bar > w_bar:
+            h_bar = w_bar * MAX_RATIO
+        else:
+            w_bar = h_bar * MAX_RATIO
+    return h_bar, w_bar
+
+
+def fetch_image(image: str | Image.Image, size_factor: int = IMAGE_FACTOR) -> Image.Image:
+    image_obj = None
+    if isinstance(image, Image.Image):
+        image_obj = image
+    elif image.startswith("http://") or image.startswith("https://"):
+        image_obj = Image.open(requests.get(image, stream=True).raw)
+    elif image.startswith("file://"):
+        image_obj = Image.open(image[7:])
+    elif image.startswith("data:image"):
+        if "base64," in image:
+            _, base64_data = image.split("base64,", 1)
+            data = base64.b64decode(base64_data)
+            image_obj = Image.open(BytesIO(data))
+    else:
+        image_obj = Image.open(image)
+    if image_obj is None:
+        raise ValueError(f"Unrecognized image input, support local path, http url, base64 and PIL.Image, got {image}")
+    image = image_obj.convert("RGB")
+    ## resize
+    # if "resized_height" in ele and "resized_width" in ele:
+    #     resized_height, resized_width = smart_resize(
+    #         ele["resized_height"],
+    #         ele["resized_width"],
+    #         factor=size_factor,
+    #     )
+    # else:
+    width, height = image.size
+    # min_pixels = ele.get("min_pixels", MIN_PIXELS)
+    # max_pixels = ele.get("max_pixels", MAX_PIXELS)
+    resized_height, resized_width = smart_resize(
+        height,
+        width,
+        factor=size_factor,
+        min_pixels=MIN_PIXELS,
+        max_pixels=MAX_PIXELS,
+    )
+    image = image.resize((resized_width, resized_height))
+
+    return image
+###

generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.05,
+  "temperature": 0.1,
+  "top_k": 1,
+  "top_p": 0.001,
+  "transformers_version": "4.45.0.dev0"
+}

gme_inference.py

@@ -0,0 +1,329 @@
+from __future__ import annotations
+
+import logging
+import math
+import os
+from typing import Dict, List, Optional
+
+import torch
+from PIL import Image
+from torch.utils.data import DataLoader
+from tqdm.autonotebook import tqdm
+from transformers import AutoModelForVision2Seq, AutoProcessor
+
+
+class GmeQwen2VL:
+    def __init__(
+        self,
+        model_name: str = "Alibaba-NLP/gme-Qwen2-VL-2B-Instruct",
+        model_path: Optional[str] = None,
+        device: str = "cuda" if torch.cuda.is_available() else "cpu",
+        min_image_tokens=256,
+        max_image_tokens=1280,
+        max_length=1800,
+        **kwargs,
+    ) -> None:
+        model_name = model_path or model_name
+        self.base = AutoModelForVision2Seq.from_pretrained(
+            model_name, torch_dtype=torch.float16, **kwargs
+        )
+        self.base.eval()
+        self.normalize = True
+        self.device = device
+        min_pixels = min_image_tokens * 28 * 28
+        max_pixels = max_image_tokens * 28 * 28
+        self.max_length = max_length
+        self.processor = AutoProcessor.from_pretrained(
+            model_name, min_pixels=min_pixels, max_pixels=max_pixels, **kwargs
+        )
+        self.processor.tokenizer.padding_side = 'right'
+        self.default_instruction = 'You are a helpful assistant.'
+        self.sep = ' '
+
+    def forward(
+        self,
+        input_ids: Optional[torch.LongTensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        pixel_values: Optional[torch.Tensor] = None,
+        # pixel_values_videos: Optional[torch.FloatTensor] = None,
+        image_grid_thw: Optional[torch.LongTensor] = None,
+        # video_grid_thw: Optional[torch.LongTensor] = None,
+        pooling_mask: Optional[torch.LongTensor] = None,
+        **kwargs
+    ) -> torch.Tensor:
+        if inputs_embeds is None:
+            inputs_embeds = self.base.model.embed_tokens(input_ids)
+            if pixel_values is not None:
+                pixel_values = pixel_values.type(self.base.visual.get_dtype())
+                image_embeds = self.base.visual(pixel_values, grid_thw=image_grid_thw).to(inputs_embeds.device)
+                image_mask = input_ids == self.base.config.image_token_id
+                inputs_embeds[image_mask] = image_embeds
+            # if pixel_values_videos is not None:
+            #     pixel_values_videos = pixel_values_videos.type(self.base.visual.get_dtype())
+            #     video_embeds = self.base.visual(pixel_values_videos, grid_thw=video_grid_thw).to(inputs_embeds.device)
+            #     video_mask = input_ids == self.base.config.video_token_id
+            #     inputs_embeds[video_mask] = video_embeds
+            if attention_mask is not None:
+                attention_mask = attention_mask.to(inputs_embeds.device)
+
+        outputs = self.base.model(
+            input_ids=None,
+            position_ids=position_ids,
+            attention_mask=attention_mask,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+        )
+
+        pooling_mask = attention_mask if pooling_mask is None else pooling_mask
+        left_padding = (pooling_mask[:, -1].sum() == pooling_mask.shape[0])  # TODO
+        if left_padding:
+            embeddings = outputs.last_hidden_state[:, -1]
+        else:
+            sequence_lengths = pooling_mask.sum(dim=1) - 1
+            batch_size = outputs.last_hidden_state.shape[0]
+            embeddings = outputs.last_hidden_state[torch.arange(
+                batch_size, device=outputs.last_hidden_state.device
+            ), sequence_lengths]
+        if self.normalize:
+            embeddings = torch.nn.functional.normalize(embeddings, p=2, dim=1)
+        return embeddings.contiguous()
+
+    def embed(self, texts: list[str], images: list[Image.Image], is_query=True, instruction=None, **kwargs):
+        self.base.to(self.device)
+        # Inputs must be batched
+        input_texts, input_images = list(), list()
+        for t, i in zip(texts, images):
+            if not is_query or instruction is None:
+                instruction = self.default_instruction
+            input_str = ''
+            if i is None:
+                input_images = None  # All examples in the same batch are consistent
+            else:
+                input_str += '<|vision_start|><|image_pad|><|vision_end|>'
+                i = fetch_image(i)
+                input_images.append(i)
+            if t is not None:
+                input_str += t
+            msg = f'<|im_start|>system\n{instruction}<|im_end|>\n<|im_start|>user\n{input_str}<|im_end|>\n<|im_start|>assistant\n<|endoftext|>'
+            input_texts.append(msg)
+
+        inputs = self.processor(
+            text=input_texts,
+            images=input_images,
+            padding=True,
+            truncation=True,
+            max_length=self.max_length,
+            return_tensors='pt'
+        )
+        inputs = {k: v.to(self.device) for k, v in inputs.items()}  # TODO
+        with torch.no_grad():
+            embeddings = self.forward(**inputs)
+        return embeddings
+
+    def encode(self, sentences: list[str], *, prompt_name=None, **kwargs):
+        return self.get_fused_embeddings(texts=sentences, prompt_name=prompt_name, **kwargs)
+
+    def encode_queries(self, queries: List[str], **kwargs):
+        embeddings = self.encode(queries, **kwargs)
+        return embeddings
+
+    def encode_corpus(self, corpus: List[Dict[str, str]], **kwargs):
+        if type(corpus) is dict:
+            sentences = [
+                (corpus["title"][i] + self.sep + corpus["text"][i]).strip()
+                if "title" in corpus
+                else corpus["text"][i].strip()
+                for i in range(len(corpus["text"]))
+            ]
+        else:
+            sentences = [
+                (doc["title"] + self.sep + doc["text"]).strip() if "title" in doc else doc["text"].strip()
+                for doc in corpus
+            ]
+        embeddings = self.encode(sentences, is_query=False, **kwargs)
+        return embeddings
+
+    def get_image_embeddings(self, images: list[Image.Image] | DataLoader, **kwargs):
+        return self.get_fused_embeddings(images=images, **kwargs)
+
+    def get_text_embeddings(self, texts: list[str], **kwargs):
+        return self.get_fused_embeddings(texts=texts, **kwargs)
+
+    def get_fused_embeddings(self, texts: list[str] = None, images: list[Image.Image] | DataLoader = None, **kwargs):
+        if isinstance(images, DataLoader):
+            image_loader = images
+            batch_size = image_loader.batch_size
+            image_loader.dataset.transform = None
+        else:
+            batch_size = kwargs.pop('batch_size', 32)
+            if images is None:
+                image_loader = None
+            else:
+                image_loader = DataLoader(
+                    images,
+                    batch_size=batch_size,
+                    shuffle=False,
+                    collate_fn=custom_collate_fn,
+                    num_workers=min(math.floor(os.cpu_count() / 2), 8),
+                )
+
+        if texts is None:
+            assert image_loader is not None
+            n_batch = len(image_loader)
+        else:
+            n_batch = len(texts) // batch_size + int(len(texts) % batch_size > 0)
+            image_loader = image_loader or [None] * n_batch
+
+        all_embeddings = list()
+        none_batch = [None] * batch_size
+        show_progress_bar = kwargs.pop('show_progress_bar', True)
+        pbar = tqdm(total=n_batch, disable=not show_progress_bar, mininterval=1, miniters=10, desc='encode')
+        for n, img_batch in zip(range(0, n_batch * batch_size, batch_size), image_loader):
+            text_batch = none_batch if texts is None else texts[n: n+batch_size]
+            img_batch = none_batch if img_batch is None else img_batch
+            embeddings = self.embed(texts=text_batch, images=img_batch, **kwargs)
+            pbar.update(1)
+            all_embeddings.append(embeddings.cpu())
+        pbar.close()
+        all_embeddings = torch.cat(all_embeddings, dim=0)
+        return all_embeddings
+
+
+def custom_collate_fn(batch):
+    return batch
+
+
+### Copied from qwen_vl_utils.vision_process.py
+import base64
+from io import BytesIO
+import requests
+
+IMAGE_FACTOR = 28
+MIN_PIXELS = 4 * 28 * 28
+MAX_PIXELS = 16384 * 28 * 28
+MAX_RATIO = 200
+
+
+def round_by_factor(number: int, factor: int) -> int:
+    """Returns the closest integer to 'number' that is divisible by 'factor'."""
+    return round(number / factor) * factor
+
+
+def ceil_by_factor(number: int, factor: int) -> int:
+    """Returns the smallest integer greater than or equal to 'number' that is divisible by 'factor'."""
+    return math.ceil(number / factor) * factor
+
+
+def floor_by_factor(number: int, factor: int) -> int:
+    """Returns the largest integer less than or equal to 'number' that is divisible by 'factor'."""
+    return math.floor(number / factor) * factor
+
+
+def smart_resize(
+    height: int, width: int, factor: int = IMAGE_FACTOR, min_pixels: int = MIN_PIXELS, max_pixels: int = MAX_PIXELS
+) -> tuple[int, int]:
+    """
+    Rescales the image so that the following conditions are met:
+
+    1. Both dimensions (height and width) are divisible by 'factor'.
+
+    2. The total number of pixels is within the range ['min_pixels', 'max_pixels'].
+
+    3. The aspect ratio of the image is maintained as closely as possible.
+    """
+    h_bar = max(factor, round_by_factor(height, factor))
+    w_bar = max(factor, round_by_factor(width, factor))
+    if h_bar * w_bar > max_pixels:
+        beta = math.sqrt((height * width) / max_pixels)
+        h_bar = floor_by_factor(height / beta, factor)
+        w_bar = floor_by_factor(width / beta, factor)
+    elif h_bar * w_bar < min_pixels:
+        beta = math.sqrt(min_pixels / (height * width))
+        h_bar = ceil_by_factor(height * beta, factor)
+        w_bar = ceil_by_factor(width * beta, factor)
+
+    if max(h_bar, w_bar) / min(h_bar, w_bar) > MAX_RATIO:
+        logging.warning(
+            f"Absolute aspect ratio must be smaller than {MAX_RATIO}, got {max(h_bar, w_bar) / min(h_bar, w_bar)}"
+        )
+        if h_bar > w_bar:
+            h_bar = w_bar * MAX_RATIO
+        else:
+            w_bar = h_bar * MAX_RATIO
+    return h_bar, w_bar
+
+
+def fetch_image(image: str | Image.Image, size_factor: int = IMAGE_FACTOR) -> Image.Image:
+    image_obj = None
+    if isinstance(image, Image.Image):
+        image_obj = image
+    elif image.startswith("http://") or image.startswith("https://"):
+        image_obj = Image.open(requests.get(image, stream=True).raw)
+    elif image.startswith("file://"):
+        image_obj = Image.open(image[7:])
+    elif image.startswith("data:image"):
+        if "base64," in image:
+            _, base64_data = image.split("base64,", 1)
+            data = base64.b64decode(base64_data)
+            image_obj = Image.open(BytesIO(data))
+    else:
+        image_obj = Image.open(image)
+    if image_obj is None:
+        raise ValueError(f"Unrecognized image input, support local path, http url, base64 and PIL.Image, got {image}")
+    image = image_obj.convert("RGB")
+    ## resize
+    # if "resized_height" in ele and "resized_width" in ele:
+    #     resized_height, resized_width = smart_resize(
+    #         ele["resized_height"],
+    #         ele["resized_width"],
+    #         factor=size_factor,
+    #     )
+    # else:
+    width, height = image.size
+    # min_pixels = ele.get("min_pixels", MIN_PIXELS)
+    # max_pixels = ele.get("max_pixels", MAX_PIXELS)
+    resized_height, resized_width = smart_resize(
+        height,
+        width,
+        factor=size_factor,
+        min_pixels=MIN_PIXELS,
+        max_pixels=MAX_PIXELS,
+    )
+    image = image.resize((resized_width, resized_height))
+
+    return image
+###
+
+
+if __name__ == '__main__':
+    texts = [
+        "What kind of car is this?",
+        "The Tesla Cybertruck is a battery electric pickup truck built by Tesla, Inc. since 2023."
+    ]
+    images = [
+        'https://upload.wikimedia.org/wikipedia/commons/e/e9/Tesla_Cybertruck_damaged_window.jpg',
+        'https://upload.wikimedia.org/wikipedia/commons/9/95/2024_Tesla_Cybertruck_Foundation_Series%2C_front_left_%28Greenwich%29.jpg',
+    ]
+
+    gme = GmeQwen2VL("Alibaba-NLP/gme-Qwen2-VL-2B-Instruct")
+
+    # Single-modal embedding
+    e_text = gme.get_text_embeddings(texts=texts)
+    e_image = gme.get_image_embeddings(images=images)
+    print((e_text * e_image).sum(-1))
+    ## tensor([0.2281, 0.6001], dtype=torch.float16)
+
+    # How to set embedding instruction
+    e_query = gme.get_text_embeddings(texts=texts, instruction='Find an image that matches the given text.')
+    # If is_query=False, we always use the default instruction.
+    e_corpus = gme.get_image_embeddings(images=images, is_query=False)
+    print((e_query * e_corpus).sum(-1))
+    ## tensor([0.2433, 0.7051], dtype=torch.float16)
+
+    # Fused-modal embedding
+    e_fused = gme.get_fused_embeddings(texts=texts, images=images)
+    print((e_fused[0] * e_fused[1]).sum())
+    ## tensor(0.6108, dtype=torch.float16)

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce525de1d47307ebf63285ce4f2f80ca8ffffd94235b2151585e5714307e4e99
+size 4046085544

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b42b76064f5d14ee9d0390f9a1f3571fc66eae4f40764f818cb5e9aafc5d6f5
+size 4063188000

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3f1342fe1dd7df1582be00ebd45557316015050ae2edb841e61150825f16adc
+size 726747736

model.safetensors.index.json

@@ -0,0 +1,736 @@
+{
+  "metadata": {
+    "total_size": 8835942400
+  },
+  "weight_map": {
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors",
+    "visual.blocks.0.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.0.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.0.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.0.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.0.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.0.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.0.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.0.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.0.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.0.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.0.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.0.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.1.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.1.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.1.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.1.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.1.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.1.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.1.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.1.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.1.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.1.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.1.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.1.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.10.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.10.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.10.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.10.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.10.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.10.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.10.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.10.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.10.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.10.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.10.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.10.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.11.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.11.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.11.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.11.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.11.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.11.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.11.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.11.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.11.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.11.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.11.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.11.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.12.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.12.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.12.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.12.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.12.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.12.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.12.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.12.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.12.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.12.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.12.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.12.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.13.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.13.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.13.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.13.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.13.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.13.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.13.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.13.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.13.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.13.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.13.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.13.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.14.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.14.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.14.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.14.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.14.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.14.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.14.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.14.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.14.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.14.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.14.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.14.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.15.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.15.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.15.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.15.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.15.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.15.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.15.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.15.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.15.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.15.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.15.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.15.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.16.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.16.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.16.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.16.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.16.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.16.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.16.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.16.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.16.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.16.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.16.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.16.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.17.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.17.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.17.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.17.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.17.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.17.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.17.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.17.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.17.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.17.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.17.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.17.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.18.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.18.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.18.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.18.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.18.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.18.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.18.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.18.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.18.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.18.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.18.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.18.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.19.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.19.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.19.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.19.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.19.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.19.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.19.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.19.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.19.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.19.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.19.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.19.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.2.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.2.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.2.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.2.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.2.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.2.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.2.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.2.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.2.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.2.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.2.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.2.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.20.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.20.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.20.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.20.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.20.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.20.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.20.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.20.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.20.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.20.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.20.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.20.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.21.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.21.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.21.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.21.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.21.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.21.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.21.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.21.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.21.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.21.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.21.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.21.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.22.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.22.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.22.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.22.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.22.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.22.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.22.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.22.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.22.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.22.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.22.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.22.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.23.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.23.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.23.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.23.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.23.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.23.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.23.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.23.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.23.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.23.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.23.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.23.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.24.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.24.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.24.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.24.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.24.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.24.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.24.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.24.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.24.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.24.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.24.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.24.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.25.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.25.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.25.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.25.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.25.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.25.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.25.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.25.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.25.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.25.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.25.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.25.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.26.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.26.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.26.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.26.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.26.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.26.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.26.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.26.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.26.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.26.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.26.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.26.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.27.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.27.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.27.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.27.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.27.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.27.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.27.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.27.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.27.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.27.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.27.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.27.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.28.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.28.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.28.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.28.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.28.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.28.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.28.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.28.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.28.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.28.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.28.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.28.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.29.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.29.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.29.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.29.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.29.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.29.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.29.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.29.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.29.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.29.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.29.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.29.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.3.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.3.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.3.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.3.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.3.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.3.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.3.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.3.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.3.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.3.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.3.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.3.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.30.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.30.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.30.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.30.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.30.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.30.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.30.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.30.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.30.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.30.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.30.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.30.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.31.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.31.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.31.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.31.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.31.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.31.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.31.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.31.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.31.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.31.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.31.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.31.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.4.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.4.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.4.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.4.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.4.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.4.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.4.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.4.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.4.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.4.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.4.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.4.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.5.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.5.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.5.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.5.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.5.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.5.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.5.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.5.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.5.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.5.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.5.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.5.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.6.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.6.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.6.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.6.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.6.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.6.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.6.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.6.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.6.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.6.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.6.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.6.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.7.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.7.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.7.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.7.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.7.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.7.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.7.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.7.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.7.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.7.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.7.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.7.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.8.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.8.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.8.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.8.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.8.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.8.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.8.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.8.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.8.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.8.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.8.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.8.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.9.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.9.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.9.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.9.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.9.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.9.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.9.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.9.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.9.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.9.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.9.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.9.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.merger.ln_q.bias": "model-00001-of-00003.safetensors",
+    "visual.merger.ln_q.weight": "model-00001-of-00003.safetensors",
+    "visual.merger.mlp.0.bias": "model-00001-of-00003.safetensors",
+    "visual.merger.mlp.0.weight": "model-00001-of-00003.safetensors",
+    "visual.merger.mlp.2.bias": "model-00001-of-00003.safetensors",
+    "visual.merger.mlp.2.weight": "model-00001-of-00003.safetensors",
+    "visual.patch_embed.proj.weight": "model-00001-of-00003.safetensors"
+  }
+}

modeling_gme_qwen2vl.py

@@ -0,0 +1,337 @@
+from __future__ import annotations
+
+import base64
+import logging
+import math
+import os
+from io import BytesIO
+from typing import Any, Dict, List, Optional, Union
+
+import requests
+import torch
+from PIL import Image
+from torch.utils.data import DataLoader
+from tqdm.autonotebook import tqdm
+from transformers import AutoProcessor, PreTrainedModel
+from transformers.models.qwen2_vl.modeling_qwen2_vl import (
+    Qwen2VisionTransformerPretrainedModel,
+    Qwen2VLConfig,
+    Qwen2VLForConditionalGeneration,
+    Qwen2VLModel,
+)
+from transformers.utils.versions import require_version
+
+
+# require_version(
+#     "transformers<4.52.0",
+#     "This code has some issues with transformers>=4.52.0, please downgrade: pip install transformers==4.51.3"
+# )
+
+
+class GmeQwen2VLConfig(Qwen2VLConfig):
+    # model_type = ''
+
+    def __init__(
+        self,
+        min_image_tokens: int = 256,
+        max_image_tokens: int = 1280,
+        max_length: int = 1800,
+        **kwargs: Any,
+    ) -> None:
+        super().__init__(**kwargs)
+        self.min_image_tokens = min_image_tokens
+        self.max_image_tokens = max_image_tokens
+        self.max_length = max_length
+
+
+class GmeQwen2VL(PreTrainedModel):
+    config_class = GmeQwen2VLConfig
+    base_model_prefix = "model"
+    supports_gradient_checkpointing = True
+    _no_split_modules = ["Qwen2VLDecoderLayer", "Qwen2VLVisionBlock"]
+    # _skip_keys_device_placement = "past_key_values"
+    _supports_flash_attn_2 = True
+    _supports_sdpa = True
+    # _supports_cache_class = True
+    _supports_static_cache = False  # TODO (joao): fix. torch.compile failing probably due to `cache_positions`
+    # _tied_weights_keys = ["lm_head.weight"]
+
+    def __init__(self, config: GmeQwen2VLConfig, **kwargs: Any) -> None:
+        super().__init__(config)
+        self.visual = Qwen2VisionTransformerPretrainedModel._from_config(config.vision_config)
+        self.model = Qwen2VLModel(config)
+        self.vocab_size = config.vocab_size
+        # self.lm_head = torch.nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+        self.rope_deltas = None  # cache rope_deltas here
+
+        min_pixels: int = config.min_image_tokens * 28 * 28
+        max_pixels: int = config.max_image_tokens * 28 * 28
+        self.processor = AutoProcessor.from_pretrained(
+            config._name_or_path, min_pixels=min_pixels, max_pixels=max_pixels, **kwargs
+        )
+        self.max_length: int = config.max_length
+        self.normalize: bool = True
+        self.processor.tokenizer.padding_side = "right"
+        self.default_instruction: str = "You are a helpful assistant."
+        self.sep: str = " "
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def forward(
+        self,
+        input_ids: Optional[torch.LongTensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        pixel_values: Optional[torch.Tensor] = None,
+        # pixel_values_videos: Optional[torch.FloatTensor] = None,
+        image_grid_thw: Optional[torch.LongTensor] = None,
+        # video_grid_thw: Optional[torch.LongTensor] = None,
+        pooling_mask: Optional[torch.LongTensor] = None,
+        **kwargs
+    ) -> torch.Tensor:
+        if inputs_embeds is None:
+            inputs_embeds = self.model.get_input_embeddings()(input_ids)
+            if pixel_values is not None:
+                pixel_values = pixel_values.type(self.visual.get_dtype())
+                image_embeds = self.visual(pixel_values, grid_thw=image_grid_thw).to(inputs_embeds.device)
+                image_mask = input_ids == self.config.image_token_id
+                inputs_embeds[image_mask] = image_embeds
+            # if pixel_values_videos is not None:
+            #     pixel_values_videos = pixel_values_videos.type(self.visual.get_dtype())
+            #     video_embeds = self.visual(pixel_values_videos, grid_thw=video_grid_thw).to(inputs_embeds.device)
+            #     video_mask = input_ids == self.config.video_token_id
+            #     inputs_embeds[video_mask] = video_embeds
+            if attention_mask is not None:
+                attention_mask = attention_mask.to(inputs_embeds.device)
+
+        outputs = self.model(
+            input_ids=None,
+            position_ids=position_ids,
+            attention_mask=attention_mask,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+        )
+
+        pooling_mask = attention_mask if pooling_mask is None else pooling_mask
+        left_padding = (pooling_mask[:, -1].sum() == pooling_mask.shape[0])  # TODO
+        if left_padding:
+            embeddings = outputs.last_hidden_state[:, -1]
+        else:
+            sequence_lengths = pooling_mask.sum(dim=1) - 1
+            batch_size = outputs.last_hidden_state.shape[0]
+            embeddings = outputs.last_hidden_state[torch.arange(
+                batch_size, device=outputs.last_hidden_state.device
+            ), sequence_lengths]
+        if self.normalize:
+            embeddings = torch.nn.functional.normalize(embeddings, p=2, dim=1)
+        return embeddings.contiguous()
+
+    def embed(self, texts: list[str], images: list[Image.Image], is_query=True, instruction=None, **kwargs):
+        self.eval()
+        # Inputs must be batched
+        input_texts, input_images = list(), list()
+        for t, i in zip(texts, images):
+            if not is_query or instruction is None:
+                instruction = self.default_instruction
+            input_str = ''
+            if i is None:
+                input_images = None  # All examples in the same batch are consistent
+            else:
+                input_str += '<|vision_start|><|image_pad|><|vision_end|>'
+                i = fetch_image(i)
+                input_images.append(i)
+            if t is not None:
+                input_str += t
+            msg = f'<|im_start|>system\n{instruction}<|im_end|>\n<|im_start|>user\n{input_str}<|im_end|>\n<|im_start|>assistant\n<|endoftext|>'
+            input_texts.append(msg)
+
+        inputs = self.processor(
+            text=input_texts,
+            images=input_images,
+            padding=True,
+            truncation=True,
+            max_length=self.max_length,
+            return_tensors='pt'
+        )
+        inputs = {k: v.to(self.device) for k, v in inputs.items()}  # TODO
+        with torch.inference_mode():
+            embeddings = self.forward(**inputs)
+        return embeddings
+
+    def encode(self, sentences: list[str], *, prompt_name=None, **kwargs):
+        return self.get_fused_embeddings(texts=sentences, prompt_name=prompt_name, **kwargs)
+
+    def encode_queries(self, queries: List[str], **kwargs):
+        embeddings = self.encode(queries, **kwargs)
+        return embeddings
+
+    def encode_corpus(self, corpus: List[Dict[str, str]], **kwargs):
+        if type(corpus) is dict:
+            sentences = [
+                (corpus["title"][i] + self.sep + corpus["text"][i]).strip()
+                if "title" in corpus
+                else corpus["text"][i].strip()
+                for i in range(len(corpus["text"]))
+            ]
+        else:
+            sentences = [
+                (doc["title"] + self.sep + doc["text"]).strip() if "title" in doc else doc["text"].strip()
+                for doc in corpus
+            ]
+        embeddings = self.encode(sentences, is_query=False, **kwargs)
+        return embeddings
+
+    def get_image_embeddings(self, images: list[Image.Image] | DataLoader, **kwargs):
+        return self.get_fused_embeddings(images=images, **kwargs)
+
+    def get_text_embeddings(self, texts: list[str], **kwargs):
+        return self.get_fused_embeddings(texts=texts, **kwargs)
+
+    def get_fused_embeddings(self, texts: list[str] = None, images: list[Image.Image] | DataLoader = None, **kwargs):
+        if isinstance(images, DataLoader):
+            image_loader = images
+            batch_size = image_loader.batch_size
+            image_loader.dataset.transform = None
+        else:
+            batch_size = kwargs.pop('batch_size', 32)
+            if images is None:
+                image_loader = None
+            else:
+                image_loader = DataLoader(
+                    images,
+                    batch_size=batch_size,
+                    shuffle=False,
+                    collate_fn=custom_collate_fn,
+                    num_workers=min(math.floor(os.cpu_count() / 2), 8),
+                )
+
+        if texts is None:
+            assert image_loader is not None
+            n_batch = len(image_loader)
+        else:
+            n_batch = len(texts) // batch_size + int(len(texts) % batch_size > 0)
+            image_loader = image_loader or [None] * n_batch
+
+        all_embeddings = list()
+        none_batch = [None] * batch_size
+        show_progress_bar = kwargs.pop('show_progress_bar', False)
+        pbar = tqdm(total=n_batch, disable=not show_progress_bar, mininterval=1, miniters=10, desc='encode')
+        for n, img_batch in zip(range(0, n_batch * batch_size, batch_size), image_loader):
+            text_batch = none_batch if texts is None else texts[n: n+batch_size]
+            img_batch = none_batch if img_batch is None else img_batch
+            embeddings = self.embed(texts=text_batch, images=img_batch, **kwargs)
+            pbar.update(1)
+            all_embeddings.append(embeddings.cpu())
+        pbar.close()
+        all_embeddings = torch.cat(all_embeddings, dim=0)
+        return all_embeddings
+
+
+def custom_collate_fn(batch):
+    return batch
+
+
+### Copied from qwen_vl_utils.vision_process.py
+import base64
+from io import BytesIO
+import requests
+
+IMAGE_FACTOR = 28
+MIN_PIXELS = 4 * 28 * 28
+MAX_PIXELS = 16384 * 28 * 28
+MAX_RATIO = 200
+
+
+def round_by_factor(number: int, factor: int) -> int:
+    """Returns the closest integer to 'number' that is divisible by 'factor'."""
+    return round(number / factor) * factor
+
+
+def ceil_by_factor(number: int, factor: int) -> int:
+    """Returns the smallest integer greater than or equal to 'number' that is divisible by 'factor'."""
+    return math.ceil(number / factor) * factor
+
+
+def floor_by_factor(number: int, factor: int) -> int:
+    """Returns the largest integer less than or equal to 'number' that is divisible by 'factor'."""
+    return math.floor(number / factor) * factor
+
+
+def smart_resize(
+    height: int, width: int, factor: int = IMAGE_FACTOR, min_pixels: int = MIN_PIXELS, max_pixels: int = MAX_PIXELS
+) -> tuple[int, int]:
+    """
+    Rescales the image so that the following conditions are met:
+
+    1. Both dimensions (height and width) are divisible by 'factor'.
+
+    2. The total number of pixels is within the range ['min_pixels', 'max_pixels'].
+
+    3. The aspect ratio of the image is maintained as closely as possible.
+    """
+    h_bar = max(factor, round_by_factor(height, factor))
+    w_bar = max(factor, round_by_factor(width, factor))
+    if h_bar * w_bar > max_pixels:
+        beta = math.sqrt((height * width) / max_pixels)
+        h_bar = floor_by_factor(height / beta, factor)
+        w_bar = floor_by_factor(width / beta, factor)
+    elif h_bar * w_bar < min_pixels:
+        beta = math.sqrt(min_pixels / (height * width))
+        h_bar = ceil_by_factor(height * beta, factor)
+        w_bar = ceil_by_factor(width * beta, factor)
+
+    if max(h_bar, w_bar) / min(h_bar, w_bar) > MAX_RATIO:
+        logging.warning(
+            f"Absolute aspect ratio must be smaller than {MAX_RATIO}, got {max(h_bar, w_bar) / min(h_bar, w_bar)}"
+        )
+        if h_bar > w_bar:
+            h_bar = w_bar * MAX_RATIO
+        else:
+            w_bar = h_bar * MAX_RATIO
+    return h_bar, w_bar
+
+
+def fetch_image(image: str | Image.Image, size_factor: int = IMAGE_FACTOR) -> Image.Image:
+    image_obj = None
+    if isinstance(image, Image.Image):
+        image_obj = image
+    elif image.startswith("http://") or image.startswith("https://"):
+        image_obj = Image.open(requests.get(image, stream=True).raw)
+    elif image.startswith("file://"):
+        image_obj = Image.open(image[7:])
+    elif image.startswith("data:image"):
+        if "base64," in image:
+            _, base64_data = image.split("base64,", 1)
+            data = base64.b64decode(base64_data)
+            image_obj = Image.open(BytesIO(data))
+    else:
+        image_obj = Image.open(image)
+    if image_obj is None:
+        raise ValueError(f"Unrecognized image input, support local path, http url, base64 and PIL.Image, got {image}")
+    image = image_obj.convert("RGB")
+    ## resize
+    # if "resized_height" in ele and "resized_width" in ele:
+    #     resized_height, resized_width = smart_resize(
+    #         ele["resized_height"],
+    #         ele["resized_width"],
+    #         factor=size_factor,
+    #     )
+    # else:
+    width, height = image.size
+    # min_pixels = ele.get("min_pixels", MIN_PIXELS)
+    # max_pixels = ele.get("max_pixels", MAX_PIXELS)
+    resized_height, resized_width = smart_resize(
+        height,
+        width,
+        factor=size_factor,
+        min_pixels=MIN_PIXELS,
+        max_pixels=MAX_PIXELS,
+    )
+    image = image.resize((resized_width, resized_height))
+
+    return image
+###
+

modules.json

@@ -0,0 +1,20 @@
+[
+  {
+    "idx": 0,
+    "name": "0",
+    "path": "",
+    "type": "custom_st.MultiModalTransformer"
+  },
+  {
+    "idx": 1,
+    "name": "1",
+    "path": "1_Pooling",
+    "type": "sentence_transformers.models.Pooling"
+  },
+  {
+    "idx": 2,
+    "name": "2",
+    "path": "2_Normalize",
+    "type": "sentence_transformers.models.Normalize"
+  }
+]

preprocessor_config.json

@@ -0,0 +1,19 @@
+{
+  "min_pixels": 3136,
+  "max_pixels": 12845056,
+  "patch_size": 14,
+  "temporal_patch_size": 2,
+  "merge_size": 2,
+  "image_mean": [
+    0.48145466,
+    0.4578275,
+    0.40821073
+  ],
+  "image_std": [
+    0.26862954,
+    0.26130258,
+    0.27577711
+  ],
+  "image_processor_type": "Qwen2VLImageProcessor",
+  "processor_class": "Qwen2VLProcessor"
+}

results.json

@@ -0,0 +1 @@
+{"arxivqa_test_subsampled": {"ndcg_at_1": 0.786, "ndcg_at_3": 0.82859, "ndcg_at_5": 0.83909, "ndcg_at_10": 0.85118, "ndcg_at_20": 0.85912, "ndcg_at_50": 0.86489, "ndcg_at_100": 0.8669, "map_at_1": 0.786, "map_at_3": 0.819, "map_at_5": 0.8247, "map_at_10": 0.82957, "map_at_20": 0.83167, "map_at_50": 0.83269, "map_at_100": 0.83289, "recall_at_1": 0.786, "recall_at_3": 0.856, "recall_at_5": 0.882, "recall_at_10": 0.92, "recall_at_20": 0.952, "recall_at_50": 0.98, "recall_at_100": 0.992, "precision_at_1": 0.786, "precision_at_3": 0.28533, "precision_at_5": 0.1764, "precision_at_10": 0.092, "precision_at_20": 0.0476, "precision_at_50": 0.0196, "precision_at_100": 0.00992, "mrr_at_1": 0.786, "mrr_at_3": 0.8189999999999998, "mrr_at_5": 0.8246999999999999, "mrr_at_10": 0.8295706349206347, "mrr_at_20": 0.8316712173447467, "mrr_at_50": 0.8326901700317237, "mrr_at_100": 0.832885280842064, "naucs_at_1_max": 0.8892284946618716, "naucs_at_1_std": -0.3267276061583954, "naucs_at_1_diff1": 0.9336110202863116, "naucs_at_3_max": 0.8940728333693538, "naucs_at_3_std": -0.2892397881996969, "naucs_at_3_diff1": 0.9035214501837059, "naucs_at_5_max": 0.8728866928256561, "naucs_at_5_std": -0.2504965363561503, "naucs_at_5_diff1": 0.8887031923654505, "naucs_at_10_max": 0.8634570494864612, "naucs_at_10_std": -0.32796451914099106, "naucs_at_10_diff1": 0.8821195144724552, "naucs_at_20_max": 0.8949579831932764, "naucs_at_20_std": -0.29034391534391896, "naucs_at_20_diff1": 0.878618113912229, "naucs_at_50_max": 0.9096638655462147, "naucs_at_50_std": 0.12301587301586674, "naucs_at_50_diff1": 0.9014939309056892, "naucs_at_100_max": 1.0, "naucs_at_100_std": 0.8558590102707644, "naucs_at_100_diff1": 0.9673202614378978}, "docvqa_test_subsampled": {"ndcg_at_1": 0.4612, "ndcg_at_3": 0.52679, "ndcg_at_5": 0.54568, "ndcg_at_10": 0.56932, "ndcg_at_20": 0.58751, "ndcg_at_50": 0.59746, "ndcg_at_100": 0.60597, "map_at_1": 0.4612, "map_at_3": 0.51109, "map_at_5": 0.5214, "map_at_10": 0.53156, "map_at_20": 0.53639, "map_at_50": 0.5381, "map_at_100": 0.53881, "recall_at_1": 0.4612, "recall_at_3": 0.57206, "recall_at_5": 0.61863, "recall_at_10": 0.68958, "recall_at_20": 0.76275, "recall_at_50": 0.81153, "recall_at_100": 0.86475, "precision_at_1": 0.4612, "precision_at_3": 0.19069, "precision_at_5": 0.12373, "precision_at_10": 0.06896, "precision_at_20": 0.03814, "precision_at_50": 0.01623, "precision_at_100": 0.00865, "mrr_at_1": 0.4611973392461197, "mrr_at_3": 0.5110864745011088, "mrr_at_5": 0.5213968957871397, "mrr_at_10": 0.531558617534227, "mrr_at_20": 0.5363705459439568, "mrr_at_50": 0.5380818653716594, "mrr_at_100": 0.5387956639409233, "naucs_at_1_max": 0.6018461520563698, "naucs_at_1_std": -0.29453860375419927, "naucs_at_1_diff1": 0.7645813967276972, "naucs_at_3_max": 0.58164206836642, "naucs_at_3_std": -0.25403366910903274, "naucs_at_3_diff1": 0.6515883726118482, "naucs_at_5_max": 0.5721975818175079, "naucs_at_5_std": -0.2744904256592273, "naucs_at_5_diff1": 0.61928231963369, "naucs_at_10_max": 0.5671552158378993, "naucs_at_10_std": -0.28750264247429225, "naucs_at_10_diff1": 0.5626707252181952, "naucs_at_20_max": 0.6156578041155646, "naucs_at_20_std": -0.2749165128258015, "naucs_at_20_diff1": 0.6002227825216901, "naucs_at_50_max": 0.6012704408386546, "naucs_at_50_std": -0.33739300418375234, "naucs_at_50_diff1": 0.6419776051914634, "naucs_at_100_max": 0.6412951544246476, "naucs_at_100_std": -0.2690460787063671, "naucs_at_100_diff1": 0.6375549090035819}, "infovqa_test_subsampled": {"ndcg_at_1": 0.8664, "ndcg_at_3": 0.89876, "ndcg_at_5": 0.91113, "ndcg_at_10": 0.91568, "ndcg_at_20": 0.91815, "ndcg_at_50": 0.92068, "ndcg_at_100": 0.92099, "map_at_1": 0.8664, "map_at_3": 0.89103, "map_at_5": 0.89781, "map_at_10": 0.89966, "map_at_20": 0.9003, "map_at_50": 0.90076, "map_at_100": 0.90078, "recall_at_1": 0.8664, "recall_at_3": 0.92105, "recall_at_5": 0.95142, "recall_at_10": 0.96559, "recall_at_20": 0.97571, "recall_at_50": 0.98785, "recall_at_100": 0.98988, "precision_at_1": 0.8664, "precision_at_3": 0.30702, "precision_at_5": 0.19028, "precision_at_10": 0.09656, "precision_at_20": 0.04879, "precision_at_50": 0.01976, "precision_at_100": 0.0099, "mrr_at_1": 0.8663967611336032, "mrr_at_3": 0.891025641025641, "mrr_at_5": 0.8978070175438595, "mrr_at_10": 0.8996626180836705, "mrr_at_20": 0.9003085288927154, "mrr_at_50": 0.9007665297125582, "mrr_at_100": 0.9007882962878053, "naucs_at_1_max": 0.6852647369506982, "naucs_at_1_std": -0.11019839462727245, "naucs_at_1_diff1": 0.9097315890589761, "naucs_at_3_max": 0.6980505621537562, "naucs_at_3_std": -0.1587891880170896, "naucs_at_3_diff1": 0.8829217951800641, "naucs_at_5_max": 0.857855262670585, "naucs_at_5_std": 0.07362769173568413, "naucs_at_5_diff1": 0.9054241580005612, "naucs_at_10_max": 0.8918421488449596, "naucs_at_10_std": 0.19317064064529682, "naucs_at_10_diff1": 0.9222031546896321, "naucs_at_20_max": 0.8948019000326467, "naucs_at_20_std": 0.19941878566952734, "naucs_at_20_diff1": 0.9238158184622186, "naucs_at_50_max": 0.9564661819783937, "naucs_at_50_std": 0.6394965227510232, "naucs_at_50_diff1": 0.9129323639568211, "naucs_at_100_max": 0.9477594183740937, "naucs_at_100_std": 0.5935161181141998, "naucs_at_100_diff1": 0.9216391275611305}, "shiftproject_test": {"ndcg_at_1": 0.88, "ndcg_at_3": 0.94286, "ndcg_at_5": 0.94286, "ndcg_at_10": 0.94601, "ndcg_at_20": 0.94601, "ndcg_at_50": 0.94601, "ndcg_at_100": 0.94601, "map_at_1": 0.88, "map_at_3": 0.92667, "map_at_5": 0.92667, "map_at_10": 0.92792, "map_at_20": 0.92792, "map_at_50": 0.92792, "map_at_100": 0.92792, "recall_at_1": 0.88, "recall_at_3": 0.99, "recall_at_5": 0.99, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.88, "precision_at_3": 0.33, "precision_at_5": 0.198, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.89, "mrr_at_3": 0.9316666666666668, "mrr_at_5": 0.9316666666666668, "mrr_at_10": 0.9329166666666667, "mrr_at_20": 0.9329166666666667, "mrr_at_50": 0.9329166666666667, "mrr_at_100": 0.9329166666666667, "naucs_at_1_max": 0.477529430480433, "naucs_at_1_std": -0.17256601972637695, "naucs_at_1_diff1": 0.8993795736557426, "naucs_at_3_max": 1.0, "naucs_at_3_std": 0.35807656395889226, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 0.35807656395891135, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null}, "syntheticDocQA_artificial_intelligence_test": {"ndcg_at_1": 0.98, "ndcg_at_3": 0.98631, "ndcg_at_5": 0.99018, "ndcg_at_10": 0.99018, "ndcg_at_20": 0.99018, "ndcg_at_50": 0.99018, "ndcg_at_100": 0.99018, "map_at_1": 0.98, "map_at_3": 0.985, "map_at_5": 0.987, "map_at_10": 0.987, "map_at_20": 0.987, "map_at_50": 0.987, "map_at_100": 0.987, "recall_at_1": 0.98, "recall_at_3": 0.99, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.98, "precision_at_3": 0.33, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.98, "mrr_at_3": 0.985, "mrr_at_5": 0.987, "mrr_at_10": 0.987, "mrr_at_20": 0.987, "mrr_at_50": 0.987, "mrr_at_100": 0.987, "naucs_at_1_max": 1.0, "naucs_at_1_std": 0.8692810457516332, "naucs_at_1_diff1": 1.0, "naucs_at_3_max": 1.0, "naucs_at_3_std": 0.8692810457516356, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null}, "syntheticDocQA_energy_test": {"ndcg_at_1": 0.9, "ndcg_at_3": 0.92762, "ndcg_at_5": 0.93149, "ndcg_at_10": 0.93149, "ndcg_at_20": 0.93698, "ndcg_at_50": 0.9409, "ndcg_at_100": 0.9409, "map_at_1": 0.9, "map_at_3": 0.92, "map_at_5": 0.922, "map_at_10": 0.922, "map_at_20": 0.92374, "map_at_50": 0.92435, "map_at_100": 0.92435, "recall_at_1": 0.9, "recall_at_3": 0.95, "recall_at_5": 0.96, "recall_at_10": 0.96, "recall_at_20": 0.98, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.9, "precision_at_3": 0.31667, "precision_at_5": 0.192, "precision_at_10": 0.096, "precision_at_20": 0.049, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.9, "mrr_at_3": 0.92, "mrr_at_5": 0.9220000000000002, "mrr_at_10": 0.9220000000000002, "mrr_at_20": 0.9237424242424244, "mrr_at_50": 0.924346027846028, "mrr_at_100": 0.924346027846028, "naucs_at_1_max": 0.7248832866479916, "naucs_at_1_std": -0.5339402427637734, "naucs_at_1_diff1": 0.9477124183006546, "naucs_at_3_max": 0.5314659197012136, "naucs_at_3_std": -0.9360410830999026, "naucs_at_3_diff1": 0.9477124183006521, "naucs_at_5_max": 0.7071661998132599, "naucs_at_5_std": -0.7350606909430335, "naucs_at_5_diff1": 0.9346405228758139, "naucs_at_10_max": 0.7071661998132599, "naucs_at_10_std": -0.7350606909430335, "naucs_at_10_diff1": 0.9346405228758139, "naucs_at_20_max": 1.0, "naucs_at_20_std": -0.024276377217554025, "naucs_at_20_diff1": 0.9346405228758136, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null}, "syntheticDocQA_government_reports_test": {"ndcg_at_1": 0.96, "ndcg_at_3": 0.97893, "ndcg_at_5": 0.97893, "ndcg_at_10": 0.98194, "ndcg_at_20": 0.98194, "ndcg_at_50": 0.98194, "ndcg_at_100": 0.98194, "map_at_1": 0.96, "map_at_3": 0.975, "map_at_5": 0.975, "map_at_10": 0.97611, "map_at_20": 0.97611, "map_at_50": 0.97611, "map_at_100": 0.97611, "recall_at_1": 0.96, "recall_at_3": 0.99, "recall_at_5": 0.99, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.96, "precision_at_3": 0.33, "precision_at_5": 0.198, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.96, "mrr_at_3": 0.975, "mrr_at_5": 0.975, "mrr_at_10": 0.9761111111111112, "mrr_at_20": 0.9761111111111112, "mrr_at_50": 0.9761111111111112, "mrr_at_100": 0.9761111111111112, "naucs_at_1_max": 0.5997899159663848, "naucs_at_1_std": -1.2007469654528418, "naucs_at_1_diff1": 0.96732026143791, "naucs_at_3_max": 0.7222222222222157, "naucs_at_3_std": -1.7399626517274398, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 0.7222222222222276, "naucs_at_5_std": -1.7399626517273863, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null}, "syntheticDocQA_healthcare_industry_test": {"ndcg_at_1": 0.97, "ndcg_at_3": 0.98893, "ndcg_at_5": 0.98893, "ndcg_at_10": 0.98893, "ndcg_at_20": 0.98893, "ndcg_at_50": 0.98893, "ndcg_at_100": 0.98893, "map_at_1": 0.97, "map_at_3": 0.985, "map_at_5": 0.985, "map_at_10": 0.985, "map_at_20": 0.985, "map_at_50": 0.985, "map_at_100": 0.985, "recall_at_1": 0.97, "recall_at_3": 1.0, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.97, "precision_at_3": 0.33333, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.97, "mrr_at_3": 0.985, "mrr_at_5": 0.985, "mrr_at_10": 0.985, "mrr_at_20": 0.985, "mrr_at_50": 0.985, "mrr_at_100": 0.985, "naucs_at_1_max": 0.8078120136943662, "naucs_at_1_std": 0.4733893557422995, "naucs_at_1_diff1": 1.0, "naucs_at_3_max": 1.0, "naucs_at_3_std": 1.0, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null}, "tabfquad_test_subsampled": {"ndcg_at_1": 0.91429, "ndcg_at_3": 0.9372, "ndcg_at_5": 0.94612, "ndcg_at_10": 0.94612, "ndcg_at_20": 0.94992, "ndcg_at_50": 0.9522, "ndcg_at_100": 0.9522, "map_at_1": 0.91429, "map_at_3": 0.93155, "map_at_5": 0.93655, "map_at_10": 0.93655, "map_at_20": 0.93769, "map_at_50": 0.93813, "map_at_100": 0.93813, "recall_at_1": 0.91429, "recall_at_3": 0.95357, "recall_at_5": 0.975, "recall_at_10": 0.975, "recall_at_20": 0.98929, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.91429, "precision_at_3": 0.31786, "precision_at_5": 0.195, "precision_at_10": 0.0975, "precision_at_20": 0.04946, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.9142857142857143, "mrr_at_3": 0.9315476190476192, "mrr_at_5": 0.9365476190476191, "mrr_at_10": 0.9365476190476191, "mrr_at_20": 0.9376930014430014, "mrr_at_50": 0.9381267131585722, "mrr_at_100": 0.9381267131585722, "naucs_at_1_max": 0.8191332088390891, "naucs_at_1_std": 0.4374610955493308, "naucs_at_1_diff1": 0.8954442888266413, "naucs_at_3_max": 0.9484665661136237, "naucs_at_3_std": 0.7109818286288868, "naucs_at_3_diff1": 0.9484665661136237, "naucs_at_5_max": 0.9416433239962654, "naucs_at_5_std": 0.8163265306122475, "naucs_at_5_diff1": 0.9439775910364095, "naucs_at_10_max": 0.9416433239962654, "naucs_at_10_std": 0.8163265306122475, "naucs_at_10_diff1": 0.9439775910364095, "naucs_at_20_max": 0.9564270152505505, "naucs_at_20_std": 0.8638344226579515, "naucs_at_20_diff1": 0.9564270152505505, "naucs_at_50_max": 1.0, "naucs_at_50_std": 1.0, "naucs_at_50_diff1": 1.0, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0}, "tatdqa_test": {"ndcg_at_1": 0.57594, "ndcg_at_3": 0.68196, "ndcg_at_5": 0.71051, "ndcg_at_10": 0.73262, "ndcg_at_20": 0.74285, "ndcg_at_50": 0.75016, "ndcg_at_100": 0.75232, "map_at_1": 0.57594, "map_at_3": 0.65583, "map_at_5": 0.67169, "map_at_10": 0.68088, "map_at_20": 0.68373, "map_at_50": 0.68489, "map_at_100": 0.68507, "recall_at_1": 0.57594, "recall_at_3": 0.75759, "recall_at_5": 0.82685, "recall_at_10": 0.8949, "recall_at_20": 0.93499, "recall_at_50": 0.97205, "recall_at_100": 0.98542, "precision_at_1": 0.57594, "precision_at_3": 0.25253, "precision_at_5": 0.16537, "precision_at_10": 0.08949, "precision_at_20": 0.04675, "precision_at_50": 0.01944, "precision_at_100": 0.00985, "mrr_at_1": 0.5765492102065614, "mrr_at_3": 0.6560348319157563, "mrr_at_5": 0.6721952207371417, "mrr_at_10": 0.6813224459488133, "mrr_at_20": 0.6841812145327757, "mrr_at_50": 0.6853427671085895, "mrr_at_100": 0.6855309554803338, "naucs_at_1_max": 0.5301694754533977, "naucs_at_1_std": -0.1619263455379304, "naucs_at_1_diff1": 0.7278065996239447, "naucs_at_3_max": 0.5908076044993518, "naucs_at_3_std": -0.1323722551834884, "naucs_at_3_diff1": 0.6135897839866084, "naucs_at_5_max": 0.5964907222680631, "naucs_at_5_std": -0.12904588858296043, "naucs_at_5_diff1": 0.5731447174171511, "naucs_at_10_max": 0.6224287679728321, "naucs_at_10_std": 0.0010190169826118439, "naucs_at_10_diff1": 0.5292659611089736, "naucs_at_20_max": 0.7484085475471819, "naucs_at_20_std": 0.2475268741958173, "naucs_at_20_diff1": 0.5572740928407118, "naucs_at_50_max": 0.7916713620973378, "naucs_at_50_std": 0.35271506013966625, "naucs_at_50_diff1": 0.6134648860897702, "naucs_at_100_max": 0.83478407909221, "naucs_at_100_std": 0.6562392206248636, "naucs_at_100_diff1": 0.594171346126163}}

special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,143 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "left",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}