|
|
@@ -0,0 +1,633 @@
|
|
|
+# Copyright © 2024 Apple Inc.
|
|
|
+
|
|
|
+import math
|
|
|
+import inspect
|
|
|
+from dataclasses import dataclass, field
|
|
|
+from typing import Optional, Dict, Union
|
|
|
+
|
|
|
+import mlx.core as mx
|
|
|
+import mlx.nn as nn
|
|
|
+from mlx_lm.models.base import BaseModelArgs, KVCache
|
|
|
+from exo.inference.shard import Shard
|
|
|
+from .base import IdentityBlock
|
|
|
+import numpy as np
|
|
|
+
|
|
|
+
|
|
|
+@dataclass
|
|
|
+class VisionConfig:
|
|
|
+ model_type: str
|
|
|
+ num_hidden_layers: int = 24
|
|
|
+ hidden_size: int = 1024
|
|
|
+ intermediate_size: int = 4096
|
|
|
+ num_attention_heads: int = 16
|
|
|
+ image_size: int = 336
|
|
|
+ patch_size: int = 14
|
|
|
+ projection_dim: int = 768
|
|
|
+ vocab_size: int = 32000
|
|
|
+ num_channels: int = 3
|
|
|
+ layer_norm_eps: float = 1e-5
|
|
|
+
|
|
|
+ @classmethod
|
|
|
+ def from_dict(cls, params):
|
|
|
+ return cls(
|
|
|
+ **{
|
|
|
+ k: v
|
|
|
+ for k, v in params.items()
|
|
|
+ if k in inspect.signature(cls).parameters
|
|
|
+ }
|
|
|
+ )
|
|
|
+
|
|
|
+
|
|
|
+class VisionAttention(nn.Module):
|
|
|
+ def __init__(
|
|
|
+ self,
|
|
|
+ dims: int,
|
|
|
+ num_heads: int,
|
|
|
+ query_input_dims: Optional[int] = None,
|
|
|
+ key_input_dims: Optional[int] = None,
|
|
|
+ value_input_dims: Optional[int] = None,
|
|
|
+ value_dims: Optional[int] = None,
|
|
|
+ value_output_dims: Optional[int] = None,
|
|
|
+ bias: bool = False,
|
|
|
+ ):
|
|
|
+ super().__init__()
|
|
|
+
|
|
|
+ if (dims % num_heads) != 0:
|
|
|
+ raise ValueError(
|
|
|
+ "The input feature dimensions should be divisible by the "
|
|
|
+ f"number of heads ({dims} % {num_heads}) != 0"
|
|
|
+ )
|
|
|
+
|
|
|
+ query_input_dims = query_input_dims or dims
|
|
|
+ key_input_dims = key_input_dims or dims
|
|
|
+ value_input_dims = value_input_dims or key_input_dims
|
|
|
+ value_dims = value_dims or dims
|
|
|
+ value_output_dims = value_output_dims or dims
|
|
|
+
|
|
|
+ self.num_heads = num_heads
|
|
|
+ self.q_proj = nn.Linear(query_input_dims, dims, bias=bias)
|
|
|
+ self.k_proj = nn.Linear(key_input_dims, dims, bias=bias)
|
|
|
+ self.v_proj = nn.Linear(value_input_dims, value_dims, bias=bias)
|
|
|
+ self.out_proj = nn.Linear(value_dims, value_output_dims, bias=bias)
|
|
|
+
|
|
|
+ def __call__(self, queries, keys, values, mask=None):
|
|
|
+ queries = self.q_proj(queries)
|
|
|
+ keys = self.k_proj(keys)
|
|
|
+ values = self.v_proj(values)
|
|
|
+
|
|
|
+ num_heads = self.num_heads
|
|
|
+ B, L, D = queries.shape
|
|
|
+ _, S, _ = keys.shape
|
|
|
+ queries = queries.reshape(B, L, num_heads, -1).transpose(0, 2, 1, 3)
|
|
|
+ keys = keys.reshape(B, S, num_heads, -1).transpose(0, 2, 3, 1)
|
|
|
+ values = values.reshape(B, S, num_heads, -1).transpose(0, 2, 1, 3)
|
|
|
+
|
|
|
+ scale = math.sqrt(1 / queries.shape[-1])
|
|
|
+ scores = (queries * scale) @ keys
|
|
|
+ if mask is not None:
|
|
|
+ scores = scores + mask.astype(scores.dtype)
|
|
|
+ scores = mx.softmax(scores, axis=-1)
|
|
|
+ values_hat = (scores @ values).transpose(0, 2, 1, 3).reshape(B, L, -1)
|
|
|
+
|
|
|
+ return self.out_proj(values_hat)
|
|
|
+
|
|
|
+
|
|
|
+class VisionMLP(nn.Module):
|
|
|
+ def __init__(self, config: VisionConfig):
|
|
|
+ super().__init__()
|
|
|
+ self.activation_fn = nn.GELU(approx="fast")
|
|
|
+ self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size)
|
|
|
+ self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size)
|
|
|
+
|
|
|
+ def __call__(self, x: mx.array) -> mx.array:
|
|
|
+ x = self.activation_fn(self.fc1(x))
|
|
|
+ x = self.fc2(x)
|
|
|
+ return x
|
|
|
+
|
|
|
+
|
|
|
+class VisionEncoderLayer(nn.Module):
|
|
|
+ def __init__(self, config: VisionConfig):
|
|
|
+ super().__init__()
|
|
|
+ self.embed_dim = config.hidden_size
|
|
|
+ self.self_attn = VisionAttention(
|
|
|
+ config.hidden_size, config.num_attention_heads, bias=True
|
|
|
+ )
|
|
|
+ self.layer_norm1 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
|
|
|
+ self.mlp = VisionMLP(config)
|
|
|
+ self.layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
|
|
|
+
|
|
|
+ def __call__(self, x: mx.array, mask: Optional[mx.array] = None) -> mx.array:
|
|
|
+ y = self.layer_norm1(x)
|
|
|
+ y = self.self_attn(y, y, y, mask)
|
|
|
+ x = x + y
|
|
|
+ y = self.layer_norm2(x)
|
|
|
+ y = self.mlp(y)
|
|
|
+ return x + y
|
|
|
+
|
|
|
+
|
|
|
+class VisionEncoder(nn.Module):
|
|
|
+ def __init__(self, config: VisionConfig):
|
|
|
+ super().__init__()
|
|
|
+ self.layers = [VisionEncoderLayer(config) for _ in range(config.num_hidden_layers)]
|
|
|
+
|
|
|
+
|
|
|
+class VisionEmbeddings(nn.Module):
|
|
|
+ def __init__(self, config: VisionConfig):
|
|
|
+ super().__init__()
|
|
|
+ self.config = config
|
|
|
+ self.embed_dim = config.hidden_size
|
|
|
+ self.image_size = config.image_size
|
|
|
+ self.patch_size = config.patch_size
|
|
|
+
|
|
|
+ self.class_embedding = mx.zeros((config.hidden_size,))
|
|
|
+
|
|
|
+ self.patch_embedding = nn.Conv2d(
|
|
|
+ in_channels=config.num_channels,
|
|
|
+ out_channels=self.embed_dim,
|
|
|
+ kernel_size=self.patch_size,
|
|
|
+ stride=self.patch_size,
|
|
|
+ bias=False,
|
|
|
+ )
|
|
|
+
|
|
|
+ self.num_patches = (self.image_size // self.patch_size) ** 2
|
|
|
+ self.num_positions = self.num_patches + 1
|
|
|
+ self.position_embedding = nn.Embedding(self.num_positions, self.embed_dim)
|
|
|
+
|
|
|
+ def __call__(self, x: mx.array) -> mx.array:
|
|
|
+ batch_size = x.shape[0]
|
|
|
+ patch_embeddings = self.patch_embedding(x)
|
|
|
+ patch_embeddings = mx.flatten(patch_embeddings, start_axis=1, end_axis=2)
|
|
|
+ embed_dim = patch_embeddings.shape[-1]
|
|
|
+ cls_embeddings = mx.broadcast_to(
|
|
|
+ self.class_embedding, (batch_size, 1, embed_dim)
|
|
|
+ )
|
|
|
+ embeddings = mx.concatenate((cls_embeddings, patch_embeddings), axis=1)
|
|
|
+ embeddings += self.position_embedding.weight
|
|
|
+ return embeddings
|
|
|
+
|
|
|
+
|
|
|
+class ClipVisionModel(nn.Module):
|
|
|
+ def __init__(self, config: VisionConfig):
|
|
|
+ super().__init__()
|
|
|
+ self.embeddings = VisionEmbeddings(config)
|
|
|
+ self.pre_layrnorm = nn.LayerNorm(config.hidden_size)
|
|
|
+ self.encoder = VisionEncoder(config)
|
|
|
+ self.post_layernorm = nn.LayerNorm(config.hidden_size)
|
|
|
+
|
|
|
+ def __call__(
|
|
|
+ self,
|
|
|
+ x: mx.array,
|
|
|
+ output_hidden_states: Optional[bool] = None,
|
|
|
+ ) -> mx.array:
|
|
|
+ x = self.embeddings(x)
|
|
|
+ x = self.pre_layrnorm(x)
|
|
|
+
|
|
|
+ encoder_states = (x,) if output_hidden_states else None
|
|
|
+
|
|
|
+ for l in self.encoder.layers:
|
|
|
+ x = l(x, mask=None)
|
|
|
+ if output_hidden_states:
|
|
|
+ encoder_states = encoder_states + (x,)
|
|
|
+
|
|
|
+ pooler_output = self.post_layernorm(x[:, 0, :])
|
|
|
+ return pooler_output, x, encoder_states
|
|
|
+
|
|
|
+
|
|
|
+class VisionModel(nn.Module):
|
|
|
+ def __init__(self, config: VisionConfig):
|
|
|
+ super().__init__()
|
|
|
+
|
|
|
+ self.model_type = config.model_type
|
|
|
+ if self.model_type != "clip_vision_model":
|
|
|
+ raise ValueError(f"Unsupported model type: {self.model_type}")
|
|
|
+
|
|
|
+ self.vision_model = ClipVisionModel(config)
|
|
|
+
|
|
|
+ def __call__(
|
|
|
+ self, x: mx.array, output_hidden_states: Optional[bool] = None
|
|
|
+ ) -> mx.array:
|
|
|
+ return self.vision_model(x, output_hidden_states)
|
|
|
+
|
|
|
+ def sanitize(self, weights):
|
|
|
+ sanitized_weights = {}
|
|
|
+ for k, v in weights.items():
|
|
|
+ if "position_ids" in k:
|
|
|
+ # Remove unused position_ids
|
|
|
+ continue
|
|
|
+ elif "patch_embedding.weight" in k:
|
|
|
+ # PyTorch conv2d weight tensors have shape:
|
|
|
+ # [out_channels, in_channels, kH, KW]
|
|
|
+ # MLX conv2d expects the weight be of shape:
|
|
|
+ # [out_channels, kH, KW, in_channels]
|
|
|
+ sanitized_weights[k] = v.transpose(0, 2, 3, 1)
|
|
|
+ else:
|
|
|
+ sanitized_weights[k] = v
|
|
|
+
|
|
|
+ return sanitized_weights
|
|
|
+
|
|
|
+
|
|
|
+@dataclass
|
|
|
+class TextConfig:
|
|
|
+ model_type: str
|
|
|
+ hidden_size: int = 4096
|
|
|
+ num_hidden_layers: int = 32
|
|
|
+ intermediate_size: int = 11008
|
|
|
+ num_attention_heads: int = 32
|
|
|
+ head_dim: int = None
|
|
|
+ rms_norm_eps: float = 1e-6
|
|
|
+ vocab_size: int = 32000
|
|
|
+ num_key_value_heads: int = None
|
|
|
+ rope_theta: float = 10000
|
|
|
+ rope_traditional: bool = False
|
|
|
+ rope_scaling: Optional[Dict[str, Union[float, str]]] = None
|
|
|
+
|
|
|
+ @classmethod
|
|
|
+ def from_dict(cls, params):
|
|
|
+ return cls(
|
|
|
+ **{
|
|
|
+ k: v
|
|
|
+ for k, v in params.items()
|
|
|
+ if k in inspect.signature(cls).parameters
|
|
|
+ }
|
|
|
+ )
|
|
|
+
|
|
|
+ def __post_init__(self):
|
|
|
+ if self.num_key_value_heads is None:
|
|
|
+ self.num_key_value_heads = self.num_attention_heads
|
|
|
+
|
|
|
+ if self.head_dim is None:
|
|
|
+ self.head_dim = self.hidden_size // self.num_attention_heads
|
|
|
+
|
|
|
+ if self.model_type is None:
|
|
|
+ self.model_type = "llama"
|
|
|
+
|
|
|
+ if self.rope_scaling:
|
|
|
+ required_keys = {"factor", "type"}
|
|
|
+ if not all(key in self.rope_scaling for key in required_keys):
|
|
|
+ raise ValueError(f"rope_scaling must contain keys {required_keys}")
|
|
|
+
|
|
|
+ if self.rope_scaling["type"] != "linear":
|
|
|
+ raise ValueError("rope_scaling 'type' currently only supports 'linear'")
|
|
|
+
|
|
|
+
|
|
|
+class TextAttention(nn.Module):
|
|
|
+ def __init__(self, config: TextConfig):
|
|
|
+ super().__init__()
|
|
|
+
|
|
|
+ dim = config.hidden_size
|
|
|
+ self.n_heads = n_heads = config.num_attention_heads
|
|
|
+ self.n_kv_heads = n_kv_heads = config.num_key_value_heads
|
|
|
+
|
|
|
+ self.repeats = n_heads // n_kv_heads
|
|
|
+
|
|
|
+ head_dim = config.hidden_size // n_heads
|
|
|
+ self.scale = head_dim ** -0.5
|
|
|
+
|
|
|
+ self.q_proj = nn.Linear(dim, n_heads * head_dim, bias=False)
|
|
|
+ self.k_proj = nn.Linear(dim, n_kv_heads * head_dim, bias=False)
|
|
|
+ self.v_proj = nn.Linear(dim, n_kv_heads * head_dim, bias=False)
|
|
|
+ self.o_proj = nn.Linear(n_heads * head_dim, dim, bias=False)
|
|
|
+
|
|
|
+ rope_scale = (
|
|
|
+ 1 / config.rope_scaling["factor"]
|
|
|
+ if config.rope_scaling is not None
|
|
|
+ and config.rope_scaling["type"] == "linear"
|
|
|
+ else 1
|
|
|
+ )
|
|
|
+ self.rope = nn.RoPE(
|
|
|
+ head_dim,
|
|
|
+ traditional=config.rope_traditional,
|
|
|
+ base=config.rope_theta,
|
|
|
+ scale=rope_scale,
|
|
|
+ )
|
|
|
+
|
|
|
+ def __call__(
|
|
|
+ self,
|
|
|
+ x: mx.array,
|
|
|
+ mask: Optional[mx.array] = None,
|
|
|
+ cache: Optional[KVCache] = None,
|
|
|
+ ) -> mx.array:
|
|
|
+ B, L, D = x.shape
|
|
|
+
|
|
|
+ queries, keys, values = self.q_proj(x), self.k_proj(x), self.v_proj(x)
|
|
|
+
|
|
|
+ # Prepare the queries, keys and values for the attention computation
|
|
|
+ queries = queries.reshape(B, L, self.n_heads, -1).transpose(0, 2, 1, 3)
|
|
|
+ keys = keys.reshape(B, L, self.n_kv_heads, -1).transpose(0, 2, 1, 3)
|
|
|
+ values = values.reshape(B, L, self.n_kv_heads, -1).transpose(0, 2, 1, 3)
|
|
|
+
|
|
|
+ if cache is not None:
|
|
|
+ queries = self.rope(queries, offset=cache.offset)
|
|
|
+ keys = self.rope(keys, offset=cache.offset)
|
|
|
+ keys, values = cache.update_and_fetch(keys, values)
|
|
|
+ else:
|
|
|
+ queries = self.rope(queries)
|
|
|
+ keys = self.rope(keys)
|
|
|
+
|
|
|
+ output = mx.fast.scaled_dot_product_attention(
|
|
|
+ queries, keys, values, scale=self.scale, mask=mask
|
|
|
+ )
|
|
|
+ output = output.transpose(0, 2, 1, 3).reshape(B, L, -1)
|
|
|
+ return self.o_proj(output)
|
|
|
+
|
|
|
+
|
|
|
+class TextMLP(nn.Module):
|
|
|
+ def __init__(self, dim, hidden_dim):
|
|
|
+ super().__init__()
|
|
|
+ self.gate_proj = nn.Linear(dim, hidden_dim, bias=False)
|
|
|
+ self.down_proj = nn.Linear(hidden_dim, dim, bias=False)
|
|
|
+ self.up_proj = nn.Linear(dim, hidden_dim, bias=False)
|
|
|
+
|
|
|
+ def __call__(self, x) -> mx.array:
|
|
|
+ return self.down_proj(nn.silu(self.gate_proj(x)) * self.up_proj(x))
|
|
|
+
|
|
|
+
|
|
|
+class TransformerBlock(nn.Module):
|
|
|
+ def __init__(self, config: TextConfig):
|
|
|
+ super().__init__()
|
|
|
+ self.num_attention_heads = config.num_attention_heads
|
|
|
+ self.hidden_size = config.hidden_size
|
|
|
+ self.self_attn = TextAttention(config)
|
|
|
+ self.mlp = TextMLP(config.hidden_size, config.intermediate_size)
|
|
|
+ self.input_layernorm = nn.RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
|
|
+ self.post_attention_layernorm = nn.RMSNorm(
|
|
|
+ config.hidden_size, eps=config.rms_norm_eps
|
|
|
+ )
|
|
|
+ self.config = config
|
|
|
+
|
|
|
+ def __call__(
|
|
|
+ self,
|
|
|
+ x: mx.array,
|
|
|
+ mask: Optional[mx.array] = None,
|
|
|
+ cache: Optional[KVCache] = None,
|
|
|
+ ) -> mx.array:
|
|
|
+ r = self.self_attn(self.input_layernorm(x), mask, cache)
|
|
|
+ h = x + r
|
|
|
+ r = self.mlp(self.post_attention_layernorm(h))
|
|
|
+ out = h + r
|
|
|
+ return out
|
|
|
+
|
|
|
+
|
|
|
+class Llama(nn.Module):
|
|
|
+ def __init__(self, config: TextConfig, shard: Shard):
|
|
|
+ super().__init__()
|
|
|
+ self.config = config
|
|
|
+ self.shard = shard
|
|
|
+ self.vocab_size = config.vocab_size
|
|
|
+ self.model_type = config.model_type
|
|
|
+ self.num_hidden_layers = config.num_hidden_layers
|
|
|
+ self.num_key_value_heads = config.num_key_value_heads
|
|
|
+ self.head_dim = config.head_dim
|
|
|
+ assert self.vocab_size > 0
|
|
|
+ if self.shard.is_first_layer():
|
|
|
+ self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size)
|
|
|
+ self.layers = []
|
|
|
+ for i in range(self.num_hidden_layers):
|
|
|
+ if self.shard.start_layer <= i <= self.shard.end_layer:
|
|
|
+ self.layers.append(TransformerBlock(config=config))
|
|
|
+ else:
|
|
|
+ self.layers.append(IdentityBlock())
|
|
|
+ if self.shard.is_last_layer():
|
|
|
+ self.norm = nn.RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
|
|
+
|
|
|
+ def __call__(
|
|
|
+ self,
|
|
|
+ inputs: mx.array,
|
|
|
+ cache=None,
|
|
|
+ inputs_embeds=None,
|
|
|
+ ):
|
|
|
+ # for passing merged input embeddings
|
|
|
+ if inputs_embeds is None:
|
|
|
+ if self.shard.is_first_layer():
|
|
|
+ h = self.embed_tokens(inputs)
|
|
|
+ else:
|
|
|
+ h = inputs
|
|
|
+ else:
|
|
|
+ h = inputs_embeds
|
|
|
+
|
|
|
+ mask = None
|
|
|
+ if h.shape[1] > 1:
|
|
|
+ mask = nn.MultiHeadAttention.create_additive_causal_mask(h.shape[1])
|
|
|
+ mask = mask.astype(h.dtype)
|
|
|
+
|
|
|
+ if cache is None:
|
|
|
+ cache = [None] * len(self.layers)
|
|
|
+
|
|
|
+
|
|
|
+ for layer, c in zip(self.layers, cache):
|
|
|
+ h = layer(h, mask, c)
|
|
|
+
|
|
|
+ if self.shard.is_last_layer():
|
|
|
+ h = self.norm(h)
|
|
|
+ return h
|
|
|
+
|
|
|
+class LanguageModel(nn.Module):
|
|
|
+ def __init__(self, config: TextConfig, shard: Shard):
|
|
|
+ super().__init__()
|
|
|
+ self.model_type = config.model_type
|
|
|
+ if self.model_type != "llama":
|
|
|
+ raise ValueError(
|
|
|
+ f"Model type {self.model_type} not supported. Currently only 'llama' is supported"
|
|
|
+ )
|
|
|
+ self.shard = shard
|
|
|
+ self.model = Llama(config, shard)
|
|
|
+ if self.shard.is_last_layer():
|
|
|
+ self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
|
|
|
+
|
|
|
+ def __call__(
|
|
|
+ self,
|
|
|
+ inputs: mx.array,
|
|
|
+ cache=None,
|
|
|
+ inputs_embeds=None,
|
|
|
+ ):
|
|
|
+ out = self.model(inputs, cache, inputs_embeds)
|
|
|
+ if self.shard.is_last_layer():
|
|
|
+ out = self.lm_head(out)
|
|
|
+ return out
|
|
|
+
|
|
|
+ def sanitize(self, weights):
|
|
|
+ shard_state_dict = {}
|
|
|
+ for key, value in weights.items():
|
|
|
+ if "self_attn.rotary_emb.inv_freq" in key:
|
|
|
+ continue
|
|
|
+
|
|
|
+ if key.startswith('language_model.model.layers.'):
|
|
|
+ layer_num = int(key.split('.')[3])
|
|
|
+ if layer_num < self.shard.start_layer or layer_num > self.shard.end_layer:
|
|
|
+ continue
|
|
|
+ if not self.shard.is_first_layer() and key.startswith('language_model.model.embed_tokens'):
|
|
|
+ continue
|
|
|
+ elif not self.shard.is_last_layer() and (key.startswith('language_model.model.norm') or key.startswith('language_model.lm_head')):
|
|
|
+ continue
|
|
|
+
|
|
|
+ shard_state_dict[key] = value
|
|
|
+
|
|
|
+ return shard_state_dict
|
|
|
+
|
|
|
+@dataclass
|
|
|
+class LlaVAConfig(BaseModelArgs):
|
|
|
+ text_config: TextConfig
|
|
|
+ vision_config: VisionConfig = None
|
|
|
+ model_type: str = "llava"
|
|
|
+ ignore_index: int = -100
|
|
|
+ image_token_index: int = 32000
|
|
|
+ vision_feature_select_strategy: str = "default"
|
|
|
+ vision_feature_layer: int = -2
|
|
|
+ vocab_size: int = 32000
|
|
|
+
|
|
|
+ @classmethod
|
|
|
+ def from_dict(cls, params):
|
|
|
+ updated_params = {}
|
|
|
+ class_params = inspect.signature(cls).parameters
|
|
|
+ for k, v in params.items():
|
|
|
+ if k in class_params:
|
|
|
+ if k in ["text_config", "vision_config"]:
|
|
|
+ v = class_params[k].annotation.from_dict(v)
|
|
|
+ updated_params.update({k: v})
|
|
|
+
|
|
|
+ return cls(**updated_params)
|
|
|
+
|
|
|
+
|
|
|
+@dataclass
|
|
|
+class ModelArgs(LlaVAConfig):
|
|
|
+ shard: Shard = field(default_factory=lambda: Shard("", 0, 0, 0))
|
|
|
+
|
|
|
+ def __post_init__(self):
|
|
|
+ if isinstance(self.shard, dict):
|
|
|
+ self.shard = Shard(**self.shard)
|
|
|
+
|
|
|
+ if not isinstance(self.shard, Shard):
|
|
|
+ raise TypeError(f"Expected shard to be a Shard instance or a dict, got {type(self.shard)} instead")
|
|
|
+
|
|
|
+ if not self.shard.is_first_layer():
|
|
|
+ self.vision_config = None
|
|
|
+
|
|
|
+
|
|
|
+class LlavaMultiModalProjector(nn.Module):
|
|
|
+ def __init__(self, config: LlaVAConfig):
|
|
|
+ super().__init__()
|
|
|
+ self.linear_1 = nn.Linear(
|
|
|
+ config.vision_config.hidden_size, config.text_config.hidden_size, bias=True
|
|
|
+ )
|
|
|
+ self.gelu = nn.GELU()
|
|
|
+ self.linear_2 = nn.Linear(
|
|
|
+ config.text_config.hidden_size, config.text_config.hidden_size, bias=True
|
|
|
+ )
|
|
|
+
|
|
|
+ def __call__(self, x: mx.array) -> mx.array:
|
|
|
+ x = self.linear_1(x)
|
|
|
+ x = self.gelu(x)
|
|
|
+ x = self.linear_2(x)
|
|
|
+ return x
|
|
|
+
|
|
|
+
|
|
|
+class Model(nn.Module):
|
|
|
+ def __init__(self, config: ModelArgs):
|
|
|
+ super().__init__()
|
|
|
+ self.config = config
|
|
|
+ self.model_type = config.model_type
|
|
|
+ if config.vision_config:
|
|
|
+ self.vision_tower = VisionModel(config.vision_config)
|
|
|
+ self.multi_modal_projector = LlavaMultiModalProjector(config)
|
|
|
+ self.vision_feature_layer = config.vision_feature_layer
|
|
|
+ self.vision_feature_select_strategy = config.vision_feature_select_strategy
|
|
|
+ self.language_model = LanguageModel(config.text_config, config.shard)
|
|
|
+
|
|
|
+ def get_input_embeddings(
|
|
|
+ self,
|
|
|
+ input_ids: Optional[mx.array] = None,
|
|
|
+ pixel_values: Optional[mx.array] = None,
|
|
|
+ ):
|
|
|
+ if pixel_values is None:
|
|
|
+ return self.language_model(input_ids)
|
|
|
+
|
|
|
+ # Get the input embeddings from the language model
|
|
|
+ inputs_embeds = self.language_model.model.embed_tokens(input_ids)
|
|
|
+
|
|
|
+ # Get the ouptut hidden states from the vision model
|
|
|
+ *_, hidden_states = self.vision_tower(
|
|
|
+ pixel_values.transpose(0, 2, 3, 1), output_hidden_states=True
|
|
|
+ )
|
|
|
+
|
|
|
+ # Select the hidden states from the desired layer
|
|
|
+ selected_image_feature = hidden_states[self.vision_feature_layer]
|
|
|
+
|
|
|
+ if self.vision_feature_select_strategy == "default":
|
|
|
+ selected_image_feature = selected_image_feature[:, 1:]
|
|
|
+ elif self.vision_feature_select_strategy == "full":
|
|
|
+ selected_image_feature = selected_image_feature
|
|
|
+ else:
|
|
|
+ raise ValueError(
|
|
|
+ "Unexpected feature selection strategy: "
|
|
|
+ f"{self.vision_feature_select_strategy}"
|
|
|
+ )
|
|
|
+
|
|
|
+ # Pass image features through the multi-modal projector
|
|
|
+ image_features = self.multi_modal_projector(selected_image_feature)
|
|
|
+
|
|
|
+ # Insert special image tokens in the input_ids
|
|
|
+ final_inputs_embeds = self._merge_input_ids_with_image_features(
|
|
|
+ image_features, inputs_embeds, input_ids
|
|
|
+ )
|
|
|
+ return final_inputs_embeds
|
|
|
+
|
|
|
+ def _merge_input_ids_with_image_features(
|
|
|
+ self, image_features, inputs_embeds, input_ids
|
|
|
+ ):
|
|
|
+ image_token_index = self.config.image_token_index
|
|
|
+ num_images, num_image_patches, embed_dim = image_features.shape
|
|
|
+
|
|
|
+ # Positions of <image> tokens in input_ids, assuming batch size is 1
|
|
|
+ image_positions = np.where(input_ids[0] == image_token_index)[0].tolist()
|
|
|
+
|
|
|
+ if len(image_positions) != num_images:
|
|
|
+ raise ValueError(
|
|
|
+ f"The number of image tokens ({len(image_positions)}) does not "
|
|
|
+ f" match the number of image inputs ({num_images})."
|
|
|
+ )
|
|
|
+
|
|
|
+ text_segments = []
|
|
|
+ start_idx = 0
|
|
|
+
|
|
|
+ for position in image_positions:
|
|
|
+ text_segments.append(inputs_embeds[:, start_idx:position])
|
|
|
+ start_idx = position + 1
|
|
|
+
|
|
|
+ image_embeddings = mx.split(image_features, image_features.shape[0])
|
|
|
+ final_embeddings = [v for p in zip(text_segments, image_embeddings) for v in p]
|
|
|
+ final_embeddings += [inputs_embeds[:, start_idx:]]
|
|
|
+
|
|
|
+ # Create a final embedding of shape
|
|
|
+ # (1, num_image_patches*num_images + sequence_len, embed_dim)
|
|
|
+ return mx.concatenate(final_embeddings, axis=1)
|
|
|
+
|
|
|
+ def __call__(self, input_ids: mx.array, pixel_values: mx.array = None, cache=None):
|
|
|
+ input_embddings = None
|
|
|
+ if pixel_values is not None:
|
|
|
+ input_embddings = self.get_input_embeddings(input_ids, pixel_values)
|
|
|
+ logits = self.language_model(
|
|
|
+ input_ids, cache=cache, inputs_embeds=input_embddings
|
|
|
+ )
|
|
|
+ return logits
|
|
|
+
|
|
|
+ def sanitize(self, weights):
|
|
|
+ if self.config.vision_config:
|
|
|
+ weights = self.vision_tower.sanitize(weights)
|
|
|
+ else:
|
|
|
+ weights = {k: v for k, v in weights.items() if not k.startswith(('vision_tower', 'multi_modal_projector', 'vision_feature_layer', 'vision_feature_select_strategy'))}
|
|
|
+ weights = self.language_model.sanitize(weights)
|
|
|
+ return weights
|
|
|
+
|
|
|
+ @property
|
|
|
+ def layers(self):
|
|
|
+ return self.language_model.model.layers
|
|
|
+
|
|
|
+ @property
|
|
|
+ def head_dim(self):
|
|
|
+ return (
|
|
|
+ self.language_model.model.head_dim or self.language_model.model.hidden_size // self.language_model.model.num_attention_heads
|
|
|
+ )
|
|
|
+
|
|
|
+ @property
|
|
|
+ def n_kv_heads(self):
|
|
|
+ return self.language_model.model.num_key_value_heads
|
Alex Cheema