Export Quark Instruct checkpoint

README.md

@@ -0,0 +1,56 @@
+---
+language:
+- en
+- it
+license: mit
+tags:
+- quark
+- causal-lm
+- bash
+- code
+pipeline_tag: text-generation
+---
+
+# Quark-72M-Instruct
+
+Quark-72M Instruct — compact autoregressive language model trained by [ThingAI](https://things-ai.org).
+
+## Model Details
+
+| Parameter | Value |
+|-----------|-------|
+| Parameters | 71.6M |
+| Architecture | Decoder-only Transformer |
+| Layers | 14 |
+| Hidden size | 512 |
+| Attention heads | 8 (GQA, 2 KV) |
+| FFN | SwiGLU (1344) |
+| Norm | RMSNorm |
+| Position | RoPE |
+| Vocab size | 65,538 |
+| Context length | 2,048 |
+
+## Usage
+
+```python
+from transformers import AutoTokenizer, AutoModelForCausalLM
+import torch
+
+tokenizer = AutoTokenizer.from_pretrained("ThingAI/Quark-72M-Instruct")
+model     = AutoModelForCausalLM.from_pretrained("ThingAI/Quark-72M-Instruct", trust_remote_code=True)
+
+prompt = "<|user|>\nHow do I find files larger than 100MB?\n<|end|>\n<|assistant|>\n"
+ids    = tokenizer(prompt, return_tensors="pt").input_ids
+out    = model.generate_text(ids, max_new_tokens=200, temperature=0.2)
+print(tokenizer.decode(out[0], skip_special_tokens=False))
+```
+
+## Training
+
+- **Pre-training**: 5B tokens on math, code, EN/IT text
+- **SFT**: bash commands, code, conversations (ChatML template)
+- **Tokenizer**: BPE byte-level, 65536 vocab
+
+## License
+
+MIT

config.json

@@ -0,0 +1,26 @@
+{
+  "model_type": "quark",
+  "architectures": [
+    "QuarkForCausalLM"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_quark.QuarkConfig",
+    "AutoModelForCausalLM": "modeling_quark.QuarkForCausalLM"
+  },
+  "vocab_size": 65538,
+  "d_model": 512,
+  "n_heads": 8,
+  "n_kv_heads": 2,
+  "n_layers": 14,
+  "d_ff": 1344,
+  "head_dim": 64,
+  "max_seq_len": 2048,
+  "max_position_embeddings": 2048,
+  "rope_theta": 10000.0,
+  "rms_eps": 1e-05,
+  "qkv_bias": true,
+  "dropout": 0.0,
+  "tie_word_embeddings": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.40.0"
+}

configuration_quark.py

@@ -0,0 +1,39 @@
+"""
+Quark model configuration — compatibile con AutoConfig / AutoModel HuggingFace.
+"""
+from transformers import PretrainedConfig
+
+
+class QuarkConfig(PretrainedConfig):
+    model_type = "quark"
+
+    def __init__(
+        self,
+        vocab_size   = 65_536,
+        d_model      = 512,
+        n_heads      = 8,
+        n_kv_heads   = 2,
+        n_layers     = 14,
+        d_ff         = 1344,
+        head_dim     = 64,
+        max_seq_len  = 2048,
+        rope_theta   = 10_000.0,
+        rms_eps      = 1e-5,
+        qkv_bias     = True,
+        dropout      = 0.0,
+        tie_word_embeddings = True,
+        **kwargs,
+    ):
+        self.vocab_size   = vocab_size
+        self.d_model      = d_model
+        self.n_heads      = n_heads
+        self.n_kv_heads   = n_kv_heads
+        self.n_layers     = n_layers
+        self.d_ff         = d_ff
+        self.head_dim     = head_dim
+        self.max_seq_len  = max_seq_len
+        self.rope_theta   = rope_theta
+        self.rms_eps      = rms_eps
+        self.qkv_bias     = qkv_bias
+        self.dropout      = dropout
+        super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)

generation_config.json

@@ -0,0 +1,9 @@
+{
+  "eos_token_id": 3,
+  "pad_token_id": 0,
+  "bos_token_id": 2,
+  "max_new_tokens": 512,
+  "temperature": 0.7,
+  "top_p": 0.9,
+  "do_sample": true
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9cbff9d64001da4c03728705430220b2178d1934523b8d9131c4b6e198c3ade5
+size 286646720

modeling_quark.py

@@ -0,0 +1,202 @@
+"""
+Quark language model — compatibile con AutoModelForCausalLM HuggingFace.
+"""
+import math
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from transformers import PreTrainedModel
+from transformers.modeling_outputs import CausalLMOutputWithPast
+
+from .configuration_quark import QuarkConfig
+
+
+class RMSNorm(nn.Module):
+    def __init__(self, dim, eps=1e-5):
+        super().__init__()
+        self.eps   = eps
+        self.scale = nn.Parameter(torch.ones(dim))
+
+    def forward(self, x):
+        rms = x.float().pow(2).mean(-1, keepdim=True).add(self.eps).rsqrt()
+        return (x.float() * rms).to(x.dtype) * self.scale
+
+
+class RotaryEmbedding(nn.Module):
+    def __init__(self, head_dim, max_seq_len, theta=10_000.0):
+        super().__init__()
+        inv_freq = 1.0 / (theta ** (torch.arange(0, head_dim, 2).float() / head_dim))
+        self.register_buffer("inv_freq", inv_freq, persistent=False)
+        self._build_cache(max_seq_len)
+
+    def _build_cache(self, seq_len):
+        t     = torch.arange(seq_len, device=self.inv_freq.device).float()
+        freqs = torch.outer(t, self.inv_freq)
+        emb   = torch.cat([freqs, freqs], dim=-1)
+        self.register_buffer("cos_cache", emb.cos()[None, None], persistent=False)
+        self.register_buffer("sin_cache", emb.sin()[None, None], persistent=False)
+        self._max = seq_len
+
+    @staticmethod
+    def _rotate_half(x):
+        x1, x2 = x.chunk(2, dim=-1)
+        return torch.cat([-x2, x1], dim=-1)
+
+    def forward(self, q, k):
+        T = q.size(2)
+        if T > self._max:
+            self._build_cache(T)
+        cos = self.cos_cache[:, :, :T, :]
+        sin = self.sin_cache[:, :, :T, :]
+        return q * cos + self._rotate_half(q) * sin,                k * cos + self._rotate_half(k) * sin
+
+
+class GroupedQueryAttention(nn.Module):
+    def __init__(self, cfg):
+        super().__init__()
+        self.n_heads    = cfg.n_heads
+        self.n_kv_heads = cfg.n_kv_heads
+        self.n_groups   = cfg.n_heads // cfg.n_kv_heads
+        self.head_dim   = cfg.head_dim
+
+        self.q_proj = nn.Linear(cfg.d_model, cfg.n_heads    * cfg.head_dim, bias=cfg.qkv_bias)
+        self.k_proj = nn.Linear(cfg.d_model, cfg.n_kv_heads * cfg.head_dim, bias=cfg.qkv_bias)
+        self.v_proj = nn.Linear(cfg.d_model, cfg.n_kv_heads * cfg.head_dim, bias=cfg.qkv_bias)
+        self.o_proj = nn.Linear(cfg.n_heads * cfg.head_dim, cfg.d_model,    bias=False)
+        self.rope   = RotaryEmbedding(cfg.head_dim, cfg.max_seq_len, cfg.rope_theta)
+        self.drop   = cfg.dropout
+
+    def forward(self, x, attention_mask=None, **kwargs):
+        B, T, _ = x.shape
+        q = self.q_proj(x).view(B, T, self.n_heads,    self.head_dim).transpose(1, 2)
+        k = self.k_proj(x).view(B, T, self.n_kv_heads, self.head_dim).transpose(1, 2)
+        v = self.v_proj(x).view(B, T, self.n_kv_heads, self.head_dim).transpose(1, 2)
+        q, k = self.rope(q, k)
+        if self.n_groups > 1:
+            k = k.repeat_interleave(self.n_groups, dim=1)
+            v = v.repeat_interleave(self.n_groups, dim=1)
+        out = F.scaled_dot_product_attention(
+            q, k, v, attn_mask=None,
+            dropout_p=self.drop if self.training else 0.0,
+            is_causal=True,
+        )
+        out = out.transpose(1, 2).contiguous().view(B, T, self.n_heads * self.head_dim)
+        return self.o_proj(out)
+
+
+class SwiGLUFFN(nn.Module):
+    def __init__(self, cfg):
+        super().__init__()
+        self.gate_proj = nn.Linear(cfg.d_model, cfg.d_ff, bias=False)
+        self.up_proj   = nn.Linear(cfg.d_model, cfg.d_ff, bias=False)
+        self.down_proj = nn.Linear(cfg.d_ff, cfg.d_model, bias=False)
+
+    def forward(self, x):
+        return self.down_proj(F.silu(self.gate_proj(x)) * self.up_proj(x))
+
+
+class TransformerBlock(nn.Module):
+    def __init__(self, cfg):
+        super().__init__()
+        self.norm_attn = RMSNorm(cfg.d_model, cfg.rms_eps)
+        self.attn      = GroupedQueryAttention(cfg)
+        self.norm_ffn  = RMSNorm(cfg.d_model, cfg.rms_eps)
+        self.ffn       = SwiGLUFFN(cfg)
+
+    def forward(self, x, **kwargs):
+        x = x + self.attn(self.norm_attn(x), **kwargs)
+        x = x + self.ffn(self.norm_ffn(x))
+        return x
+
+
+class QuarkPreTrainedModel(PreTrainedModel):
+    config_class     = QuarkConfig
+    base_model_prefix = "model"
+    supports_gradient_checkpointing = False
+
+    def _init_weights(self, module):
+        std = 0.02
+        if isinstance(module, nn.Linear):
+            nn.init.normal_(module.weight, 0.0, std)
+            if module.bias is not None:
+                nn.init.zeros_(module.bias)
+        elif isinstance(module, nn.Embedding):
+            nn.init.normal_(module.weight, 0.0, std)
+
+
+class QuarkForCausalLM(QuarkPreTrainedModel):
+    """
+    Quark autoregressive language model.
+    Compatibile con AutoModelForCausalLM.from_pretrained(..., trust_remote_code=True)
+    """
+    def __init__(self, config: QuarkConfig):
+        super().__init__(config)
+        self.embed_tokens = nn.Embedding(config.vocab_size, config.d_model)
+        self.layers       = nn.ModuleList([TransformerBlock(config) for _ in range(config.n_layers)])
+        self.norm         = RMSNorm(config.d_model, config.rms_eps)
+        self.lm_head      = nn.Linear(config.d_model, config.vocab_size, bias=False)
+
+        if config.tie_word_embeddings:
+            self.lm_head.weight = self.embed_tokens.weight
+
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.embed_tokens
+
+    def set_input_embeddings(self, value):
+        self.embed_tokens = value
+
+    def get_output_embeddings(self):
+        return self.lm_head
+
+    def set_output_embeddings(self, value):
+        self.lm_head = value
+
+    def forward(
+        self,
+        input_ids      = None,
+        attention_mask = None,
+        labels         = None,
+        **kwargs,
+    ):
+        x      = self.embed_tokens(input_ids)
+        for layer in self.layers:
+            x  = layer(x, attention_mask=attention_mask)
+        x      = self.norm(x)
+        logits = self.lm_head(x)
+
+        loss = None
+        if labels is not None:
+            loss = F.cross_entropy(
+                logits[:, :-1, :].contiguous().view(-1, self.config.vocab_size),
+                labels[:, 1:].contiguous().view(-1),
+                ignore_index=-100,
+            )
+
+        return CausalLMOutputWithPast(loss=loss, logits=logits)
+
+    @torch.no_grad()
+    def generate_text(self, input_ids, max_new_tokens=200, temperature=0.7,
+                      top_p=0.9, eos_token_id=None):
+        """Generazione semplice senza dipendere da .generate() HF."""
+        ctx = input_ids.clone()
+        for _ in range(max_new_tokens):
+            out    = self(ctx[:, -self.config.max_seq_len:])
+            logits = out.logits[0, -1, :].float()
+            if temperature > 0:
+                logits /= temperature
+                probs   = F.softmax(logits, dim=-1)
+                # top-p
+                sorted_p, sorted_i = torch.sort(probs, descending=True)
+                cum_p = torch.cumsum(sorted_p, dim=-1)
+                remove = cum_p - sorted_p > top_p
+                sorted_p[remove] = 0.0
+                sorted_p /= sorted_p.sum()
+                token = sorted_i[torch.multinomial(sorted_p, 1)].unsqueeze(0).unsqueeze(0)
+            else:
+                token = logits.argmax().view(1, 1)
+            ctx = torch.cat([ctx, token], dim=1)
+            if eos_token_id is not None and token.item() == eos_token_id:
+                break
+        return ctx

tokenizer_config.json

@@ -0,0 +1,9 @@
+{
+  "tokenizer_class": "AutoTokenizer",
+  "model_max_length": 2048,
+  "padding_side": "right",
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "unk_token": "<unk>",
+  "pad_token": "<pad>"
+}