Add ONNX variants for PangaiaSoftware-Rosetta2-4B

Variants: CPU-INT4, CUDA-INT4, DML-FP16 for 32-language translation

.gitattributes

@@ -1,35 +1,41 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+cpu_and_mobile/cpu-int4-rtn-block-32/model.onnx.data filter=lfs diff=lfs merge=lfs -text
+cpu_and_mobile/cpu-int4-rtn-block-32/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+cuda/cuda-int4-rtn-block-32/model.onnx.data filter=lfs diff=lfs merge=lfs -text
+cuda/cuda-int4-rtn-block-32/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+directml/dml-fp16/model.onnx.data filter=lfs diff=lfs merge=lfs -text
+directml/dml-fp16/tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,128 @@
+---
+library_name: onnx
+pipeline_tag: translation
+language:
+  - ar
+  - bg
+  - zh
+  - cs
+  - da
+  - nl
+  - en
+  - fi
+  - fr
+  - de
+  - el
+  - gu
+  - he
+  - hi
+  - hu
+  - id
+  - it
+  - ja
+  - ko
+  - fa
+  - pl
+  - pt
+  - ro
+  - ru
+  - sk
+  - es
+  - sv
+  - tl
+  - th
+  - tr
+  - uk
+  - vi
+license: gemma
+tags:
+- onnx
+- onnxruntime
+- optimum
+- translation
+- gemma
+- int4
+- quantized
+- cuda
+- directml
+base_model: google/gemma-3-4b-pt
+base_model_relation: quantized
+model-index:
+- name: PangaiaSoftware-Rosetta2-4B-ONNX
+  results:
+  - task:
+      type: translation
+      name: Translation
+    metrics:
+    - type: bleu
+      value: 31.5
+      name: BLEU Score
+---
+
+# PangaiaSoftware-Rosetta-4B-2511-ONNX
+
+## Introduction
+This repository hosts Pangaia Software's optimized versions of the [`YanoljaNEXT-Rosetta-4B-2511`](https://huggingface.co/yanolja/YanoljaNEXT-Rosetta-4B-2511) model to accelerate inference with ONNX Runtime.
+
+Optimized models are published here in ONNX format to run with ONNX Runtime on CPU and GPU across devices, including server platforms, Windows, Linux and Mac desktops, and mobile CPUs, with the precision best suited to each of these targets.
+
+Here are some of the optimized configurations we have added:  
+
+1. ONNX model for int4 CPU: ONNX model for CPU and mobile using int4 quantization via RTN.
+2. ONNX model for int4 GPU: ONNX model for GPU using fp16 quantization.
+
+## Model Run
+For CPU:
+
+```bash
+# Download the model directly using the Hugging Face CLI
+huggingface-cli download PangaiaSoftware/PangaiaSoftware-Rosetta2-4B-onnx --include cpu_and_mobile/* --local-dir .
+
+# Install the CPU package of ONNX Runtime GenAI
+pip install --pre onnxruntime-genai
+```
+
+For CUDA:
+
+```bash
+# Download the model directly using the Hugging Face CLI
+huggingface-cli download PangaiaSoftware/PangaiaSoftware-Rosetta2-4B-onnx --include cuda/* --local-dir .
+
+# Install the CUDA package of ONNX Runtime GenAI
+pip install --pre onnxruntime-genai-cuda
+```
+
+For DirectML:
+
+```bash
+# Download the model directly using the Hugging Face CLI
+huggingface-cli download PangaiaSoftware/PangaiaSoftware-Rosetta2-4B-onnx --include directml/* --local-dir .
+
+# Install the DML package of ONNX Runtime GenAI
+pip install --pre onnxruntime-genai-directml
+```
+
+Execution:
+
+Refer to the [`ONNX Runtime GenAI`](https://github.com/microsoft/onnxruntime-genai) repo for the latest samples for model execution. 
+
+**Note**: since this is a Gemma-based model, use the corresponding prompt template:
+
+```
+System = "<start_of_turn>instruction\n{{CONTENT}}<end_of_turn>\n",
+User = "<start_of_turn>source\n{{CONTENT}}<end_of_turn>\n",
+Assistant = "<start_of_turn>translation\n{{CONTENT}}<end_of_turn>\n",
+Stop = ["<end_of_turn>", "<start_of_turn>"]
+```
+
+
+## Model Description
+- Developed by: Pangaia Software
+- Model type: ONNX
+- License: gemma
+- Model Description: This is a conversion of the [`YanoljaNEXT-Rosetta-4B-2511`](https://huggingface.co/yanolja/YanoljaNEXT-Rosetta-4B-2511) model for ONNX Runtime inference, which in turn is based on the [`google/gemma-3-4b-pt`](https://huggingface.co/google/gemma-3-4b-pt) model.
+
+**Disclaimer:** Model is only an optimization of the base model, any risk associated with the model is the responsibility of the user of the model. Please verify and test for your scenarios. There may be a slight difference in output from the base model with the optimizations applied.
+
+## License
+This model is released under the Gemma license, inherited from its base model, [`google/gemma-3-4b-pt`](https://huggingface.co/google/gemma-3-4b-pt). Please consult the official [Gemma license terms](https://ai.google.dev/gemma/terms) for detailed usage guidelines.

config.json

@@ -0,0 +1,73 @@
+{
+  "_sliding_window_pattern": 6,
+  "architectures": [
+    "Gemma3ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "attn_logit_softcapping": null,
+  "bos_token_id": 2,
+  "cache_implementation": "hybrid",
+  "dtype": "bfloat16",
+  "eos_token_id": 106,
+  "final_logit_softcapping": null,
+  "head_dim": 256,
+  "hidden_activation": "gelu_pytorch_tanh",
+  "hidden_size": 2560,
+  "initializer_range": 0.02,
+  "intermediate_size": 10240,
+  "layer_types": [
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention"
+  ],
+  "max_position_embeddings": 131072,
+  "model_type": "gemma3_text",
+  "num_attention_heads": 8,
+  "num_hidden_layers": 34,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "query_pre_attn_scalar": 256,
+  "rms_norm_eps": 1e-06,
+  "rope_local_base_freq": 10000.0,
+  "rope_scaling": {
+    "factor": 8.0,
+    "rope_type": "linear"
+  },
+  "rope_theta": 1000000.0,
+  "sliding_window": 1024,
+  "transformers_version": "4.56.1",
+  "use_cache": false,
+  "vocab_size": 262208
+}

cpu_and_mobile/cpu-int4-rtn-block-32/chat_template.jinja

@@ -0,0 +1,22 @@
+{%- set system_msg = messages | selectattr('role', 'eq', 'system') | list | first | default(none) -%}
+{%- set last_user = messages | selectattr('role', 'eq', 'user') | list | last | default(none) -%}
+{%- set last_assistant = messages | selectattr('role', 'eq', 'assistant') | list | last | default(none) -%}
+
+{{- bos_token -}}
+
+{%- if system_msg -%}
+  <start_of_turn>instruction{{ '\n' }}
+  {{- system_msg['content'] | trim -}}<end_of_turn>{{ '\n' }}
+{%- endif -%}
+
+{%- if last_user -%}
+  <start_of_turn>source{{ '\n' }}
+  {{- last_user['content'] | trim -}}<end_of_turn>{{ '\n' }}
+{%- endif -%}
+
+{%- if add_generation_prompt -%}
+  <start_of_turn>translation{{ '\n' }}
+{%- elif last_assistant -%}
+  <start_of_turn>translation{{ '\n' }}
+  {{- last_assistant['content'] | trim -}}<end_of_turn>{{ '\n' }}
+{%- endif -%}

cpu_and_mobile/cpu-int4-rtn-block-32/config.json

@@ -0,0 +1,73 @@
+{
+  "_sliding_window_pattern": 6,
+  "architectures": [
+    "Gemma3ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "attn_logit_softcapping": null,
+  "bos_token_id": 2,
+  "cache_implementation": "hybrid",
+  "dtype": "bfloat16",
+  "eos_token_id": 106,
+  "final_logit_softcapping": null,
+  "head_dim": 256,
+  "hidden_activation": "gelu_pytorch_tanh",
+  "hidden_size": 2560,
+  "initializer_range": 0.02,
+  "intermediate_size": 10240,
+  "layer_types": [
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention"
+  ],
+  "max_position_embeddings": 131072,
+  "model_type": "gemma3_text",
+  "num_attention_heads": 8,
+  "num_hidden_layers": 34,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "query_pre_attn_scalar": 256,
+  "rms_norm_eps": 1e-06,
+  "rope_local_base_freq": 10000.0,
+  "rope_scaling": {
+    "factor": 8.0,
+    "rope_type": "linear"
+  },
+  "rope_theta": 1000000.0,
+  "sliding_window": 1024,
+  "transformers_version": "4.56.1",
+  "use_cache": false,
+  "vocab_size": 262208
+}

cpu_and_mobile/cpu-int4-rtn-block-32/genai_config.json

@@ -0,0 +1,52 @@
+{
+    "model": {
+        "bos_token_id": 2,
+        "context_length": 131072,
+        "decoder": {
+            "session_options": {
+                "log_id": "onnxruntime-genai",
+                "provider_options": []
+            },
+            "filename": "model.onnx",
+            "head_size": 256,
+            "hidden_size": 2560,
+            "inputs": {
+                "input_ids": "input_ids",
+                "attention_mask": "attention_mask",
+                "past_key_names": "past_key_values.%d.key",
+                "past_value_names": "past_key_values.%d.value"
+            },
+            "outputs": {
+                "logits": "logits",
+                "present_key_names": "present.%d.key",
+                "present_value_names": "present.%d.value"
+            },
+            "num_attention_heads": 8,
+            "num_hidden_layers": 34,
+            "num_key_value_heads": 4
+        },
+        "eos_token_id": [
+            1,
+            106
+        ],
+        "pad_token_id": 0,
+        "type": "gemma3_text",
+        "vocab_size": 262208
+    },
+    "search": {
+        "diversity_penalty": 0.0,
+        "do_sample": true,
+        "early_stopping": true,
+        "length_penalty": 1.0,
+        "max_length": 131072,
+        "min_length": 0,
+        "no_repeat_ngram_size": 0,
+        "num_beams": 1,
+        "num_return_sequences": 1,
+        "past_present_share_buffer": true,
+        "repetition_penalty": 1.0,
+        "temperature": 1.0,
+        "top_k": 64,
+        "top_p": 0.95
+    }
+}

cpu_and_mobile/cpu-int4-rtn-block-32/model.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5cc1a93fee23cbfcfce4eb2aba363ec4f478a96ae5ae3108a2f73304a0f7c018
+size 430400

cpu_and_mobile/cpu-int4-rtn-block-32/model.onnx.data

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:69450278d7c190a37c2f8a264395aaaaf634baca561153b291b181d00bf3cb33
+size 5379981312

cpu_and_mobile/cpu-int4-rtn-block-32/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<bos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<end_of_turn>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

cpu_and_mobile/cpu-int4-rtn-block-32/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6b7c9043ba3b559295e6032728ca44ba21879713a32d4a35240794b2ed66d78
+size 33384556

cuda/cuda-int4-rtn-block-32/chat_template.jinja

@@ -0,0 +1,22 @@
+{%- set system_msg = messages | selectattr('role', 'eq', 'system') | list | first | default(none) -%}
+{%- set last_user = messages | selectattr('role', 'eq', 'user') | list | last | default(none) -%}
+{%- set last_assistant = messages | selectattr('role', 'eq', 'assistant') | list | last | default(none) -%}
+
+{{- bos_token -}}
+
+{%- if system_msg -%}
+  <start_of_turn>instruction{{ '\n' }}
+  {{- system_msg['content'] | trim -}}<end_of_turn>{{ '\n' }}
+{%- endif -%}
+
+{%- if last_user -%}
+  <start_of_turn>source{{ '\n' }}
+  {{- last_user['content'] | trim -}}<end_of_turn>{{ '\n' }}
+{%- endif -%}
+
+{%- if add_generation_prompt -%}
+  <start_of_turn>translation{{ '\n' }}
+{%- elif last_assistant -%}
+  <start_of_turn>translation{{ '\n' }}
+  {{- last_assistant['content'] | trim -}}<end_of_turn>{{ '\n' }}
+{%- endif -%}

cuda/cuda-int4-rtn-block-32/config.json

@@ -0,0 +1,73 @@
+{
+  "_sliding_window_pattern": 6,
+  "architectures": [
+    "Gemma3ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "attn_logit_softcapping": null,
+  "bos_token_id": 2,
+  "cache_implementation": "hybrid",
+  "dtype": "bfloat16",
+  "eos_token_id": 106,
+  "final_logit_softcapping": null,
+  "head_dim": 256,
+  "hidden_activation": "gelu_pytorch_tanh",
+  "hidden_size": 2560,
+  "initializer_range": 0.02,
+  "intermediate_size": 10240,
+  "layer_types": [
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention"
+  ],
+  "max_position_embeddings": 131072,
+  "model_type": "gemma3_text",
+  "num_attention_heads": 8,
+  "num_hidden_layers": 34,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "query_pre_attn_scalar": 256,
+  "rms_norm_eps": 1e-06,
+  "rope_local_base_freq": 10000.0,
+  "rope_scaling": {
+    "factor": 8.0,
+    "rope_type": "linear"
+  },
+  "rope_theta": 1000000.0,
+  "sliding_window": 1024,
+  "transformers_version": "4.56.1",
+  "use_cache": false,
+  "vocab_size": 262208
+}

cuda/cuda-int4-rtn-block-32/genai_config.json

@@ -0,0 +1,59 @@
+{
+    "model": {
+        "bos_token_id": 2,
+        "context_length": 131072,
+        "decoder": {
+            "session_options": {
+                "log_id": "onnxruntime-genai",
+                "provider_options": [
+                    {
+                        "cuda": {
+                            "enable_cuda_graph": "0",
+                            "enable_skip_layer_norm_strict_mode": "1"
+                        }
+                    }
+                ]
+            },
+            "filename": "model.onnx",
+            "head_size": 256,
+            "hidden_size": 2560,
+            "inputs": {
+                "input_ids": "input_ids",
+                "attention_mask": "attention_mask",
+                "past_key_names": "past_key_values.%d.key",
+                "past_value_names": "past_key_values.%d.value"
+            },
+            "outputs": {
+                "logits": "logits",
+                "present_key_names": "present.%d.key",
+                "present_value_names": "present.%d.value"
+            },
+            "num_attention_heads": 8,
+            "num_hidden_layers": 34,
+            "num_key_value_heads": 4
+        },
+        "eos_token_id": [
+            1,
+            106
+        ],
+        "pad_token_id": 0,
+        "type": "gemma3_text",
+        "vocab_size": 262208
+    },
+    "search": {
+        "diversity_penalty": 0.0,
+        "do_sample": true,
+        "early_stopping": true,
+        "length_penalty": 1.0,
+        "max_length": 131072,
+        "min_length": 0,
+        "no_repeat_ngram_size": 0,
+        "num_beams": 1,
+        "num_return_sequences": 1,
+        "past_present_share_buffer": true,
+        "repetition_penalty": 1.0,
+        "temperature": 1.0,
+        "top_k": 64,
+        "top_p": 0.95
+    }
+}

cuda/cuda-int4-rtn-block-32/model.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7edc6ea8e90e3b14ba998279152f14d6ea36dc94e0873c4d48ff76f1c363eff0
+size 537053

cuda/cuda-int4-rtn-block-32/model.onnx.data

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f3a0fdc168acde610b4adb1a5f3131141dbe97ab8c35bb6aed568993e444e83
+size 3660775424

cuda/cuda-int4-rtn-block-32/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<bos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<end_of_turn>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

cuda/cuda-int4-rtn-block-32/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6b7c9043ba3b559295e6032728ca44ba21879713a32d4a35240794b2ed66d78
+size 33384556

directml/dml-fp16/chat_template.jinja

@@ -0,0 +1,22 @@
+{%- set system_msg = messages | selectattr('role', 'eq', 'system') | list | first | default(none) -%}
+{%- set last_user = messages | selectattr('role', 'eq', 'user') | list | last | default(none) -%}
+{%- set last_assistant = messages | selectattr('role', 'eq', 'assistant') | list | last | default(none) -%}
+
+{{- bos_token -}}
+
+{%- if system_msg -%}
+  <start_of_turn>instruction{{ '\n' }}
+  {{- system_msg['content'] | trim -}}<end_of_turn>{{ '\n' }}
+{%- endif -%}
+
+{%- if last_user -%}
+  <start_of_turn>source{{ '\n' }}
+  {{- last_user['content'] | trim -}}<end_of_turn>{{ '\n' }}
+{%- endif -%}
+
+{%- if add_generation_prompt -%}
+  <start_of_turn>translation{{ '\n' }}
+{%- elif last_assistant -%}
+  <start_of_turn>translation{{ '\n' }}
+  {{- last_assistant['content'] | trim -}}<end_of_turn>{{ '\n' }}
+{%- endif -%}

directml/dml-fp16/config.json

@@ -0,0 +1,73 @@
+{
+  "_sliding_window_pattern": 6,
+  "architectures": [
+    "Gemma3ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "attn_logit_softcapping": null,
+  "bos_token_id": 2,
+  "cache_implementation": "hybrid",
+  "dtype": "bfloat16",
+  "eos_token_id": 106,
+  "final_logit_softcapping": null,
+  "head_dim": 256,
+  "hidden_activation": "gelu_pytorch_tanh",
+  "hidden_size": 2560,
+  "initializer_range": 0.02,
+  "intermediate_size": 10240,
+  "layer_types": [
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention"
+  ],
+  "max_position_embeddings": 131072,
+  "model_type": "gemma3_text",
+  "num_attention_heads": 8,
+  "num_hidden_layers": 34,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "query_pre_attn_scalar": 256,
+  "rms_norm_eps": 1e-06,
+  "rope_local_base_freq": 10000.0,
+  "rope_scaling": {
+    "factor": 8.0,
+    "rope_type": "linear"
+  },
+  "rope_theta": 1000000.0,
+  "sliding_window": 1024,
+  "transformers_version": "4.56.1",
+  "use_cache": false,
+  "vocab_size": 262208
+}

directml/dml-fp16/genai_config.json

@@ -0,0 +1,57 @@
+{
+    "model": {
+        "bos_token_id": 2,
+        "context_length": 131072,
+        "decoder": {
+            "session_options": {
+                "log_id": "onnxruntime-genai",
+                "provider_options": [
+                    {
+                        "dml": {}
+                    }
+                ]
+            },
+            "filename": "model.onnx",
+            "head_size": 256,
+            "hidden_size": 2560,
+            "inputs": {
+                "input_ids": "input_ids",
+                "attention_mask": "attention_mask",
+                "position_ids": "position_ids",
+                "past_key_names": "past_key_values.%d.key",
+                "past_value_names": "past_key_values.%d.value"
+            },
+            "outputs": {
+                "logits": "logits",
+                "present_key_names": "present.%d.key",
+                "present_value_names": "present.%d.value"
+            },
+            "num_attention_heads": 8,
+            "num_hidden_layers": 34,
+            "num_key_value_heads": 4
+        },
+        "eos_token_id": [
+            1,
+            106
+        ],
+        "pad_token_id": 0,
+        "type": "gemma3_text",
+        "vocab_size": 262208
+    },
+    "search": {
+        "diversity_penalty": 0.0,
+        "do_sample": true,
+        "early_stopping": true,
+        "length_penalty": 1.0,
+        "max_length": 131072,
+        "min_length": 0,
+        "no_repeat_ngram_size": 0,
+        "num_beams": 1,
+        "num_return_sequences": 1,
+        "past_present_share_buffer": true,
+        "repetition_penalty": 1.0,
+        "temperature": 1.0,
+        "top_k": 64,
+        "top_p": 0.95
+    }
+}

directml/dml-fp16/model.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98e279a791a3a08b72fda21a582ae7e0f28ea372c2db08ccaf313a1af9f49322
+size 477903

directml/dml-fp16/model.onnx.data

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d9f429ceb0600cfb3a33998def33d99eadb95ba2a32d45408507b815f41cc88
+size 7895515136

directml/dml-fp16/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<bos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<end_of_turn>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

directml/dml-fp16/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6b7c9043ba3b559295e6032728ca44ba21879713a32d4a35240794b2ed66d78
+size 33384556