gemma-4-E4B-it-qat-int4-ov / openvino_detokenizer.xml

Initial release: first OpenVINO IR of Gemma 4 E4B QAT (int4 sym g32, q4_0-matched, probe-validated)

f3e82a2 verified about 2 months ago

28.1 kB

	<?xml version="1.0"?>
	<net name="detokenizer" version="11">
	<layers>
	<layer id="0" name="Parameter_209766" type="Parameter" version="opset1">
	<data shape="?,?" element_type="i64" />
	<output>
	<port id="0" precision="I64" names="Parameter_209766">
	<dim>-1</dim>
	<dim>-1</dim>
	</port>
	</output>
	</layer>
	<layer id="1" name="Convert_209801" type="Convert" version="opset1">
	<data destination_type="i32" />
	<input>
	<port id="0" precision="I64">
	<dim>-1</dim>
	<dim>-1</dim>
	</port>
	</input>
	<output>
	<port id="1" precision="I32">
	<dim>-1</dim>
	<dim>-1</dim>
	</port>
	</output>
	</layer>
	<layer id="2" name="Constant_209722" type="Const" version="opset1">
	<data element_type="i32" shape="262144" offset="0" size="1048576" />
	<output>
	<port id="0" precision="I32">
	<dim>262144</dim>
	</port>
	</output>
	</layer>
	<layer id="3" name="Constant_209724" type="Const" version="opset1">
	<data element_type="i32" shape="262144" offset="1048576" size="1048576" />
	<output>
	<port id="0" precision="I32">
	<dim>262144</dim>
	</port>
	</output>
	</layer>
	<layer id="4" name="Constant_209726" type="Const" version="opset1">
	<data element_type="u8" shape="2316295" offset="2097152" size="2316295" />
	<output>
	<port id="0" precision="U8">
	<dim>2316295</dim>
	</port>
	</output>
	</layer>
	<layer id="5" name="Slice_209771" type="Const" version="opset1">
	<data element_type="i32" shape="24" offset="4413447" size="96" />
	<output>
	<port id="0" precision="I32">
	<dim>24</dim>
	</port>
	</output>
	</layer>
	<layer id="6" name="VocabDecoder_209773" type="VocabDecoder" version="extension">
	<data skip_tokens="" />
	<input>
	<port id="0" precision="I32">
	<dim>-1</dim>
	<dim>-1</dim>
	</port>
	<port id="1" precision="I32">
	<dim>262144</dim>
	</port>
	<port id="2" precision="I32">
	<dim>262144</dim>
	</port>
	<port id="3" precision="U8">
	<dim>2316295</dim>
	</port>
	<port id="4" precision="I32">
	<dim>24</dim>
	</port>
	</input>
	<output>
	<port id="5" precision="I32">
	<dim>-1</dim>
	</port>
	<port id="6" precision="I32">
	<dim>-1</dim>
	</port>
	<port id="7" precision="I32">
	<dim>-1</dim>
	</port>
	<port id="8" precision="I32">
	<dim>-1</dim>
	</port>
	<port id="9" precision="U8">
	<dim>-1</dim>
	</port>
	</output>
	</layer>
	<layer id="7" name="Constant_209775" type="Const" version="opset1">
	<data element_type="u8" shape="3" offset="4413543" size="3" />
	<output>
	<port id="0" precision="U8">
	<dim>3</dim>
	</port>
	</output>
	</layer>
	<layer id="8" name="Constant_209777" type="Const" version="opset1">
	<data element_type="u8" shape="1" offset="4413546" size="1" />
	<output>
	<port id="0" precision="U8">
	<dim>1</dim>
	</port>
	</output>
	</layer>
	<layer id="9" name="RegexNormalization_209778" type="RegexNormalization" version="extension">
	<data global_replace="true" />
	<input>
	<port id="0" precision="I32">
	<dim>-1</dim>
	</port>
	<port id="1" precision="I32">
	<dim>-1</dim>
	</port>
	<port id="2" precision="U8">
	<dim>-1</dim>
	</port>
	<port id="3" precision="U8">
	<dim>3</dim>
	</port>
	<port id="4" precision="U8">
	<dim>1</dim>
	</port>
	</input>
	<output>
	<port id="5" precision="I32">
	<dim>-1</dim>
	</port>
	<port id="6" precision="I32">
	<dim>-1</dim>
	</port>
	<port id="7" precision="U8">
	<dim>-1</dim>
	</port>
	</output>
	</layer>
	<layer id="10" name="ByteFallback_209779" type="ByteFallback" version="extension">
	<input>
	<port id="0" precision="I32">
	<dim>-1</dim>
	</port>
	<port id="1" precision="I32">
	<dim>-1</dim>
	</port>
	<port id="2" precision="U8">
	<dim>-1</dim>
	</port>
	</input>
	<output>
	<port id="3" precision="I32">
	<dim>-1</dim>
	</port>
	<port id="4" precision="I32">
	<dim>-1</dim>
	</port>
	<port id="5" precision="U8">
	<dim>-1</dim>
	</port>
	</output>
	</layer>
	<layer id="11" name="FuzeRagged_209780" type="FuzeRagged" version="extension">
	<input>
	<port id="0" precision="I32">
	<dim>-1</dim>
	</port>
	<port id="1" precision="I32">
	<dim>-1</dim>
	</port>
	<port id="2" precision="I32">
	<dim>-1</dim>
	</port>
	<port id="3" precision="I32">
	<dim>-1</dim>
	</port>
	</input>
	<output>
	<port id="4" precision="I32">
	<dim>-1</dim>
	</port>
	<port id="5" precision="I32">
	<dim>-1</dim>
	</port>
	</output>
	</layer>
	<layer id="12" name="UTF8Validate_209781" type="UTF8Validate" version="extension">
	<data replace_mode="true" />
	<input>
	<port id="0" precision="I32">
	<dim>-1</dim>
	</port>
	<port id="1" precision="I32">
	<dim>-1</dim>
	</port>
	<port id="2" precision="U8">
	<dim>-1</dim>
	</port>
	</input>
	<output>
	<port id="3" precision="I32">
	<dim>-1</dim>
	</port>
	<port id="4" precision="I32">
	<dim>-1</dim>
	</port>
	<port id="5" precision="U8">
	<dim>-1</dim>
	</port>
	</output>
	</layer>
	<layer id="13" name="StringTensorPack_209782" type="StringTensorPack" version="opset15">
	<input>
	<port id="0" precision="I32">
	<dim>-1</dim>
	</port>
	<port id="1" precision="I32">
	<dim>-1</dim>
	</port>
	<port id="2" precision="U8">
	<dim>-1</dim>
	</port>
	</input>
	<output>
	<port id="3" precision="STRING" names="Result_209783,string_output">
	<dim>-1</dim>
	</port>
	</output>
	</layer>
	<layer id="14" name="Result_209783" type="Result" version="opset1" output_names="Result_209783,string_output">
	<input>
	<port id="0" precision="STRING">
	<dim>-1</dim>
	</port>
	</input>
	</layer>
	</layers>
	<edges>
	<edge from-layer="0" from-port="0" to-layer="1" to-port="0" />
	<edge from-layer="1" from-port="1" to-layer="6" to-port="0" />
	<edge from-layer="2" from-port="0" to-layer="6" to-port="1" />
	<edge from-layer="3" from-port="0" to-layer="6" to-port="2" />
	<edge from-layer="4" from-port="0" to-layer="6" to-port="3" />
	<edge from-layer="5" from-port="0" to-layer="6" to-port="4" />
	<edge from-layer="6" from-port="7" to-layer="9" to-port="0" />
	<edge from-layer="6" from-port="8" to-layer="9" to-port="1" />
	<edge from-layer="6" from-port="9" to-layer="9" to-port="2" />
	<edge from-layer="6" from-port="5" to-layer="11" to-port="0" />
	<edge from-layer="6" from-port="6" to-layer="11" to-port="1" />
	<edge from-layer="7" from-port="0" to-layer="9" to-port="3" />
	<edge from-layer="8" from-port="0" to-layer="9" to-port="4" />
	<edge from-layer="9" from-port="5" to-layer="10" to-port="0" />
	<edge from-layer="9" from-port="6" to-layer="10" to-port="1" />
	<edge from-layer="9" from-port="7" to-layer="10" to-port="2" />
	<edge from-layer="10" from-port="3" to-layer="11" to-port="2" />
	<edge from-layer="10" from-port="4" to-layer="11" to-port="3" />
	<edge from-layer="10" from-port="5" to-layer="12" to-port="2" />
	<edge from-layer="11" from-port="4" to-layer="12" to-port="0" />
	<edge from-layer="11" from-port="5" to-layer="12" to-port="1" />
	<edge from-layer="12" from-port="3" to-layer="13" to-port="0" />
	<edge from-layer="12" from-port="4" to-layer="13" to-port="1" />
	<edge from-layer="12" from-port="5" to-layer="13" to-port="2" />
	<edge from-layer="13" from-port="3" to-layer="14" to-port="0" />
	</edges>
	<rt_info>
	<info name="OpenVINO Runtime" value="2026.3.0-22098-367f60d8408" />
	<add_attention_mask value="True" />
	<add_prefix_space />
	<add_special_tokens value="True" />
	<bos_token_id value="2" />
	<chat_template value="{%- macro format_parameters(properties, required, filter_keys=false) -%} {%- set standard_keys = ['description', 'type', 'properties', 'required', 'nullable'] -%} {%- set ns = namespace(found_first=false) -%} {%- for key, value in properties \| dictsort -%} {%- set add_comma = false -%} {%- if not filter_keys or key not in standard_keys -%} {%- if ns.found_first %},{% endif -%} {%- set ns.found_first = true -%} {{ key }}:{ {%- if value['description'] -%} description:<\|"\|>{{ value['description'] }}<\|"\|> {%- set add_comma = true -%} {%- endif -%} {%- if value['type'] \| upper == 'STRING' -%} {%- if value['enum'] -%} {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} enum:{{ format_argument(value['enum']) }} {%- endif -%} {%- elif value['type'] \| upper == 'ARRAY' -%} {%- if value['items'] is mapping and value['items'] -%} {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} items:{ {%- set ns_items = namespace(found_first=false) -%} {%- for item_key, item_value in value['items'] \| dictsort -%} {%- if item_value is not none -%} {%- if ns_items.found_first %},{% endif -%} {%- set ns_items.found_first = true -%} {%- if item_key == 'properties' -%} properties:{ {%- if item_value is mapping -%} {{- format_parameters(item_value, value['items']['required'] \| default([])) -}} {%- endif -%} } {%- elif item_key == 'required' -%} required:[ {%- for req_item in item_value -%} <\|"\|>{{- req_item -}}<\|"\|> {%- if not loop.last %},{% endif -%} {%- endfor -%} ] {%- elif item_key == 'type' -%} {%- if item_value is string -%} type:{{ format_argument(item_value \| upper) }} {%- else -%} type:{{ format_argument(item_value \| map('upper') \| list) }} {%- endif -%} {%- else -%} {{ item_key }}:{{ format_argument(item_value) }} {%- endif -%} {%- endif -%} {%- endfor -%} } {%- endif -%} {%- endif -%} {%- if value['nullable'] %} {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} nullable:true {%- endif -%} {%- if value['type'] \| upper == 'OBJECT' -%} {%- if value['properties'] is defined and value['properties'] is mapping -%} {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} properties:{ {{- format_parameters(value['properties'], value['required'] \| default([])) -}} } {%- elif value is mapping -%} {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} properties:{ {{- format_parameters(value, value['required'] \| default([]), filter_keys=true) -}} } {%- endif -%} {%- if value['required'] -%} {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} required:[ {%- for item in value['required'] \| default([]) -%} <\|"\|>{{- item -}}<\|"\|> {%- if not loop.last %},{% endif -%} {%- endfor -%} ] {%- endif -%} {%- endif -%} {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%} type:<\|"\|>{{ value['type'] \| upper }}<\|"\|>} {%- endif -%} {%- endfor -%} {%- endmacro -%} {%- macro format_function_declaration(tool_data) -%} declaration:{{- tool_data['function']['name'] -}}{description:<\|"\|>{{- tool_data['function']['description'] -}}<\|"\|> {%- set params = tool_data['function']['parameters'] -%} {%- if params -%} ,parameters:{ {%- if params['properties'] -%} properties:{ {{- format_parameters(params['properties'], params['required']) -}} }, {%- endif -%} {%- if params['required'] -%} required:[ {%- for item in params['required'] -%} <\|"\|>{{- item -}}<\|"\|> {{- ',' if not loop.last -}} {%- endfor -%} ], {%- endif -%} {%- if params['type'] -%} type:<\|"\|>{{- params['type'] \| upper -}}<\|"\|>} {%- endif -%} {%- endif -%} {%- if 'response' in tool_data['function'] -%} {%- set response_declaration = tool_data['function']['response'] -%} ,response:{ {%- if response_declaration['description'] -%} description:<\|"\|>{{- response_declaration['description'] -}}<\|"\|>, {%- endif -%} {%- if response_declaration['type'] \| upper == 'OBJECT' -%} type:<\|"\|>{{- response_declaration['type'] \| upper -}}<\|"\|>} {%- endif -%} {%- endif -%} } {%- endmacro -%} {%- macro format_argument(argument, escape_keys=True) -%} {%- if argument is string -%} {{- '<\|"\|>' + argument + '<\|"\|>' -}} {%- elif argument is boolean -%} {{- 'true' if argument else 'false' -}} {%- elif argument is mapping -%} {{- '{' -}} {%- set ns = namespace(found_first=false) -%} {%- for key, value in argument \| dictsort -%} {%- if ns.found_first %},{% endif -%} {%- set ns.found_first = true -%} {%- if escape_keys -%} {{- '<\|"\|>' + key + '<\|"\|>' -}} {%- else -%} {{- key -}} {%- endif -%} :{{- format_argument(value, escape_keys=escape_keys) -}} {%- endfor -%} {{- '}' -}} {%- elif argument is sequence -%} {{- '[' -}} {%- for item in argument -%} {{- format_argument(item, escape_keys=escape_keys) -}} {%- if not loop.last %},{% endif -%} {%- endfor -%} {{- ']' -}} {%- else -%} {{- argument -}} {%- endif -%} {%- endmacro -%} {%- macro strip_thinking(text) -%} {%- set ns = namespace(result='') -%} {%- for part in text.split('<channel\|>') -%} {%- if '<\|channel>' in part -%} {%- set ns.result = ns.result + part.split('<\|channel>')[0] -%} {%- else -%} {%- set ns.result = ns.result + part -%} {%- endif -%} {%- endfor -%} {{- ns.result \| trim -}} {%- endmacro -%} {%- macro format_tool_response_block(tool_name, response) -%} {{- '<\|tool_response>' -}} {%- if response is mapping -%} {{- 'response:' + tool_name + '{' -}} {%- for key, value in response \| dictsort -%} {{- key -}}:{{- format_argument(value, escape_keys=False) -}} {%- if not loop.last %},{% endif -%} {%- endfor -%} {{- '}' -}} {%- else -%} {{- 'response:' + tool_name + '{value:' + format_argument(response, escape_keys=False) + '}' -}} {%- endif -%} {{- '<tool_response\|>' -}} {%- endmacro -%} {%- set ns = namespace(prev_message_type=None) -%} {%- set loop_messages = messages -%} {{- bos_token -}} {#- Handle System/Tool Definitions Block -#} {%- if (enable_thinking is defined and enable_thinking) or tools or messages[0]['role'] in ['system', 'developer'] -%} {{- '<\|turn>system\n' -}} {#- Inject Thinking token at the very top of the FIRST system turn -#} {%- if enable_thinking is defined and enable_thinking -%} {{- '<\|think\|>\n' -}} {%- set ns.prev_message_type = 'think' -%} {%- endif -%} {%- if messages[0]['role'] in ['system', 'developer'] -%} {%- if messages[0]['content'] is string -%} {{- messages[0]['content'] \| trim -}} {%- elif messages[0]['content'] is sequence -%} {%- for item in messages[0]['content'] -%} {{- item['text'] \| trim + ' '-}} {%- endfor -%} {%- endif -%} {%- set loop_messages = messages[1:] -%} {%- endif -%} {%- if tools -%} {%- for tool in tools %} {{- '<\|tool>' -}} {{- format_function_declaration(tool) \| trim -}} {{- '<tool\|>' -}} {%- endfor %} {%- set ns.prev_message_type = 'tool' -%} {%- endif -%} {{- '<turn\|>\n' -}} {%- endif %} {#- Pre-scan: find last user message index for reasoning guard -#} {%- set ns_turn = namespace(last_user_idx=-1) -%} {%- for i in range(loop_messages \| length) -%} {%- if loop_messages[i]['role'] == 'user' -%} {%- set ns_turn.last_user_idx = i -%} {%- endif -%} {%- endfor -%} {#- Loop through messages -#} {%- for message in loop_messages -%} {%- if message['role'] != 'tool' -%} {%- set ns.prev_message_type = None -%} {%- set role = 'model' if message['role'] == 'assistant' else message['role'] -%} {#- Detect continuation: suppress duplicate <\|turn>model when previous non-tool message was also assistant -#} {%- set prev_nt = namespace(role=None, found=false) -%} {%- if loop.index0 > 0 -%} {%- for j in range(loop.index0 - 1, -1, -1) -%} {%- if not prev_nt.found -%} {%- if loop_messages[j]['role'] != 'tool' -%} {%- set prev_nt.role = loop_messages[j]['role'] -%} {%- set prev_nt.found = true -%} {%- endif -%} {%- endif -%} {%- endfor -%} {%- endif -%} {%- set continue_same_model_turn = (role == 'model' and prev_nt.role == 'assistant') -%} {%- if not continue_same_model_turn -%} {{- '<\|turn>' + role + '\n' }} {%- endif -%} {#- Render reasoning/reasoning_content as thinking channel -#} {%- set thinking_text = message.get('reasoning') or message.get('reasoning_content') -%} {%- if thinking_text and loop.index0 > ns_turn.last_user_idx and message.get('tool_calls') -%} {{- '<\|channel>thought\n' + thinking_text + '\n<channel\|>' -}} {%- endif -%} {%- if message['tool_calls'] -%} {%- for tool_call in message['tool_calls'] -%} {%- set function = tool_call['function'] -%} {{- '<\|tool_call>call:' + function['name'] + '{' -}} {%- if function['arguments'] is mapping -%} {%- set ns_args = namespace(found_first=false) -%} {%- for key, value in function['arguments'] \| dictsort -%} {%- if ns_args.found_first %},{% endif -%} {%- set ns_args.found_first = true -%} {{- key -}}:{{- format_argument(value, escape_keys=False) -}} {%- endfor -%} {%- elif function['arguments'] is string -%} {{- function['arguments'] -}} {%- endif -%} {{- '}<tool_call\|>' -}} {%- endfor -%} {%- set ns.prev_message_type = 'tool_call' -%} {%- endif -%} {%- set ns_tr_out = namespace(flag=false) -%} {%- if message.get('tool_responses') -%} {#- Legacy: tool_responses embedded on the assistant message (Google/Gemma native) -#} {%- for tool_response in message['tool_responses'] -%} {{- format_tool_response_block(tool_response['name'] \| default('unknown'), tool_response['response']) -}} {%- set ns_tr_out.flag = true -%} {%- set ns.prev_message_type = 'tool_response' -%} {%- endfor -%} {%- elif message.get('tool_calls') -%} {#- OpenAI Chat Completions: forward-scan consecutive role:tool messages -#} {%- set ns_tool_scan = namespace(stopped=false) -%} {%- for k in range(loop.index0 + 1, loop_messages \| length) -%} {%- if ns_tool_scan.stopped -%} {%- elif loop_messages[k]['role'] != 'tool' -%} {%- set ns_tool_scan.stopped = true -%} {%- else -%} {%- set follow = loop_messages[k] -%} {#- Resolve tool_call_id to function name -#} {%- set ns_tname = namespace(name=follow.get('name') \| default('unknown')) -%} {%- for tc in message['tool_calls'] -%} {%- if tc.get('id') == follow.get('tool_call_id') -%} {%- set ns_tname.name = tc['function']['name'] -%} {%- endif -%} {%- endfor -%} {#- Handle content as string or content-parts array -#} {%- set tool_body = follow.get('content') -%} {%- if tool_body is string -%} {{- format_tool_response_block(ns_tname.name, tool_body) -}} {%- elif tool_body is sequence and tool_body is not string -%} {%- set ns_txt = namespace(s='') -%} {%- for part in tool_body -%} {%- if part.get('type') == 'text' -%} {%- set ns_txt.s = ns_txt.s + (part.get('text') \| default('')) -%} {%- endif -%} {%- endfor -%} {{- format_tool_response_block(ns_tname.name, ns_txt.s) -}} {%- else -%} {{- format_tool_response_block(ns_tname.name, tool_body) -}} {%- endif -%} {%- set ns_tr_out.flag = true -%} {%- set ns.prev_message_type = 'tool_response' -%} {%- endif -%} {%- endfor -%} {%- endif -%} {%- set captured_content -%} {%- if message['content'] is string -%} {%- if role == 'model' -%} {{- strip_thinking(message['content']) -}} {%- else -%} {{- message['content'] \| trim -}} {%- endif -%} {%- elif message['content'] is sequence -%} {%- for item in message['content'] -%} {%- if item['type'] == 'text' -%} {%- if role == 'model' -%} {{- strip_thinking(item['text']) -}} {%- else -%} {{- item['text'] \| trim -}} {%- endif -%} {%- elif item['type'] == 'image' -%} {{- '<\|image\|>' -}} {%- set ns.prev_message_type = 'image' -%} {%- elif item['type'] == 'audio' -%} {{- '<\|audio\|>' -}} {%- set ns.prev_message_type = 'audio' -%} {%- elif item['type'] == 'video' -%} {{- '<\|video\|>' -}} {%- set ns.prev_message_type = 'video' -%} {%- endif -%} {%- endfor -%} {%- endif -%} {%- endset -%} {{- captured_content -}} {%- set has_content = captured_content \| trim \| length > 0 -%} {%- if ns.prev_message_type == 'tool_call' and not ns_tr_out.flag -%} {{- '<\|tool_response>' -}} {%- elif not (ns_tr_out.flag and not has_content) -%} {{- '<turn\|>\n' -}} {%- endif -%} {%- endif -%} {%- endfor -%} {%- if add_generation_prompt -%} {%- if ns.prev_message_type != 'tool_response' and ns.prev_message_type != 'tool_call' -%} {{- '<\|turn>model\n' -}} {%- endif -%} {%- endif -%}" />
	<clean_up_tokenization_spaces />
	<detokenizer_input_type value="i64" />
	<eos_token_id value="1" />
	<handle_special_tokens_with_re />
	<max_length />
	<number_of_inputs value="1" />
	<openvino_tokenizers_version value="2026.3.0.0-683-6d28e696dfa" />
	<openvino_version value="2026.3.0-22098-367f60d8408" />
	<original_post_processor_template value="{"type": "TemplateProcessing", "single": [{"Sequence": {"id": "A", "type_id": 0}}], "pair": [{"Sequence": {"id": "A", "type_id": 0}}, {"Sequence": {"id": "B", "type_id": 1}}], "special_tokens": {}}" />
	<original_tokenizer_class value="<class 'transformers.models.gemma.tokenization_gemma.GemmaTokenizer'>" />
	<pad_token_id value="0" />
	<processed_post_processor_template value="{"single": {"ids": [-1], "type_ids": [0]}, "pair": {"ids": [-1, -2], "type_ids": [0, 1]}}" />
	<skip_special_tokens value="True" />
	<streaming_detokenizer value="False" />
	<tiktoken_version value="0.13.0" />
	<tokenizer_output_type value="i64" />
	<tokenizers_version value="0.23.0rc0" />
	<transformers_version value="5.5.0" />
	<use_max_padding value="False" />
	<use_sentencepiece_backend value="False" />
	<utf8_replace_mode value="replace" />
	<with_detokenizer value="True" />
	</rt_info>
	</net>