Upload tokenizer

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

chat_template.jinja

@@ -0,0 +1,61 @@
+{%- macro flint_text(t) -%}
+{%- if t is string -%}
+{{- t | replace("\x00", "") | replace("<think>", "") | replace("</think>", "") | replace("<|begin_of_text|>", "&lt;|begin_of_text|&gt;") | replace("<|end_of_text|>", "&lt;|end_of_text|&gt;") | replace("<|system|>", "&lt;|system|&gt;") | replace("<|developer|>", "&lt;|developer|&gt;") | replace("<|user|>", "&lt;|user|&gt;") | replace("<|assistant|>", "&lt;|assistant|&gt;") | replace("<|observation|>", "&lt;|observation|&gt;") -}}
+{%- else -%}
+{{- t -}}
+{%- endif -%}
+{%- endmacro -%}
+{%- set tool_preamble -%}
+{%- if tools %}<|system|>
+# Tools
+
+You may call one or more functions to assist with the user query.
+
+You are provided with function signatures within <tools></tools> XML tags:
+<tools>
+{%- for tool in tools %}
+{%- set fn = tool.function if tool.function is defined else tool %}
+{{ fn | tojson(ensure_ascii=False) }}
+{%- endfor %}
+</tools>
+
+For each function call, output the function name and arguments within the following XML format:
+<tool_call>{function-name}<arg_key>{arg-key-1}</arg_key><arg_value>{arg-value-1}</arg_value>...</tool_call>
+{% endif -%}
+{%- endset -%}
+{%- set ns = namespace(tools_pending=true) -%}
+{%- for m in messages -%}
+{%- if ns.tools_pending and m.role != 'system' -%}
+{{ tool_preamble }}
+{%- set ns.tools_pending = false -%}
+{%- endif -%}
+{%- if m.role == 'system' -%}
+<|system|>{{ flint_text(m.content) }}
+{%- elif m.role == 'developer' -%}
+<|developer|>{{ flint_text(m.content) }}
+{%- elif m.role == 'user' -%}
+<|user|>{{ flint_text(m.content) }}
+{%- elif m.role == 'assistant' -%}
+<|assistant|>
+{%- if m.reasoning_content %}<think>{{ m.reasoning_content }}</think>{% endif -%}
+{%- if m.content %}{{ m.content }}{% endif -%}
+{%- if m.tool_calls -%}
+{%- for tc in m.tool_calls -%}
+{%- set fn = tc.function if tc.function is defined else tc -%}
+<tool_call>{{ fn.name }}
+{%- for k, v in fn.arguments.items() -%}
+<arg_key>{{ k }}</arg_key><arg_value>{{ v | tojson(ensure_ascii=False) if v is not string else v }}</arg_value>
+{%- endfor -%}
+</tool_call>
+{%- endfor -%}
+{%- endif -%}
+{%- elif m.role == 'tool' -%}
+<|observation|><tool_response>{{ flint_text(m.content) }}</tool_response>
+{%- endif -%}
+{%- endfor -%}
+{%- if ns.tools_pending -%}
+{{ tool_preamble }}
+{%- endif -%}
+{%- if add_generation_prompt -%}
+<|assistant|>
+{%- endif -%}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d9a2c5dfe406d46bbc644bba1a52c14931ea076058d44d51135facf8d7f821e
+size 10879677

tokenizer_config.json

@@ -0,0 +1,143 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|end_of_text|>",
+  "extra_special_tokens": [
+    "<|system|>",
+    "<|user|>",
+    "<|assistant|>",
+    "<|observation|>",
+    "<|developer|>",
+    "<|reserved_150019|>",
+    "<|reserved_150020|>",
+    "<|reserved_150021|>",
+    "<|reserved_150022|>",
+    "<|reserved_150023|>",
+    "<|reserved_150024|>",
+    "<|reserved_150025|>",
+    "<|reserved_150026|>",
+    "<|reserved_150027|>",
+    "<|reserved_150028|>",
+    "<|reserved_150029|>",
+    "<|reserved_150030|>",
+    "<|reserved_150031|>",
+    "<|reserved_150032|>",
+    "<|reserved_150033|>",
+    "<|reserved_150034|>",
+    "<|reserved_150035|>",
+    "<|reserved_150036|>",
+    "<|reserved_150037|>",
+    "<|reserved_150038|>",
+    "<|reserved_150039|>",
+    "<|reserved_150040|>",
+    "<|reserved_150041|>",
+    "<|reserved_150042|>",
+    "<|reserved_150043|>",
+    "<|reserved_150044|>",
+    "<|reserved_150045|>",
+    "<|reserved_150046|>",
+    "<|reserved_150047|>",
+    "<|reserved_150048|>",
+    "<|reserved_150049|>",
+    "<|reserved_150050|>",
+    "<|reserved_150051|>",
+    "<|reserved_150052|>",
+    "<|reserved_150053|>",
+    "<|reserved_150054|>",
+    "<|reserved_150055|>",
+    "<|reserved_150056|>",
+    "<|reserved_150057|>",
+    "<|reserved_150058|>",
+    "<|reserved_150059|>",
+    "<|reserved_150060|>",
+    "<|reserved_150061|>",
+    "<|reserved_150062|>",
+    "<|reserved_150063|>",
+    "<|reserved_150064|>",
+    "<|reserved_150065|>",
+    "<|reserved_150066|>",
+    "<|reserved_150067|>",
+    "<|reserved_150068|>",
+    "<|reserved_150069|>",
+    "<|reserved_150070|>",
+    "<|reserved_150071|>",
+    "<|reserved_150072|>",
+    "<|reserved_150073|>",
+    "<|reserved_150074|>",
+    "<|reserved_150075|>",
+    "<|reserved_150076|>",
+    "<|reserved_150077|>",
+    "<|reserved_150078|>",
+    "<|reserved_150079|>",
+    "<|reserved_150080|>",
+    "<|reserved_150081|>",
+    "<|reserved_150082|>",
+    "<|reserved_150083|>",
+    "<|reserved_150084|>",
+    "<|reserved_150085|>",
+    "<|reserved_150086|>",
+    "<|reserved_150087|>",
+    "<|reserved_150088|>",
+    "<|reserved_150089|>",
+    "<|reserved_150090|>",
+    "<|reserved_150091|>",
+    "<|reserved_150092|>",
+    "<|reserved_150093|>",
+    "<|reserved_150094|>",
+    "<|reserved_150095|>",
+    "<|reserved_150096|>",
+    "<|reserved_150097|>",
+    "<|reserved_150098|>",
+    "<|reserved_150099|>",
+    "<|reserved_150100|>",
+    "<|reserved_150101|>",
+    "<|reserved_150102|>",
+    "<|reserved_150103|>",
+    "<|reserved_150104|>",
+    "<|reserved_150105|>",
+    "<|reserved_150106|>",
+    "<|reserved_150107|>",
+    "<|reserved_150108|>",
+    "<|reserved_150109|>",
+    "<|reserved_150110|>",
+    "<|reserved_150111|>",
+    "<|reserved_150112|>",
+    "<|reserved_150113|>",
+    "<|reserved_150114|>",
+    "<|reserved_150115|>",
+    "<|reserved_150116|>",
+    "<|reserved_150117|>",
+    "<|reserved_150118|>",
+    "<|reserved_150119|>",
+    "<|reserved_150120|>",
+    "<|reserved_150121|>",
+    "<|reserved_150122|>",
+    "<|reserved_150123|>",
+    "<|reserved_150124|>",
+    "<|reserved_150125|>",
+    "<|reserved_150126|>",
+    "<|reserved_150127|>",
+    "<|reserved_150128|>",
+    "<|reserved_150129|>",
+    "<|reserved_150130|>",
+    "<|reserved_150131|>",
+    "<|reserved_150132|>",
+    "<|reserved_150133|>",
+    "<|reserved_150134|>",
+    "<|reserved_150135|>",
+    "<|reserved_150136|>",
+    "<|reserved_150137|>",
+    "<|reserved_150138|>",
+    "<|reserved_150139|>",
+    "<|reserved_150140|>",
+    "<|reserved_150141|>",
+    "<|reserved_150142|>",
+    "<|reserved_150143|>"
+  ],
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 262144,
+  "pad_token": "<|end_of_text|>",
+  "tokenizer_class": "TokenizersBackend"
+}